diff --git a/app/src/features/browser/BrowserPanel.tsx b/app/src/features/browser/BrowserPanel.tsx index 96123989..896be730 100644 --- a/app/src/features/browser/BrowserPanel.tsx +++ b/app/src/features/browser/BrowserPanel.tsx @@ -20,6 +20,7 @@ import { ComputerDock } from "../terminal/ComputerDock"; import { AddressBar } from "./AddressBar"; import { AskCallout, ControlPill, HeldNotes, TakeNotes } from "./HandsBars"; import { HandsLayer } from "./HandsLayer"; +import { LessonArea, LessonDialog, useLesson } from "./Lesson"; import { LiveView, useCaption } from "./LiveView"; import { TabStrip } from "./TabStrip"; import { useBrowserView } from "./useBrowserView"; @@ -60,6 +61,7 @@ export function BrowserPanel({ // Nobody uses it: its pages stand still until someone does (spec 21.2). const resting = status === "open" && state?.resting === true; const hands = useHands(bot, state); + const lesson = useLesson(bot, state); const [focused, setFocused] = useState(false); const ask = state?.ask ?? null; const windowed = state?.window === true; @@ -200,7 +202,7 @@ export function BrowserPanel({ {(hands.held || (live && !ask)) && ( - + )} {status === "open" && !hands.held && ( // Its place is kept while empty, so the page never moves for it. @@ -213,7 +215,8 @@ export function BrowserPanel({ )}

)} - {hands.held && ( + {hands.held && } + {hands.held && lesson.steps === null && ( )} + {lesson.done && } ); } diff --git a/app/src/features/browser/HandsBars.tsx b/app/src/features/browser/HandsBars.tsx index 130f93de..c685197a 100644 --- a/app/src/features/browser/HandsBars.tsx +++ b/app/src/features/browser/HandsBars.tsx @@ -4,13 +4,14 @@ // it, what the owner should know, and a window of its own for sites that // refuse to sign in here (spec 21.11). -import { Check, ChevronDown, Hand, Keyboard, MousePointerClick } from "lucide-react"; +import { Check, ChevronDown, GraduationCap, Hand, Keyboard, MousePointerClick } from "lucide-react"; import { useState } from "react"; import { useT } from "../../i18n"; import type { Bot } from "../../lib/protocol.gen"; import { Button } from "../../ui/Button"; import { Callout } from "../../ui/Callout"; import { BotAvatar, moodOf } from "../bots/BotAvatar"; +import type { Lesson } from "./Lesson"; import type { Hands } from "./useHands"; import { WindowButton } from "./WindowBar"; @@ -43,9 +44,20 @@ export function AskCallout({ bot, task, hands }: { bot: Bot; task: string; hands * Who is in control, hanging from the bottom of the live screen: the bot, * with the button to take it, or the owner, with the one to give it back. */ -export function ControlPill({ bot, hands }: { bot: Bot; hands: Hands }) { - const t = useT().browser.hands; +export function ControlPill({ + bot, + hands, + lesson, +}: { + bot: Bot; + hands: Hands; + /** While the owner teaches, the pill ends the lesson first (spec 21.13). */ + lesson?: Lesson; +}) { + const words = useT().browser; + const t = words.hands; const held = hands.held; + const teaching = held && lesson?.steps != null; return (
void (held ? hands.release() : hands.take())} + onClick={() => void (teaching ? lesson?.finish() : held ? hands.release() : hands.take())} className="min-w-0 shrink! rounded-full! px-3!" > - {held ? t.giveBack(bot.name) : t.take} + + {teaching ? words.lesson.finish : held ? t.giveBack(bot.name) : t.take} +
); diff --git a/app/src/features/browser/Lesson.tsx b/app/src/features/browser/Lesson.tsx new file mode 100644 index 00000000..abfb9f16 --- /dev/null +++ b/app/src/features/browser/Lesson.tsx @@ -0,0 +1,180 @@ +// Teaching the bot a task (spec 21.13): with the browser in the owner's +// hands, a lesson records what they do as steps by what it means; at its +// end they name the task and make it a routine, or send it for the bot to +// remember. + +import { AlarmClock, GraduationCap, Send, X } from "lucide-react"; +import { type FormEvent, useState } from "react"; +import { useT } from "../../i18n"; +import type { Bot, BrowserState, LessonStep } from "../../lib/protocol.gen"; +import { useApi } from "../../store/context"; +import { Button } from "../../ui/Button"; +import { Dialog } from "../../ui/Dialog"; +import { TextField } from "../../ui/Field"; +import { attempt } from "../../ui/toast"; +import { RoutineDialog } from "../routines/RoutineDialog"; +import { lessonText, stepText } from "./lessonText"; + +export interface Lesson { + /** The steps so far, while the owner teaches; `null` otherwise. */ + steps: LessonStep[] | null; + start(): Promise; + /** Ends the lesson and opens what to do with it. */ + finish(): Promise; + cancel(): Promise; + /** The steps of the lesson just finished, until the owner is done with them. */ + done: LessonStep[] | null; + close(): void; +} + +export function useLesson(bot: Pick, state: BrowserState | null): Lesson { + const api = useApi(); + const t = useT().browser.lesson; + const [done, setDone] = useState(null); + const teach = async (on: boolean) => { + let steps: LessonStep[] = []; + const ok = await attempt(t.failed, async () => { + steps = await api.call("browser.teach", { botId: bot.id, on }); + }); + return ok ? steps : null; + }; + return { + steps: state?.lesson ?? null, + start: async () => { + await teach(true); + }, + finish: async () => { + const steps = await teach(false); + if (steps) { + setDone(steps); + } + }, + cancel: async () => { + await teach(false); + }, + done, + close: () => setDone(null), + }; +} + +/** Under the owner's pill: start a lesson, or the one being given, live. */ +export function LessonArea({ bot, lesson }: { bot: Bot; lesson: Lesson }) { + const t = useT().browser.lesson; + if (lesson.steps === null) { + return ( +
+ +

{t.teachWhy(bot.name)}

+
+ ); + } + return ( +
+

+ + {t.recording} +

+ {lesson.steps.length === 0 ? ( +

{t.empty}

+ ) : ( +
    + {lesson.steps.map((step, index) => ( + // biome-ignore lint/suspicious/noArrayIndexKey: steps only grow at the end +
  1. + {index + 1}. {stepText(step, t)} +
    2. + ))} +
+ )} + {/* The pill above finishes it; here it can be dropped. */} +
+ +
+
+ ); +} + +/** What to do with the lesson just finished: a routine, or a message. */ +export function LessonDialog({ bot, lesson }: { bot: Bot; lesson: Lesson }) { + const t = useT().browser.lesson; + const api = useApi(); + const [name, setName] = useState(""); + const [steps, setSteps] = useState(lesson.done ?? []); + const [routine, setRoutine] = useState(false); + const ready = name.trim().length > 0 && steps.length > 0; + const text = lessonText(name, steps, t); + + const send = async (event?: FormEvent) => { + event?.preventDefault(); + if (!ready) return; + const body = `${text}\n\n${t.forMemory(name.trim())}`; + // It shows in the bot's chat, beside the panel. + if (await attempt(t.sendFailed, () => api.call("messages.send", { botId: bot.id, body }))) { + lesson.close(); + } + }; + + if (routine) { + return ( + + ); + } + return ( + +
+ setName(event.target.value)} + autoFocus + /> +
+

{t.steps}

+
    + {steps.map((step, index) => ( + // biome-ignore lint/suspicious/noArrayIndexKey: a list the owner only shortens +
  1. + {index + 1}. + + {stepText(step, t)} + +
    2. + ))} +
+
+
+ + +
+ +
+ ); +} diff --git a/app/src/features/browser/lesson.test.tsx b/app/src/features/browser/lesson.test.tsx new file mode 100644 index 00000000..ea531873 --- /dev/null +++ b/app/src/features/browser/lesson.test.tsx @@ -0,0 +1,113 @@ +import { act, cleanup, fireEvent, screen, waitFor, within } from "@testing-library/react"; +import { afterEach, describe, expect, test } from "vitest"; +import { en } from "../../i18n/en"; +import { FakeBotloft } from "../../lib/fake"; +import { crewOpened, openBot, renderApp } from "../../test/app"; +import { lessonText } from "./lessonText"; + +afterEach(cleanup); + +/** Scout's browser open on a sign-in page, in the owner's hands. */ +async function inHands() { + const fake = new FakeBotloft(); + const ops = fake.addCrew("Ops"); + const scout = fake.addBot(ops.id, "Scout", "Finds sources"); + fake.setBotState(scout.id, "busy"); + fake.browser.open(scout.id, "https://shop.example/login?next=/orders", "Sign in"); + renderApp(fake); + await crewOpened("Ops"); + openBot("Scout"); + await screen.findByRole("list", { name: "Messages" }); + fireEvent.click(screen.getByRole("button", { name: /^Show browser/ })); + await waitFor(() => expect(fake.browser.watching).toBe(scout.id)); + act(() => { + fake.browser.frame(scout.id, "AAAA"); + }); + fireEvent.click(screen.getByRole("button", { name: "Take control" })); + await screen.findByText("You are in control"); + return { fake, scout }; +} + +/** The daemon reads what the owner did as steps. */ +function teachSteps(fake: FakeBotloft, botId: string) { + act(() => { + fake.lesson.step(botId, { kind: "click", label: "E-mail", role: "textbox" }); + fake.lesson.step(botId, { kind: "type", label: "E-mail", role: "textbox" }); + fake.lesson.step(botId, { kind: "type", label: "Password", role: "textbox", secret: true }); + fake.lesson.step(botId, { kind: "press", label: "Enter" }); + }); +} + +async function finish(fake: FakeBotloft, botId: string) { + fireEvent.click(screen.getByRole("button", { name: "Teach a task" })); + const recording = await screen.findByRole("region", { name: "Recording the lesson" }); + expect(within(recording).getByText("1. Open https://shop.example/login")).toBeDefined(); + teachSteps(fake, botId); + expect( + within(recording).getByText('4. Type the password in "Password": ask me to type it'), + ).toBeDefined(); + // The pill ends the lesson before giving anything back. + fireEvent.click(screen.getByRole("button", { name: "Finish the lesson" })); + const dialog = await screen.findByRole("dialog", { name: "Teach Scout a task" }); + fireEvent.change(within(dialog).getByLabelText("Name of the task"), { + target: { value: "Check my orders" }, + }); + return dialog; +} + +describe("teaching a task", () => { + test("the words say where things were typed, never what", () => { + const text = lessonText( + "Check my orders", + [ + { kind: "open", label: "https://shop.example/login", role: null, secret: false }, + { kind: "type", label: "E-mail", role: "textbox", secret: false }, + { kind: "type", label: "Password", role: "textbox", secret: true }, + ], + en.browser.lesson, + ); + expect(text).toBe( + [ + 'How to do "Check my orders" in the browser, as I showed you:', + "1. Open https://shop.example/login", + '2. Type in "E-mail"', + '3. Type the password in "Password": ask me to type it', + "", + "Where I typed something, I don't show what: use what the task needs, or ask me.", + ].join("\n"), + ); + }); + + test("a lesson records live and goes to the bot to remember, without the steps taken out", async () => { + const { fake, scout } = await inHands(); + const dialog = await finish(fake, scout.id); + fireEvent.click(within(dialog).getByRole("button", { name: "Remove step 2" })); + fireEvent.click(within(dialog).getByRole("button", { name: "Send to Scout to remember" })); + await waitFor(() => + expect(fake.calls.some((call) => call.method === "messages.send")).toBe(true), + ); + const sent = fake.calls.find((call) => call.method === "messages.send")?.params as { + body: string; + }; + expect(sent.body).toContain('How to do "Check my orders"'); + expect(sent.body).toContain('2. Type in "E-mail"'); + expect(sent.body).not.toContain('Click "E-mail"'); + expect(sent.body).toContain('so you can do "Check my orders" when I ask.'); + await waitFor(() => expect(screen.queryByRole("dialog")).toBeNull()); + // The owner still has the browser. + expect(screen.getByText("You are in control")).toBeDefined(); + }); + + test("a lesson becomes a routine, its request already written", async () => { + const { fake, scout } = await inHands(); + const dialog = await finish(fake, scout.id); + fireEvent.click(within(dialog).getByRole("button", { name: "Make it a routine…" })); + const routine = await screen.findByRole("dialog", { name: "New routine for Scout" }); + expect((within(routine).getByLabelText("Name") as HTMLInputElement).value).toBe( + "Check my orders", + ); + const prompt = within(routine).getByLabelText("What should Scout do?") as HTMLTextAreaElement; + expect(prompt.value).toContain("1. Open https://shop.example/login"); + expect(prompt.value.endsWith("Do it now.")).toBe(true); + }); +}); diff --git a/app/src/features/browser/lessonText.ts b/app/src/features/browser/lessonText.ts new file mode 100644 index 00000000..aeb1af8b --- /dev/null +++ b/app/src/features/browser/lessonText.ts @@ -0,0 +1,28 @@ +// A lesson as words (spec 21.13): the steps the owner showed, numbered, in +// the owner's language, for a routine's request or a message to the bot. +// What was typed was never kept, so the words say where, not what. + +import type { Messages } from "../../i18n/en"; +import type { LessonStep } from "../../lib/protocol.gen"; + +type Words = Messages["browser"]["lesson"]; + +export function stepText(step: LessonStep, w: Words): string { + switch (step.kind) { + case "open": + return w.step.open(step.label); + case "click": + return w.step.click(step.label); + case "type": + return step.secret ? w.step.secret(step.label) : w.step.type(step.label); + case "press": + return w.step.press(step.label); + } +} + +/** The whole lesson: what it is, each step, and a word on what was typed. */ +export function lessonText(name: string, steps: LessonStep[], w: Words): string { + const lines = steps.map((step, index) => `${index + 1}. ${stepText(step, w)}`); + const typed = steps.some((step) => step.kind === "type" && !step.secret); + return [w.intro(name.trim()), ...lines, ...(typed ? ["", w.typedNote] : [])].join("\n"); +} diff --git a/app/src/features/routines/RoutineDialog.tsx b/app/src/features/routines/RoutineDialog.tsx index 40a5db19..f0b208ee 100644 --- a/app/src/features/routines/RoutineDialog.tsx +++ b/app/src/features/routines/RoutineDialog.tsx @@ -15,14 +15,21 @@ import { blankForm, formOf, type RoutineForm, scheduleOf } from "./form"; import { MoreOptions } from "./MoreOptions"; import { ScheduleFields } from "./ScheduleFields"; -type Props = { bot: Bot; onClose(): void } & ({ routine?: undefined } | { routine: Routine }); +type Props = { + bot: Bot; + onClose(): void; + /** A new routine's name and request, already written (a lesson, spec 21.13). */ + start?: { name: string; prompt: string }; +} & ({ routine?: undefined } | { routine: Routine }); -export function RoutineDialog({ bot, routine, onClose }: Props) { +export function RoutineDialog({ bot, routine, start, onClose }: Props) { const t = useT(); const d = t.routines.dialog; const api = useApi(); const putRoutine = useApp((state) => state.putRoutine); - const [form, setForm] = useState(() => (routine ? formOf(routine) : blankForm())); + const [form, setForm] = useState(() => + routine ? formOf(routine) : { ...blankForm(), ...start }, + ); const [error, setError] = useState(null); const [busy, setBusy] = useState(false); const change = (patch: Partial) => setForm((current) => ({ ...current, ...patch })); diff --git a/app/src/i18n/en/browser.ts b/app/src/i18n/en/browser.ts index 1bdc5d42..82c5cfed 100644 --- a/app/src/i18n/en/browser.ts +++ b/app/src/i18n/en/browser.ts @@ -83,6 +83,36 @@ export const browser = { typing: "What you type goes to the page", screen: (bot: string) => `${bot}'s browser, in your hands`, }, + /** Teaching the bot a task in its browser (spec 21.13). */ + lesson: { + teach: "Teach a task", + teachWhy: (bot: string) => + `Do the task here once: ${bot} keeps each step, never what you type.`, + recording: "Recording the lesson", + empty: "Do the task on the page: each step shows up here.", + finish: "Finish the lesson", + cancel: "Cancel", + failed: "Could not start or end the lesson", + title: (bot: string) => `Teach ${bot} a task`, + name: "Name of the task", + namePlaceholder: "Check new listings", + steps: "Steps", + remove: (n: number) => `Remove step ${n}`, + makeRoutine: "Make it a routine…", + sendToBot: (bot: string) => `Send to ${bot} to remember`, + sendFailed: "Could not send the lesson", + step: { + open: (url: string) => `Open ${url}`, + click: (label: string) => `Click "${label}"`, + type: (label: string) => `Type in "${label}"`, + secret: (label: string) => `Type the password in "${label}": ask me to type it`, + press: (key: string) => `Press ${key}`, + }, + intro: (name: string) => `How to do "${name}" in the browser, as I showed you:`, + typedNote: "Where I typed something, I don't show what: use what the task needs, or ask me.", + forRoutine: "Do it now.", + forMemory: (name: string) => `Keep this in your memory, so you can do "${name}" when I ask.`, + }, window: { open: "Sign in in a window", why: (bot: string) => diff --git a/app/src/i18n/es/browser.ts b/app/src/i18n/es/browser.ts index 22fd0671..264f29eb 100644 --- a/app/src/i18n/es/browser.ts +++ b/app/src/i18n/es/browser.ts @@ -81,6 +81,34 @@ export const browser: Messages["browser"] = { typing: "Lo que escribes va a la página", screen: (bot) => `Navegador de ${bot}, en tus manos`, }, + lesson: { + teach: "Enseñar una tarea", + teachWhy: (bot) => `Haz la tarea aquí una vez: ${bot} guarda cada paso, nunca lo que escribes.`, + recording: "Grabando la lección", + empty: "Haz la tarea en la página: cada paso aparece aquí.", + finish: "Terminar la lección", + cancel: "Cancelar", + failed: "No se pudo empezar o terminar la lección", + title: (bot) => `Enseñar una tarea a ${bot}`, + name: "Nombre de la tarea", + namePlaceholder: "Ver anuncios nuevos", + steps: "Pasos", + remove: (n) => `Quitar el paso ${n}`, + makeRoutine: "Convertir en rutina…", + sendToBot: (bot) => `Enviar a ${bot} para que lo guarde`, + sendFailed: "No se pudo enviar la lección", + step: { + open: (url) => `Abre ${url}`, + click: (label) => `Haz clic en "${label}"`, + type: (label) => `Escribe en "${label}"`, + secret: (label) => `Escribe la contraseña en "${label}": pídeme que la escriba`, + press: (key) => `Pulsa ${key}`, + }, + intro: (name) => `Cómo hacer "${name}" en el navegador, como te mostré:`, + typedNote: "Donde escribí algo, no muestro qué: usa lo que pida la tarea, o pregúntame.", + forRoutine: "Hazlo ahora.", + forMemory: (name) => `Guarda esto en tu memoria, para hacer "${name}" cuando te lo pida.`, + }, window: { open: "Iniciar sesión en una ventana", why: (bot) => diff --git a/app/src/i18n/pt-BR/browser.ts b/app/src/i18n/pt-BR/browser.ts index 38ecba57..42f66965 100644 --- a/app/src/i18n/pt-BR/browser.ts +++ b/app/src/i18n/pt-BR/browser.ts @@ -83,6 +83,36 @@ export const browser: Messages["browser"] = { typing: "O que você digita vai para a página", screen: (bot) => `Navegador de ${bot}, nas suas mãos`, }, + lesson: { + teach: "Ensinar uma tarefa", + teachWhy: (bot) => + `Faça a tarefa aqui uma vez: ${bot} guarda cada passo, nunca o que você digita.`, + recording: "Gravando a lição", + empty: "Faça a tarefa na página: cada passo aparece aqui.", + finish: "Terminar a lição", + cancel: "Cancelar", + failed: "Não foi possível começar ou terminar a lição", + title: (bot) => `Ensinar uma tarefa a ${bot}`, + name: "Nome da tarefa", + namePlaceholder: "Ver anúncios novos", + steps: "Passos", + remove: (n) => `Tirar o passo ${n}`, + makeRoutine: "Fazer virar rotina…", + sendToBot: (bot) => `Mandar para ${bot} guardar`, + sendFailed: "Não foi possível mandar a lição", + step: { + open: (url) => `Abra ${url}`, + click: (label) => `Clique em "${label}"`, + type: (label) => `Escreva em "${label}"`, + secret: (label) => `Escreva a senha em "${label}": peça para eu digitar`, + press: (key) => `Aperte ${key}`, + }, + intro: (name) => `Como fazer "${name}" no navegador, do jeito que eu mostrei:`, + typedNote: + "Onde eu escrevi algo, eu não mostro o quê: use o que a tarefa pede, ou me pergunte.", + forRoutine: "Faça isso agora.", + forMemory: (name) => `Guarde isso na sua memória, para fazer "${name}" quando eu pedir.`, + }, window: { open: "Entrar numa janela", why: (bot) => diff --git a/app/src/lib/fake.ts b/app/src/lib/fake.ts index 13b75c38..0a6edd68 100644 --- a/app/src/lib/fake.ts +++ b/app/src/lib/fake.ts @@ -10,6 +10,7 @@ import { FakeChat } from "./fakeChat"; import { FakeConversation } from "./fakeConversation"; import { crewHandlers } from "./fakeCrews"; import { FakeFiles } from "./fakeFiles"; +import { FakeLesson } from "./fakeLesson"; import { FakeQuestions } from "./fakeQuestions"; import { FakeReactions } from "./fakeReactions"; import { FakeRoutines } from "./fakeRoutines"; @@ -79,6 +80,7 @@ export class FakeBotloft implements BotloftApi { readonly screens = new FakeScreens(this); readonly questions = new FakeQuestions(this); readonly reactions = new FakeReactions(this); + readonly lesson = new FakeLesson(this); /** Clock for created and updated times. */ now = Date.now(); closed = false; @@ -264,6 +266,7 @@ export class FakeBotloft implements BotloftApi { ...this.routines.handlers(), ...this.questions.handlers(), ...this.reactions.handlers(), + ...this.lesson.handlers(), ...usageHandlers(this), ...searchHandlers(this), }; diff --git a/app/src/lib/fakeBrowser.ts b/app/src/lib/fakeBrowser.ts index 6b525daa..85145890 100644 --- a/app/src/lib/fakeBrowser.ts +++ b/app/src/lib/fakeBrowser.ts @@ -53,6 +53,7 @@ export class FakeBrowser { resting: false, ask: null, window: false, + lesson: null, updatedAt: this.fake.now, } ); @@ -188,11 +189,12 @@ export class FakeBrowser { if (botId === null || state?.control !== "owner" || state.window) { return null; } - return this.set(botId, { control: "bot" }); + // Giving the browser back ends a lesson not finished (spec 21.13). + return this.set(botId, { control: "bot", lesson: null }); } /** The owner has the watched browser, or the call is refused. */ - private held(botId: BotId): void { + held(botId: BotId): void { if (this.watching !== botId || this.state(botId).control !== "owner") { throw conflict("take the browser before using it"); } diff --git a/app/src/lib/fakeLesson.ts b/app/src/lib/fakeLesson.ts new file mode 100644 index 00000000..89d99cd6 --- /dev/null +++ b/app/src/lib/fakeLesson.ts @@ -0,0 +1,39 @@ +// The fake daemon's lessons (spec 21.13): teaching a bot a task with its +// browser in the owner's hands. `step` plays the daemon reading what the +// owner did; `browser.teach` starts and ends a lesson as the daemon does. + +import type { FakeBotloft, Handlers } from "./fake"; +import type { BotId, LessonStep } from "./protocol.gen"; + +export class FakeLesson { + constructor(private readonly fake: FakeBotloft) {} + + /** The owner did something that means `step`, while teaching. */ + step(botId: BotId, step: Partial & Pick): void { + const lesson = this.fake.browser.state(botId).lesson; + if (lesson) { + this.fake.browser.set(botId, { + lesson: [...lesson, { role: null, secret: false, ...step }], + }); + } + } + + handlers(): Pick { + return { + "browser.teach": ({ botId, on }) => { + this.fake.browser.held(botId); + const state = this.fake.browser.state(botId); + if (on) { + const url = state.url?.split(/[?#]/)[0] ?? null; + const steps: LessonStep[] = url + ? [{ kind: "open", label: url, role: null, secret: false }] + : []; + this.fake.browser.set(botId, { lesson: steps }); + return steps; + } + this.fake.browser.set(botId, { lesson: null }); + return state.lesson ?? []; + }, + }; + } +} diff --git a/app/src/lib/protocol.gen.ts b/app/src/lib/protocol.gen.ts index c9034d31..10b47a15 100644 --- a/app/src/lib/protocol.gen.ts +++ b/app/src/lib/protocol.gen.ts @@ -1233,6 +1233,23 @@ title: string, url: string, */ active: boolean, }; +export type LessonStepKind = "open" | "click" | "type" | "press"; + +export type LessonStep = { kind: LessonStepKind, +/** + * The address for `open`, the element's name for `click` and `type`, + * the key for `press`; at most 80 characters. + */ +label: string, +/** + * The element's role (`button`, `link`, `textbox`...), when it has one. + */ +role: string | null, +/** + * Typing in a password field: the bot asks the owner to type it. + */ +secret: boolean, }; + export type BrowserState = { botId: BotId, status: BrowserStatus, /** * The active tab's address. @@ -1269,6 +1286,11 @@ ask: string | null, * `closed` meanwhile, and the bot's tools wait. */ window: boolean, +/** + * The steps of the lesson the owner is giving, while they teach the + * bot a task (spec 21.13); `null` when nobody teaches. + */ +lesson: Array | null, /** * Unix time in milliseconds. */ @@ -1356,6 +1378,11 @@ export type BrowserTabParams = { botId: BotId, tabId: string, }; */ export type BrowserOpenParams = { botId: BotId, url: string, }; +/** + * Starts or ends a lesson in the browser the connection has in its hands. + */ +export type BrowserTeachParams = { botId: BotId, on: boolean, }; + /** * The device a screen is drawn for. */ @@ -1477,6 +1504,7 @@ export interface RpcMethods { "browser.take": { params: BrowserControlParams; result: BrowserState }; "browser.release": { params: BrowserControlParams; result: BrowserState }; "browser.window": { params: BrowserControlParams; result: BrowserState }; + "browser.teach": { params: BrowserTeachParams; result: Array }; "browser.input": { params: BrowserInputParams; result: null }; "browser.reload": { params: BrowserControlParams; result: null }; "browser.newTab": { params: BrowserControlParams; result: null }; diff --git a/crates/botloft-core/src/protocol/browser.rs b/crates/botloft-core/src/protocol/browser.rs index 48a6aff3..0360b8f5 100644 --- a/crates/botloft-core/src/protocol/browser.rs +++ b/crates/botloft-core/src/protocol/browser.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; -use super::BrowserControl; +use super::{BrowserControl, LessonStep}; use crate::ids::BotId; /// Where a bot's browser is. @@ -59,6 +59,10 @@ pub struct BrowserState { /// site refuses a browser a program drives (spec 21.11). It is /// `closed` meanwhile, and the bot's tools wait. pub window: bool, + /// The steps of the lesson the owner is giving, while they teach the + /// bot a task (spec 21.13); `null` when nobody teaches. + #[serde(default)] + pub lesson: Option>, /// Unix time in milliseconds. pub updated_at: i64, } @@ -77,6 +81,7 @@ impl BrowserState { resting: false, ask: None, window: false, + lesson: None, updated_at: now, } } diff --git a/crates/botloft-core/src/protocol/export_browser.rs b/crates/botloft-core/src/protocol/export_browser.rs index d6c4fa92..e27d9910 100644 --- a/crates/botloft-core/src/protocol/export_browser.rs +++ b/crates/botloft-core/src/protocol/export_browser.rs @@ -9,6 +9,8 @@ pub(super) fn decls(out: &mut Out) { out.decl::(); out.decl::(); out.decl::(); + out.decl::(); + out.decl::(); out.decl::(); out.decl::(); out.decl::(); @@ -23,6 +25,7 @@ pub(super) fn decls(out: &mut Out) { out.decl::(); out.decl::(); out.decl::(); + out.decl::(); out.decl::(); out.decl::(); out.decl::(); @@ -51,6 +54,11 @@ pub(super) fn methods(out: &mut Out) { out.method(method::BROWSER_TAKE, &control, &state); out.method(method::BROWSER_RELEASE, &control, &state); out.method(method::BROWSER_WINDOW, &control, &state); + out.method( + method::BROWSER_TEACH, + &out.name::(), + &out.name::>(), + ); out.method( method::BROWSER_INPUT, &out.name::(), diff --git a/crates/botloft-core/src/protocol/lesson.rs b/crates/botloft-core/src/protocol/lesson.rs new file mode 100644 index 00000000..2cb33782 --- /dev/null +++ b/crates/botloft-core/src/protocol/lesson.rs @@ -0,0 +1,48 @@ +//! Teaching a bot a task (spec 21.13): while the owner has the bot's +//! browser in their hands, what they do is kept as steps by what it means +//! (the page, the button, the field), never by what they typed. + +use serde::{Deserialize, Serialize}; + +use crate::ids::BotId; + +/// Most steps one lesson keeps; later ones are left out. +pub const LESSON_STEPS_MAX: usize = 100; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub enum LessonStepKind { + /// A web address in the active tab: where the lesson began, or one + /// the owner typed. + Open, + /// A click on something with a role (a button, a link, a field...). + Click, + /// Typing in a field. What was typed is never kept. + Type, + /// A key that does something: Enter, Escape, Tab. + Press, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct LessonStep { + pub kind: LessonStepKind, + /// The address for `open`, the element's name for `click` and `type`, + /// the key for `press`; at most 80 characters. + pub label: String, + /// The element's role (`button`, `link`, `textbox`...), when it has one. + pub role: Option, + /// Typing in a password field: the bot asks the owner to type it. + pub secret: bool, +} + +/// Starts or ends a lesson in the browser the connection has in its hands. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct BrowserTeachParams { + pub bot_id: BotId, + pub on: bool, +} diff --git a/crates/botloft-core/src/protocol/mod.rs b/crates/botloft-core/src/protocol/mod.rs index f8b9a963..b3f19547 100644 --- a/crates/botloft-core/src/protocol/mod.rs +++ b/crates/botloft-core/src/protocol/mod.rs @@ -55,6 +55,7 @@ mod chat; #[cfg(test)] mod export; mod files; +mod lesson; mod messaging; mod methods; mod model; @@ -72,6 +73,7 @@ pub use browser::*; pub use browser_input::*; pub use chat::*; pub use files::*; +pub use lesson::*; pub use messaging::*; pub use methods::*; pub use model::*; @@ -146,6 +148,7 @@ pub mod method { pub const BROWSER_SWITCH_TAB: &str = "browser.switchTab"; pub const BROWSER_OPEN: &str = "browser.open"; pub const BROWSER_WINDOW: &str = "browser.window"; + pub const BROWSER_TEACH: &str = "browser.teach"; pub const SCREENS_LIST: &str = "screens.list"; pub const QUESTIONS_LIST: &str = "questions.list"; pub const QUESTIONS_ANSWER: &str = "questions.answer"; diff --git a/crates/botloftd/src/browser/actions.js b/crates/botloftd/src/browser/actions.js index 0cd44651..e391b863 100644 --- a/crates/botloftd/src/browser/actions.js +++ b/crates/botloftd/src/browser/actions.js @@ -1,6 +1,8 @@ // What a bot does on the page (spec 21.4), beside the page reader: where to // click an element, getting a text field ready for typing, choosing an -// option and scrolling to an end. `reader.js` hands in its helpers. +// option and scrolling to an end; and, for a lesson (spec 21.13), what is +// at a point and which field has the focus. `reader.js` hands in its +// helpers. ({ get, clean, styleOf, roleOf, nameOf, token, isTextField, next }) => { // The element's box in the top page's coordinates, through same-site frames. const topRect = (el) => { @@ -90,5 +92,25 @@ return { top: window.scrollY }; }; - return { point, prepareType, select, scrollEnd }; + // What the owner acts on while teaching a task (spec 21.13): the nearest + // element with a role, by its name. Never what a field holds. + const meaning = (el) => { + let shown = el; + while (shown && !roleOf(shown, styleOf(shown), "auto")) shown = shown.parentElement; + if (!shown) return { none: true }; + return { + role: roleOf(shown, styleOf(shown), "auto"), + label: clean(nameOf(shown), 80), + secret: + shown.localName === "input" && (shown.getAttribute("type") || "").toLowerCase() === "password", + }; + }; + const at = (x, y) => meaning(document.elementFromPoint(x, y)); + const focused = () => { + let el = document.activeElement; + while (el?.shadowRoot?.activeElement) el = el.shadowRoot.activeElement; + return el && el !== document.body && isTextField(el) ? meaning(el) : { none: true }; + }; + + return { point, prepareType, select, scrollEnd, at, focused }; } diff --git a/crates/botloftd/src/browser/hands.rs b/crates/botloftd/src/browser/hands.rs index ea64e2c7..5ca72299 100644 --- a/crates/botloftd/src/browser/hands.rs +++ b/crates/botloftd/src/browser/hands.rs @@ -7,11 +7,12 @@ use std::sync::atomic::Ordering; use std::time::{Duration, Instant}; use botloft_core::ids::{ApprovalId, BotId}; -use botloft_core::protocol::{BrowserControl, BrowserInput}; +use botloft_core::protocol::{BrowserControl, BrowserInput, LessonStep, LessonStepKind}; use tokio::sync::{broadcast, mpsc}; use tracing::debug; use super::input::fits; +use super::lesson::{self, Teacher}; use super::moves::{Move, feed}; use super::rest::wake; use super::session::Session; @@ -84,7 +85,13 @@ impl Browsers { tokio::spawn(async move { // A browser at rest wakes for the owner's hands. wake(&page, &slots, &events, clock.as_ref(), &owner).await; - feed(page, queue).await; + let teacher = Teacher { + slots, + events, + clock, + bot: owner, + }; + feed(page, queue, teacher).await; }); Ok(Hands { slots: Arc::clone(&self.slots), @@ -222,6 +229,40 @@ impl Hands { self.make(Move::SwitchTab(id.to_owned())) } + /// Starts a lesson (spec 21.13), from the page the active tab shows, or + /// ends it; the steps so far. + pub fn teach(&self, on: bool) -> Result, InputError> { + self.page()?; + let start = on + .then(|| self.session.lock().active().map(|tab| tab.url.clone())) + .flatten() + .and_then(|url| lesson::address(&url)); + let mut steps = Vec::new(); + update( + &self.slots, + &self.events, + self.clock.as_ref(), + &self.bot, + |state| { + if on { + steps = start + .map(|url| LessonStep { + kind: LessonStepKind::Open, + label: url, + role: None, + secret: false, + }) + .into_iter() + .collect(); + state.lesson = Some(steps.clone()); + } else { + steps = state.lesson.take().unwrap_or_default(); + } + }, + ); + Ok(steps) + } + /// Takes the active tab to the web address the owner typed. pub fn open(&self, address: &str) -> Result<(), InputError> { self.page()?; @@ -245,7 +286,11 @@ impl Drop for Hands { &self.events, self.clock.as_ref(), &self.bot, - |state| state.control = BrowserControl::Bot, + |state| { + state.control = BrowserControl::Bot; + // Giving the browser back ends a lesson not finished. + state.lesson = None; + }, ); debug!(bot = %self.bot, "browser: the owner gave it back"); } diff --git a/crates/botloftd/src/browser/lesson.rs b/crates/botloftd/src/browser/lesson.rs new file mode 100644 index 00000000..d6207266 --- /dev/null +++ b/crates/botloftd/src/browser/lesson.rs @@ -0,0 +1,184 @@ +//! Teaching a bot a task (spec 21.13): while the owner has the browser in +//! their hands and teaches, each thing they do becomes a step by what it +//! means, read from the page before it reaches it: the button clicked, the +//! field typed in, the address opened. What they type is never read. + +use std::sync::Arc; + +use botloft_core::ids::BotId; +use botloft_core::protocol::{ + BrowserInput, LESSON_STEPS_MAX, LessonStep, LessonStepKind, MouseAction, MouseButton, modifier, +}; +use serde_json::Value; +use tokio::sync::broadcast; + +use super::moves::Move; +use super::session::Session; +use super::{Slots, lock, update}; +use crate::clock::Clock; +use crate::state::Event; + +/// Longest label a step keeps, in characters. +const LABEL_MAX: usize = 80; + +/// Where a lesson's steps go: the bot's browser state, which the app +/// follows. +pub(super) struct Teacher { + pub slots: Slots, + pub events: broadcast::Sender, + pub clock: Arc, + pub bot: BotId, +} + +impl Teacher { + fn teaching(&self) -> bool { + lock(&self.slots) + .get(&self.bot) + .is_some_and(|slot| slot.state.lesson.is_some()) + } + + /// Adds a step to the lesson going on, if one is. Typing on in the + /// field of the step before is the same step. + fn add(&self, step: LessonStep) { + update( + &self.slots, + &self.events, + self.clock.as_ref(), + &self.bot, + |state| { + let Some(steps) = state.lesson.as_mut() else { + return; + }; + let same_field = |last: &LessonStep| { + last.kind == LessonStepKind::Type + && step.kind == LessonStepKind::Type + && last.label == step.label + }; + if steps.last().is_some_and(same_field) || steps.len() >= LESSON_STEPS_MAX { + return; + } + steps.push(step); + }, + ); + } + + /// What `step` adds to the lesson, read from the page before the step + /// reaches it. + pub(super) async fn observe(&self, session: &Session, step: &Move) { + if !self.teaching() { + return; + } + let found = match step { + Move::Open(url) => address(url).map(|url| plain(LessonStepKind::Open, url)), + Move::Input(BrowserInput::Mouse { + action: MouseAction::Down, + button: MouseButton::Left, + clicks, + x, + y, + .. + }) if *clicks <= 1 => { + read(session, &format!("at({x}, {y})"), LessonStepKind::Click).await + } + Move::Input(BrowserInput::Key { key, modifiers, .. }) => { + let plain_key = modifiers & (modifier::CTRL | modifier::ALT | modifier::META) == 0; + if plain_key && matches!(key.as_str(), "Enter" | "Escape" | "Tab") { + Some(plain(LessonStepKind::Press, key.clone())) + } else if plain_key && (key.chars().count() == 1 || key == "Backspace") { + read(session, "focused()", LessonStepKind::Type).await + } else { + None + } + } + Move::Input(BrowserInput::Text { .. }) => { + read(session, "focused()", LessonStepKind::Type).await + } + _ => None, + }; + if let Some(found) = found { + self.add(found); + } + } +} + +fn plain(kind: LessonStepKind, label: String) -> LessonStep { + LessonStep { + kind, + label, + role: None, + secret: false, + } +} + +/// The element the page script found, as a step; `None` when nothing with +/// a role is there. +async fn read(session: &Session, call: &str, kind: LessonStepKind) -> Option { + let found = session.reader(call).await.ok()?; + meaning(&found, kind) +} + +fn meaning(found: &Value, kind: LessonStepKind) -> Option { + if found["none"] == true { + return None; + } + let label: String = found["label"].as_str()?.chars().take(LABEL_MAX).collect(); + Some(LessonStep { + kind, + label, + role: found["role"].as_str().map(str::to_owned), + secret: found["secret"] == true, + }) +} + +/// A web address as a lesson keeps it: its site and path, without the +/// query or the fragment, where sign-in tokens and personal data travel. +/// `None` for anything but `http` and `https`. +pub(super) fn address(url: &str) -> Option { + let (scheme, rest) = url.split_once("://")?; + if !scheme.eq_ignore_ascii_case("http") && !scheme.eq_ignore_ascii_case("https") { + return None; + } + let rest = rest.split(['?', '#']).next()?; + let (authority, path) = rest.split_once('/').unwrap_or((rest, "")); + // Never a name and password written into the address. + let host = authority.rsplit('@').next()?; + if host.is_empty() { + return None; + } + let kept = format!("{}://{host}/{path}", scheme.to_ascii_lowercase()); + Some(kept.chars().take(LABEL_MAX * 2).collect()) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn an_address_keeps_its_site_and_path_only() { + assert_eq!( + address("https://mail.example.com/inbox?token=secret#msg-1").as_deref(), + Some("https://mail.example.com/inbox") + ); + assert_eq!( + address("https://ana:hunter2@example.com").as_deref(), + Some("https://example.com/") + ); + assert_eq!(address("about:blank"), None); + assert_eq!(address("file:///C:/notes.txt"), None); + } + + #[test] + fn what_the_page_found_becomes_a_step_never_a_value() { + let field = json!({ "role": "textbox", "label": "Password", "secret": true }); + let step = meaning(&field, LessonStepKind::Type).expect("step"); + assert_eq!(step.label, "Password"); + assert_eq!(step.role.as_deref(), Some("textbox")); + assert!(step.secret); + assert_eq!( + meaning(&json!({ "none": true }), LessonStepKind::Click), + None + ); + } +} diff --git a/crates/botloftd/src/browser/mod.rs b/crates/botloftd/src/browser/mod.rs index 18ff7868..c1a68fa0 100644 --- a/crates/botloftd/src/browser/mod.rs +++ b/crates/botloftd/src/browser/mod.rs @@ -11,6 +11,7 @@ mod hands; mod input; mod keys; mod launch; +mod lesson; mod moves; mod page; mod program; diff --git a/crates/botloftd/src/browser/moves.rs b/crates/botloftd/src/browser/moves.rs index 0c9036f1..2b511469 100644 --- a/crates/botloftd/src/browser/moves.rs +++ b/crates/botloftd/src/browser/moves.rs @@ -12,6 +12,7 @@ use serde_json::json; use tokio::sync::mpsc; use tracing::debug; +use super::lesson::Teacher; use super::rest::wake; use super::session::{Session, Tab}; use super::{BrowserError, Browsers}; @@ -173,7 +174,11 @@ impl Session { /// Does what the owner did, one thing at a time, in order. Of several mouse /// moves in a row only the newest matters. Each thing queued counts in /// `owner_moves` until it is done or skipped. -pub(super) async fn feed(session: Arc, mut queue: mpsc::UnboundedReceiver) { +pub(super) async fn feed( + session: Arc, + mut queue: mpsc::UnboundedReceiver, + teacher: Teacher, +) { let done = || { session .owner_moves @@ -190,6 +195,10 @@ pub(super) async fn feed(session: Arc, mut queue: mpsc::UnboundedReceiv break; } } + if !session.is_closed() { + // A lesson reads the page before the step changes it. + teacher.observe(&session, &step).await; + } if !session.is_closed() && let Err(err) = session.make(&step).await { diff --git a/crates/botloftd/src/rpc/mod.rs b/crates/botloftd/src/rpc/mod.rs index c5e949c9..38f091f3 100644 --- a/crates/botloftd/src/rpc/mod.rs +++ b/crates/botloftd/src/rpc/mod.rs @@ -223,7 +223,8 @@ fn route( | method::BROWSER_RELOAD | method::BROWSER_NEW_TAB | method::BROWSER_SWITCH_TAB - | method::BROWSER_OPEN => watch.request(daemon, &request.method, request.params), + | method::BROWSER_OPEN + | method::BROWSER_TEACH => watch.request(daemon, &request.method, request.params), // A window of its own outlives the request (spec 21.11). method::BROWSER_WINDOW => watch.window(daemon, request.params), name if ASIDE.contains(&name) => return Routed::Aside(request), diff --git a/crates/botloftd/src/rpc/watching.rs b/crates/botloftd/src/rpc/watching.rs index 0e2f9c5c..3eeb0c24 100644 --- a/crates/botloftd/src/rpc/watching.rs +++ b/crates/botloftd/src/rpc/watching.rs @@ -9,7 +9,8 @@ use std::sync::Arc; use botloft_core::ids::BotId; use botloft_core::protocol::{ ApprovalsAnswerParams, BrowserControlParams, BrowserInputParams, BrowserOpenParams, - BrowserResizeParams, BrowserTabParams, BrowserWatchParams, error_code, method, notification, + BrowserResizeParams, BrowserTabParams, BrowserTeachParams, BrowserWatchParams, error_code, + method, notification, }; use serde::Serialize; use serde_json::Value; @@ -47,6 +48,7 @@ impl Watch { method::BROWSER_NEW_TAB => return self.new_tab(&parse(params)?), method::BROWSER_SWITCH_TAB => return self.switch_tab(&parse(params)?), method::BROWSER_OPEN => return self.open(&parse(params)?), + method::BROWSER_TEACH => return self.teach(&parse(params)?), _ => {} } // Stop the one before, even if the new one is refused; the hands @@ -188,6 +190,16 @@ impl Watch { made(self.hands(¶ms.bot_id)?.open(¶ms.url)) } + /// Starts or ends a lesson (spec 21.13), with the browser in this + /// connection's hands. + fn teach(&self, params: &BrowserTeachParams) -> Result { + let steps = self + .hands(¶ms.bot_id)? + .teach(params.on) + .map_err(refused)?; + to_value(&steps) + } + /// The watched browser's next frame, as a notification. Never resolves /// while nothing is watched. pub(super) async fn next_frame(&mut self) -> Option { @@ -220,12 +232,17 @@ fn helped(daemon: &Daemon, bot: &BotId) { /// What the owner did is on its way to the browser, or why it is not. fn made(queued: Result<(), InputError>) -> Result { - queued.map_err(|err| match err { + queued.map_err(refused)?; + Ok(Value::Null) +} + +fn refused(err: InputError) -> RpcError { + match err { InputError::Invalid | InputError::Address => ApiError::validation(err.to_string()), InputError::NoTab => ApiError::NotFound(err.to_string()), InputError::NotHeld | InputError::Closed => ApiError::Conflict(err.to_string()), - })?; - Ok(Value::Null) + } + .into() } fn to_value(value: &impl Serialize) -> Result { diff --git a/crates/botloftd/tests/browser_hands.rs b/crates/botloftd/tests/browser_hands.rs index 5be6bc9f..25131719 100644 --- a/crates/botloftd/tests/browser_hands.rs +++ b/crates/botloftd/tests/browser_hands.rs @@ -8,38 +8,9 @@ mod common; use std::time::Duration; use common::Client; -use common::browsing::{Browsing, answer_site, call, pending_approval, setup}; +use common::browsing::{Browsing, answer_site, call, click, pending_approval, press, send, setup}; use serde_json::{Value, json}; -async fn send(app: &mut Client, bot: &Value, input: Value) { - app.call( - "browser.input", - json!({ "botId": bot["id"], "input": input }), - ) - .await - .expect("input"); -} - -async fn click(app: &mut Client, bot: &Value, x: f64, y: f64) { - for (action, buttons) in [("move", 0), ("down", 1), ("up", 0)] { - let input = json!({ - "kind": "mouse", "action": action, "x": x, "y": y, "button": "left", - "buttons": buttons, "clicks": 1, "modifiers": 0, - }); - send(app, bot, input).await; - } -} - -/// Presses the keys of `text`, one by one, as the app sends them. -async fn press(app: &mut Client, bot: &Value, text: &str, modifiers: u32) { - for key in text.chars() { - let code = format!("Key{}", key.to_ascii_uppercase()); - let input = - json!({ "kind": "key", "key": key.to_string(), "code": code, "modifiers": modifiers }); - send(app, bot, input).await; - } -} - /// The bot opens the sign-in page; the owner allows the site. async fn sign_in_page(b: &mut Browsing) -> String { let url = format!("{}/login", b.site); diff --git a/crates/botloftd/tests/browser_lesson.rs b/crates/botloftd/tests/browser_lesson.rs new file mode 100644 index 00000000..3622d5cd --- /dev/null +++ b/crates/botloftd/tests/browser_lesson.rs @@ -0,0 +1,95 @@ +//! Teaching a bot a task with the real Microsoft Edge (spec 21.13): what +//! the owner does becomes steps by what it means, never by what was typed. +//! Skipped where Edge is not installed. + +mod common; + +use common::browsing::{Browsing, answer_site, call, click, press, send, setup}; +use serde_json::{Value, json}; + +async fn sign_in_page(b: &mut Browsing) { + let url = format!("{}/login", b.site); + let opening = call(&b.mcp, "browser_open", json!({ "url": url })); + answer_site(&mut b.app, true, None).await; + opening.await.expect("task").expect("page"); +} + +async fn teach(b: &mut Browsing, on: bool) -> Value { + let params = json!({ "botId": b.bot["id"], "on": on }); + b.app.call("browser.teach", params).await.expect("teach") +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_lesson_keeps_the_steps_and_never_what_was_typed() { + let Some(mut b) = setup().await else { return }; + sign_in_page(&mut b).await; + let bot = b.bot.clone(); + let id = json!({ "botId": bot["id"] }); + b.app + .call("browser.watch", id.clone()) + .await + .expect("watch"); + + // Only with the browser in the owner's hands. + let early = json!({ "botId": bot["id"], "on": true }); + assert!(b.app.call("browser.teach", early).await.is_err()); + b.app.call("browser.take", id.clone()).await.expect("take"); + + let started = teach(&mut b, true).await; + assert_eq!(started[0]["kind"], "open"); + assert_eq!(started[0]["label"], format!("{}/login", b.site)); + + click(&mut b.app, &bot, 200.0, 115.0).await; + press(&mut b.app, &bot, "ana", 0).await; + click(&mut b.app, &bot, 200.0, 215.0).await; + send( + &mut b.app, + &bot, + json!({ "kind": "text", "text": "hunter2" }), + ) + .await; + let enter = json!({ "kind": "key", "key": "Enter", "code": "Enter", "modifiers": 0 }); + send(&mut b.app, &bot, enter).await; + + // The steps reach the app as they happen. + let mut lesson = Value::Null; + for _ in 0..50 { + let list = b.app.call("browser.list", Value::Null).await.expect("list"); + lesson = list[0]["lesson"].clone(); + if lesson.as_array().is_some_and(|steps| steps.len() >= 6) { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + let steps = teach(&mut b, false).await; + assert_eq!(steps, lesson); + let kinds: Vec<_> = steps + .as_array() + .expect("steps") + .iter() + .map(|step| { + format!( + "{} {}", + step["kind"].as_str().unwrap_or(""), + step["label"].as_str().unwrap_or("") + ) + }) + .collect(); + assert_eq!( + kinds[1..], + [ + "click User", + "type User", + "click Password", + "type Password", + "press Enter" + ] + ); + assert_eq!(steps[4]["secret"], true); + assert!(!steps.to_string().contains("ana\""), "{steps}"); + assert!(!steps.to_string().contains("hunter2"), "{steps}"); + + // Ended, nothing more is kept. + let list = b.app.call("browser.list", Value::Null).await.expect("list"); + assert_eq!(list[0]["lesson"], Value::Null); +} diff --git a/crates/botloftd/tests/common/browsing.rs b/crates/botloftd/tests/common/browsing.rs index 7db97165..87a593cb 100644 --- a/crates/botloftd/tests/common/browsing.rs +++ b/crates/botloftd/tests/common/browsing.rs @@ -80,3 +80,34 @@ pub fn ref_of(page: &str, what: &str) -> String { .expect("ref") .to_owned() } + +/// One of the owner's events, as the app sends it. +pub async fn send(app: &mut Client, bot: &Value, input: Value) { + app.call( + "browser.input", + json!({ "botId": bot["id"], "input": input }), + ) + .await + .expect("input"); +} + +/// A left click at a point of the page. +pub async fn click(app: &mut Client, bot: &Value, x: f64, y: f64) { + for (action, buttons) in [("move", 0), ("down", 1), ("up", 0)] { + let input = json!({ + "kind": "mouse", "action": action, "x": x, "y": y, "button": "left", + "buttons": buttons, "clicks": 1, "modifiers": 0, + }); + send(app, bot, input).await; + } +} + +/// Presses the keys of `text`, one by one, as the app sends them. +pub async fn press(app: &mut Client, bot: &Value, text: &str, modifiers: u32) { + for key in text.chars() { + let code = format!("Key{}", key.to_ascii_uppercase()); + let input = + json!({ "kind": "key", "key": key.to_string(), "code": code, "modifiers": modifiers }); + send(app, bot, input).await; + } +} diff --git a/docs/spec.md b/docs/spec.md index b292f9ad..fb4995da 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1377,6 +1377,7 @@ Esqueceu a senha? [e5 link "Recuperar acesso"] | `browser.switchTab` | `botId`, `tabId` | `null`: essa aba vira a ativa. Só da conexão que controla; aba que não existe mais é `not_found` | | `browser.open` | `botId`, `url` | `null`: leva a aba ativa a um endereço `http` ou `https`; sem esquema, vale `https://`. Só da conexão que controla; o que não é endereço da web é `validation` | | `browser.window` | `botId` | `BrowserState`: abre o navegador numa janela própria, para o dono entrar numa conta (21.11). Da conexão que assiste esse bot; tira o controle dela, se o tinha. Já aberto numa janela, ou sem navegador no computador, é `conflict` | +| `browser.teach` | `botId`, `on` | `LessonStep[]`: começa uma lição (21.13), com o primeiro passo, ou termina e devolve os passos. Só da conexão que controla | - O que muda o navegador responde assim que o pedido é aceito; o resultado chega em `browser.changed` e nos quadros. - `BrowserState`: `botId`, `status` (`closed`, `starting`, `open`, `failed`), `url`, `title` e `loading` (da aba ativa), `tabs` (as abas abertas, na ordem em que abriram: `BrowserTab {id, title, url, active}`, com `id` opaco), `error` (por que não abriu, em `failed`), `control` (`bot` ou `owner`, 21.10), `resting` (o navegador descansa, 21.2), `ask` (o que o bot pediu ao dono com `browser_ask_owner`, enquanto o pedido está aberto), `window` (aberto numa janela própria, 21.11: o estado é `closed` e `control` é `owner`) e `updatedAt`. `browser.list` traz também os `closed` com `window`. @@ -1475,8 +1476,25 @@ O login do Google, e o de outros sites que olham mais que o user agent e `naviga | **N3.2** Abas e recarregar | a lista de abas em `BrowserState`, `browser.reload`, `browser.newTab`, `browser.switchTab`, `browser.open`, o aviso ao bot quando o dono troca de aba, as abas e o botão de recarregar no painel, o endereço que o dono digita, textos nos três idiomas, testes com o Edge real | pelo app, o dono vê as abas que o bot abriu, recarrega a página, assume, abre uma aba nova num endereço, troca de aba e devolve; o bot lê que a aba mudou antes de agir | | **N3.3** Descanso | o navegador descansa quando o turno do bot termina e acorda quando é usado, `resting` em `BrowserState`, fechar por falta de uso mesmo com o painel aberto, a etiqueta "Em descanso", textos nos três idiomas, testes com o Edge real | com um bot real parado numa página que se mexe, o Edge dele não gasta processador; o bot volta a usar a página sem perder nada | | **N3.4** Janela de verdade | `browser.window`, `window` em `BrowserState`, o Edge com janela no perfil do bot, a espera das tools e do pedido de ajuda, o botão e a faixa no painel, textos nos três idiomas | com o Claude Code real, o dono entra numa conta do Google do bot pela janela, fecha, e o bot abre o Gmail já dentro da conta | +| **N3.5** Ensinar uma tarefa | `browser.teach`, `lesson` em `BrowserState`, a leitura do que o dono faz em `actions.js`, o texto da lição, a faixa e o diálogo no painel, textos nos três idiomas, teste com o Edge real | 21.13 | | **N4** Telas | o bot desenhando telas (HTML) que aparecem lado a lado numa área de design, atualizadas enquanto ele escreve | seção 22 | +### 21.13 Ensinar uma tarefa (N3.5) + +O dono mostra ao bot, uma vez, como fazer uma tarefa no navegador dele; o que fez vira passos, e os passos viram uma rotina (20) ou uma message para o bot guardar na memória. + +- **Só nas mãos do dono.** Com o navegador no controle do dono (21.10), "Ensinar uma tarefa" embaixo da pílula chama `browser.teach {botId, on: true}`. A lição começa com um passo `open` com o endereço da aba ativa. `browser.teach {on: false}` a termina e devolve os passos. Devolver o navegador, fechar a conexão ou o navegador fechar descartam a lição. +- **O que vira passo.** Cada coisa que o dono faz passa pela fila de 21.10; com uma lição aberta, o daemon lê a página antes de ela chegar lá, pelo script do mundo isolado (21.6, `actions.js`), que devolve só o papel, o nome e se é um campo de senha: + - um clique (botão esquerdo, não o segundo de um clique duplo) vira `click` com o nome do elemento com papel mais próximo do ponto (`at(x, y)`); sem nada com papel ali, não vira passo; + - teclas de texto, Backspace e texto colado ou composto viram `type` com o nome do campo que tem o foco (`focused()`), um passo por campo enquanto o dono escreve nele; o campo de senha marca `secret`; + - Enter, Escape e Tab sem Ctrl, Alt ou Meta viram `press`; + - um endereço que o dono digita (`browser.open`) vira `open`. +- **O que nunca é guardado.** O que o dono digita, cola ou escolhe: o script não lê valor de campo. Um endereço guarda só o esquema, o site e o caminho, sem usuário e senha, query e fragmento, onde viajam tokens e dados pessoais. O rótulo de cada passo tem no máximo 80 caracteres, e uma lição, 100 passos. Os passos ficam só em memória, em `BrowserState.lesson`, e não vão para o log. +- **No app.** Embaixo da pílula, "Ensinar uma tarefa" e a explicação de que o bot guarda cada passo, nunca o que é digitado. Gravando, uma faixa "Gravando a lição" mostra os passos ao vivo, numerados, com "Cancelar"; o botão da pílula vira "Terminar a lição". Terminada, um diálogo pede o nome da tarefa e mostra os passos, cada um com um X para tirar, e oferece: + - **Fazer virar rotina…**: abre o diálogo de rotina nova (20.9) com o nome e o pedido já escritos: a lição em palavras e "Faça isso agora."; + - **Mandar para guardar**: manda ao bot, como message do dono, a lição em palavras e o pedido de guardá-la na memória (5.1) para fazer quando o dono pedir. +- **A lição em palavras** é montada pelo app, no idioma do dono, e pode ser editada na rotina: "Como fazer "" no navegador, do jeito que eu mostrei:", os passos numerados ("Abra ", "Clique em """, "Escreva em """, "Escreva a senha em "": peça para eu digitar", "Aperte Enter") e, se houve texto digitado, que o dono não mostra o quê: o bot usa o que a tarefa pede ou pergunta. Senhas, o bot pede com `browser_ask_owner` (21.10). + ## 22. Telas Status: **T1, T2 e T3 implementados** (22.8). É o N4 da seção 21.