);
diff --git a/app/src/features/browser/Lesson.tsx b/app/src/features/browser/Lesson.tsx
new file mode 100644
index 00000000..abfb9f16
--- /dev/null
+++ b/app/src/features/browser/Lesson.tsx
@@ -0,0 +1,180 @@
+// Teaching the bot a task (spec 21.13): with the browser in the owner's
+// hands, a lesson records what they do as steps by what it means; at its
+// end they name the task and make it a routine, or send it for the bot to
+// remember.
+
+import { AlarmClock, GraduationCap, Send, X } from "lucide-react";
+import { type FormEvent, useState } from "react";
+import { useT } from "../../i18n";
+import type { Bot, BrowserState, LessonStep } from "../../lib/protocol.gen";
+import { useApi } from "../../store/context";
+import { Button } from "../../ui/Button";
+import { Dialog } from "../../ui/Dialog";
+import { TextField } from "../../ui/Field";
+import { attempt } from "../../ui/toast";
+import { RoutineDialog } from "../routines/RoutineDialog";
+import { lessonText, stepText } from "./lessonText";
+
+export interface Lesson {
+ /** The steps so far, while the owner teaches; `null` otherwise. */
+ steps: LessonStep[] | null;
+ start(): Promise;
+ /** Ends the lesson and opens what to do with it. */
+ finish(): Promise;
+ cancel(): Promise;
+ /** The steps of the lesson just finished, until the owner is done with them. */
+ done: LessonStep[] | null;
+ close(): void;
+}
+
+export function useLesson(bot: Pick, state: BrowserState | null): Lesson {
+ const api = useApi();
+ const t = useT().browser.lesson;
+ const [done, setDone] = useState(null);
+ const teach = async (on: boolean) => {
+ let steps: LessonStep[] = [];
+ const ok = await attempt(t.failed, async () => {
+ steps = await api.call("browser.teach", { botId: bot.id, on });
+ });
+ return ok ? steps : null;
+ };
+ return {
+ steps: state?.lesson ?? null,
+ start: async () => {
+ await teach(true);
+ },
+ finish: async () => {
+ const steps = await teach(false);
+ if (steps) {
+ setDone(steps);
+ }
+ },
+ cancel: async () => {
+ await teach(false);
+ },
+ done,
+ close: () => setDone(null),
+ };
+}
+
+/** Under the owner's pill: start a lesson, or the one being given, live. */
+export function LessonArea({ bot, lesson }: { bot: Bot; lesson: Lesson }) {
+ const t = useT().browser.lesson;
+ if (lesson.steps === null) {
+ return (
+
+
+
{t.teachWhy(bot.name)}
+
+ );
+ }
+ return (
+
+
+
+ {t.recording}
+
+ {lesson.steps.length === 0 ? (
+
{t.empty}
+ ) : (
+
+ {lesson.steps.map((step, index) => (
+ // biome-ignore lint/suspicious/noArrayIndexKey: steps only grow at the end
+
+ {index + 1}. {stepText(step, t)}
+
+ ))}
+
+ )}
+ {/* The pill above finishes it; here it can be dropped. */}
+
+
+
+
+ );
+}
+
+/** What to do with the lesson just finished: a routine, or a message. */
+export function LessonDialog({ bot, lesson }: { bot: Bot; lesson: Lesson }) {
+ const t = useT().browser.lesson;
+ const api = useApi();
+ const [name, setName] = useState("");
+ const [steps, setSteps] = useState(lesson.done ?? []);
+ const [routine, setRoutine] = useState(false);
+ const ready = name.trim().length > 0 && steps.length > 0;
+ const text = lessonText(name, steps, t);
+
+ const send = async (event?: FormEvent) => {
+ event?.preventDefault();
+ if (!ready) return;
+ const body = `${text}\n\n${t.forMemory(name.trim())}`;
+ // It shows in the bot's chat, beside the panel.
+ if (await attempt(t.sendFailed, () => api.call("messages.send", { botId: bot.id, body }))) {
+ lesson.close();
+ }
+ };
+
+ if (routine) {
+ return (
+
+ );
+ }
+ return (
+
+ );
+}
diff --git a/app/src/features/browser/lesson.test.tsx b/app/src/features/browser/lesson.test.tsx
new file mode 100644
index 00000000..ea531873
--- /dev/null
+++ b/app/src/features/browser/lesson.test.tsx
@@ -0,0 +1,113 @@
+import { act, cleanup, fireEvent, screen, waitFor, within } from "@testing-library/react";
+import { afterEach, describe, expect, test } from "vitest";
+import { en } from "../../i18n/en";
+import { FakeBotloft } from "../../lib/fake";
+import { crewOpened, openBot, renderApp } from "../../test/app";
+import { lessonText } from "./lessonText";
+
+afterEach(cleanup);
+
+/** Scout's browser open on a sign-in page, in the owner's hands. */
+async function inHands() {
+ const fake = new FakeBotloft();
+ const ops = fake.addCrew("Ops");
+ const scout = fake.addBot(ops.id, "Scout", "Finds sources");
+ fake.setBotState(scout.id, "busy");
+ fake.browser.open(scout.id, "https://shop.example/login?next=/orders", "Sign in");
+ renderApp(fake);
+ await crewOpened("Ops");
+ openBot("Scout");
+ await screen.findByRole("list", { name: "Messages" });
+ fireEvent.click(screen.getByRole("button", { name: /^Show browser/ }));
+ await waitFor(() => expect(fake.browser.watching).toBe(scout.id));
+ act(() => {
+ fake.browser.frame(scout.id, "AAAA");
+ });
+ fireEvent.click(screen.getByRole("button", { name: "Take control" }));
+ await screen.findByText("You are in control");
+ return { fake, scout };
+}
+
+/** The daemon reads what the owner did as steps. */
+function teachSteps(fake: FakeBotloft, botId: string) {
+ act(() => {
+ fake.lesson.step(botId, { kind: "click", label: "E-mail", role: "textbox" });
+ fake.lesson.step(botId, { kind: "type", label: "E-mail", role: "textbox" });
+ fake.lesson.step(botId, { kind: "type", label: "Password", role: "textbox", secret: true });
+ fake.lesson.step(botId, { kind: "press", label: "Enter" });
+ });
+}
+
+async function finish(fake: FakeBotloft, botId: string) {
+ fireEvent.click(screen.getByRole("button", { name: "Teach a task" }));
+ const recording = await screen.findByRole("region", { name: "Recording the lesson" });
+ expect(within(recording).getByText("1. Open https://shop.example/login")).toBeDefined();
+ teachSteps(fake, botId);
+ expect(
+ within(recording).getByText('4. Type the password in "Password": ask me to type it'),
+ ).toBeDefined();
+ // The pill ends the lesson before giving anything back.
+ fireEvent.click(screen.getByRole("button", { name: "Finish the lesson" }));
+ const dialog = await screen.findByRole("dialog", { name: "Teach Scout a task" });
+ fireEvent.change(within(dialog).getByLabelText("Name of the task"), {
+ target: { value: "Check my orders" },
+ });
+ return dialog;
+}
+
+describe("teaching a task", () => {
+ test("the words say where things were typed, never what", () => {
+ const text = lessonText(
+ "Check my orders",
+ [
+ { kind: "open", label: "https://shop.example/login", role: null, secret: false },
+ { kind: "type", label: "E-mail", role: "textbox", secret: false },
+ { kind: "type", label: "Password", role: "textbox", secret: true },
+ ],
+ en.browser.lesson,
+ );
+ expect(text).toBe(
+ [
+ 'How to do "Check my orders" in the browser, as I showed you:',
+ "1. Open https://shop.example/login",
+ '2. Type in "E-mail"',
+ '3. Type the password in "Password": ask me to type it',
+ "",
+ "Where I typed something, I don't show what: use what the task needs, or ask me.",
+ ].join("\n"),
+ );
+ });
+
+ test("a lesson records live and goes to the bot to remember, without the steps taken out", async () => {
+ const { fake, scout } = await inHands();
+ const dialog = await finish(fake, scout.id);
+ fireEvent.click(within(dialog).getByRole("button", { name: "Remove step 2" }));
+ fireEvent.click(within(dialog).getByRole("button", { name: "Send to Scout to remember" }));
+ await waitFor(() =>
+ expect(fake.calls.some((call) => call.method === "messages.send")).toBe(true),
+ );
+ const sent = fake.calls.find((call) => call.method === "messages.send")?.params as {
+ body: string;
+ };
+ expect(sent.body).toContain('How to do "Check my orders"');
+ expect(sent.body).toContain('2. Type in "E-mail"');
+ expect(sent.body).not.toContain('Click "E-mail"');
+ expect(sent.body).toContain('so you can do "Check my orders" when I ask.');
+ await waitFor(() => expect(screen.queryByRole("dialog")).toBeNull());
+ // The owner still has the browser.
+ expect(screen.getByText("You are in control")).toBeDefined();
+ });
+
+ test("a lesson becomes a routine, its request already written", async () => {
+ const { fake, scout } = await inHands();
+ const dialog = await finish(fake, scout.id);
+ fireEvent.click(within(dialog).getByRole("button", { name: "Make it a routine…" }));
+ const routine = await screen.findByRole("dialog", { name: "New routine for Scout" });
+ expect((within(routine).getByLabelText("Name") as HTMLInputElement).value).toBe(
+ "Check my orders",
+ );
+ const prompt = within(routine).getByLabelText("What should Scout do?") as HTMLTextAreaElement;
+ expect(prompt.value).toContain("1. Open https://shop.example/login");
+ expect(prompt.value.endsWith("Do it now.")).toBe(true);
+ });
+});
diff --git a/app/src/features/browser/lessonText.ts b/app/src/features/browser/lessonText.ts
new file mode 100644
index 00000000..aeb1af8b
--- /dev/null
+++ b/app/src/features/browser/lessonText.ts
@@ -0,0 +1,28 @@
+// A lesson as words (spec 21.13): the steps the owner showed, numbered, in
+// the owner's language, for a routine's request or a message to the bot.
+// What was typed was never kept, so the words say where, not what.
+
+import type { Messages } from "../../i18n/en";
+import type { LessonStep } from "../../lib/protocol.gen";
+
+type Words = Messages["browser"]["lesson"];
+
+export function stepText(step: LessonStep, w: Words): string {
+ switch (step.kind) {
+ case "open":
+ return w.step.open(step.label);
+ case "click":
+ return w.step.click(step.label);
+ case "type":
+ return step.secret ? w.step.secret(step.label) : w.step.type(step.label);
+ case "press":
+ return w.step.press(step.label);
+ }
+}
+
+/** The whole lesson: what it is, each step, and a word on what was typed. */
+export function lessonText(name: string, steps: LessonStep[], w: Words): string {
+ const lines = steps.map((step, index) => `${index + 1}. ${stepText(step, w)}`);
+ const typed = steps.some((step) => step.kind === "type" && !step.secret);
+ return [w.intro(name.trim()), ...lines, ...(typed ? ["", w.typedNote] : [])].join("\n");
+}
diff --git a/app/src/features/routines/RoutineDialog.tsx b/app/src/features/routines/RoutineDialog.tsx
index 40a5db19..f0b208ee 100644
--- a/app/src/features/routines/RoutineDialog.tsx
+++ b/app/src/features/routines/RoutineDialog.tsx
@@ -15,14 +15,21 @@ import { blankForm, formOf, type RoutineForm, scheduleOf } from "./form";
import { MoreOptions } from "./MoreOptions";
import { ScheduleFields } from "./ScheduleFields";
-type Props = { bot: Bot; onClose(): void } & ({ routine?: undefined } | { routine: Routine });
+type Props = {
+ bot: Bot;
+ onClose(): void;
+ /** A new routine's name and request, already written (a lesson, spec 21.13). */
+ start?: { name: string; prompt: string };
+} & ({ routine?: undefined } | { routine: Routine });
-export function RoutineDialog({ bot, routine, onClose }: Props) {
+export function RoutineDialog({ bot, routine, start, onClose }: Props) {
const t = useT();
const d = t.routines.dialog;
const api = useApi();
const putRoutine = useApp((state) => state.putRoutine);
- const [form, setForm] = useState(() => (routine ? formOf(routine) : blankForm()));
+ const [form, setForm] = useState(() =>
+ routine ? formOf(routine) : { ...blankForm(), ...start },
+ );
const [error, setError] = useState(null);
const [busy, setBusy] = useState(false);
const change = (patch: Partial) => setForm((current) => ({ ...current, ...patch }));
diff --git a/app/src/i18n/en/browser.ts b/app/src/i18n/en/browser.ts
index 1bdc5d42..82c5cfed 100644
--- a/app/src/i18n/en/browser.ts
+++ b/app/src/i18n/en/browser.ts
@@ -83,6 +83,36 @@ export const browser = {
typing: "What you type goes to the page",
screen: (bot: string) => `${bot}'s browser, in your hands`,
},
+ /** Teaching the bot a task in its browser (spec 21.13). */
+ lesson: {
+ teach: "Teach a task",
+ teachWhy: (bot: string) =>
+ `Do the task here once: ${bot} keeps each step, never what you type.`,
+ recording: "Recording the lesson",
+ empty: "Do the task on the page: each step shows up here.",
+ finish: "Finish the lesson",
+ cancel: "Cancel",
+ failed: "Could not start or end the lesson",
+ title: (bot: string) => `Teach ${bot} a task`,
+ name: "Name of the task",
+ namePlaceholder: "Check new listings",
+ steps: "Steps",
+ remove: (n: number) => `Remove step ${n}`,
+ makeRoutine: "Make it a routine…",
+ sendToBot: (bot: string) => `Send to ${bot} to remember`,
+ sendFailed: "Could not send the lesson",
+ step: {
+ open: (url: string) => `Open ${url}`,
+ click: (label: string) => `Click "${label}"`,
+ type: (label: string) => `Type in "${label}"`,
+ secret: (label: string) => `Type the password in "${label}": ask me to type it`,
+ press: (key: string) => `Press ${key}`,
+ },
+ intro: (name: string) => `How to do "${name}" in the browser, as I showed you:`,
+ typedNote: "Where I typed something, I don't show what: use what the task needs, or ask me.",
+ forRoutine: "Do it now.",
+ forMemory: (name: string) => `Keep this in your memory, so you can do "${name}" when I ask.`,
+ },
window: {
open: "Sign in in a window",
why: (bot: string) =>
diff --git a/app/src/i18n/es/browser.ts b/app/src/i18n/es/browser.ts
index 22fd0671..264f29eb 100644
--- a/app/src/i18n/es/browser.ts
+++ b/app/src/i18n/es/browser.ts
@@ -81,6 +81,34 @@ export const browser: Messages["browser"] = {
typing: "Lo que escribes va a la página",
screen: (bot) => `Navegador de ${bot}, en tus manos`,
},
+ lesson: {
+ teach: "Enseñar una tarea",
+ teachWhy: (bot) => `Haz la tarea aquí una vez: ${bot} guarda cada paso, nunca lo que escribes.`,
+ recording: "Grabando la lección",
+ empty: "Haz la tarea en la página: cada paso aparece aquí.",
+ finish: "Terminar la lección",
+ cancel: "Cancelar",
+ failed: "No se pudo empezar o terminar la lección",
+ title: (bot) => `Enseñar una tarea a ${bot}`,
+ name: "Nombre de la tarea",
+ namePlaceholder: "Ver anuncios nuevos",
+ steps: "Pasos",
+ remove: (n) => `Quitar el paso ${n}`,
+ makeRoutine: "Convertir en rutina…",
+ sendToBot: (bot) => `Enviar a ${bot} para que lo guarde`,
+ sendFailed: "No se pudo enviar la lección",
+ step: {
+ open: (url) => `Abre ${url}`,
+ click: (label) => `Haz clic en "${label}"`,
+ type: (label) => `Escribe en "${label}"`,
+ secret: (label) => `Escribe la contraseña en "${label}": pídeme que la escriba`,
+ press: (key) => `Pulsa ${key}`,
+ },
+ intro: (name) => `Cómo hacer "${name}" en el navegador, como te mostré:`,
+ typedNote: "Donde escribí algo, no muestro qué: usa lo que pida la tarea, o pregúntame.",
+ forRoutine: "Hazlo ahora.",
+ forMemory: (name) => `Guarda esto en tu memoria, para hacer "${name}" cuando te lo pida.`,
+ },
window: {
open: "Iniciar sesión en una ventana",
why: (bot) =>
diff --git a/app/src/i18n/pt-BR/browser.ts b/app/src/i18n/pt-BR/browser.ts
index 38ecba57..42f66965 100644
--- a/app/src/i18n/pt-BR/browser.ts
+++ b/app/src/i18n/pt-BR/browser.ts
@@ -83,6 +83,36 @@ export const browser: Messages["browser"] = {
typing: "O que você digita vai para a página",
screen: (bot) => `Navegador de ${bot}, nas suas mãos`,
},
+ lesson: {
+ teach: "Ensinar uma tarefa",
+ teachWhy: (bot) =>
+ `Faça a tarefa aqui uma vez: ${bot} guarda cada passo, nunca o que você digita.`,
+ recording: "Gravando a lição",
+ empty: "Faça a tarefa na página: cada passo aparece aqui.",
+ finish: "Terminar a lição",
+ cancel: "Cancelar",
+ failed: "Não foi possível começar ou terminar a lição",
+ title: (bot) => `Ensinar uma tarefa a ${bot}`,
+ name: "Nome da tarefa",
+ namePlaceholder: "Ver anúncios novos",
+ steps: "Passos",
+ remove: (n) => `Tirar o passo ${n}`,
+ makeRoutine: "Fazer virar rotina…",
+ sendToBot: (bot) => `Mandar para ${bot} guardar`,
+ sendFailed: "Não foi possível mandar a lição",
+ step: {
+ open: (url) => `Abra ${url}`,
+ click: (label) => `Clique em "${label}"`,
+ type: (label) => `Escreva em "${label}"`,
+ secret: (label) => `Escreva a senha em "${label}": peça para eu digitar`,
+ press: (key) => `Aperte ${key}`,
+ },
+ intro: (name) => `Como fazer "${name}" no navegador, do jeito que eu mostrei:`,
+ typedNote:
+ "Onde eu escrevi algo, eu não mostro o quê: use o que a tarefa pede, ou me pergunte.",
+ forRoutine: "Faça isso agora.",
+ forMemory: (name) => `Guarde isso na sua memória, para fazer "${name}" quando eu pedir.`,
+ },
window: {
open: "Entrar numa janela",
why: (bot) =>
diff --git a/app/src/lib/fake.ts b/app/src/lib/fake.ts
index 13b75c38..0a6edd68 100644
--- a/app/src/lib/fake.ts
+++ b/app/src/lib/fake.ts
@@ -10,6 +10,7 @@ import { FakeChat } from "./fakeChat";
import { FakeConversation } from "./fakeConversation";
import { crewHandlers } from "./fakeCrews";
import { FakeFiles } from "./fakeFiles";
+import { FakeLesson } from "./fakeLesson";
import { FakeQuestions } from "./fakeQuestions";
import { FakeReactions } from "./fakeReactions";
import { FakeRoutines } from "./fakeRoutines";
@@ -79,6 +80,7 @@ export class FakeBotloft implements BotloftApi {
readonly screens = new FakeScreens(this);
readonly questions = new FakeQuestions(this);
readonly reactions = new FakeReactions(this);
+ readonly lesson = new FakeLesson(this);
/** Clock for created and updated times. */
now = Date.now();
closed = false;
@@ -264,6 +266,7 @@ export class FakeBotloft implements BotloftApi {
...this.routines.handlers(),
...this.questions.handlers(),
...this.reactions.handlers(),
+ ...this.lesson.handlers(),
...usageHandlers(this),
...searchHandlers(this),
};
diff --git a/app/src/lib/fakeBrowser.ts b/app/src/lib/fakeBrowser.ts
index 6b525daa..85145890 100644
--- a/app/src/lib/fakeBrowser.ts
+++ b/app/src/lib/fakeBrowser.ts
@@ -53,6 +53,7 @@ export class FakeBrowser {
resting: false,
ask: null,
window: false,
+ lesson: null,
updatedAt: this.fake.now,
}
);
@@ -188,11 +189,12 @@ export class FakeBrowser {
if (botId === null || state?.control !== "owner" || state.window) {
return null;
}
- return this.set(botId, { control: "bot" });
+ // Giving the browser back ends a lesson not finished (spec 21.13).
+ return this.set(botId, { control: "bot", lesson: null });
}
/** The owner has the watched browser, or the call is refused. */
- private held(botId: BotId): void {
+ held(botId: BotId): void {
if (this.watching !== botId || this.state(botId).control !== "owner") {
throw conflict("take the browser before using it");
}
diff --git a/app/src/lib/fakeLesson.ts b/app/src/lib/fakeLesson.ts
new file mode 100644
index 00000000..89d99cd6
--- /dev/null
+++ b/app/src/lib/fakeLesson.ts
@@ -0,0 +1,39 @@
+// The fake daemon's lessons (spec 21.13): teaching a bot a task with its
+// browser in the owner's hands. `step` plays the daemon reading what the
+// owner did; `browser.teach` starts and ends a lesson as the daemon does.
+
+import type { FakeBotloft, Handlers } from "./fake";
+import type { BotId, LessonStep } from "./protocol.gen";
+
+export class FakeLesson {
+ constructor(private readonly fake: FakeBotloft) {}
+
+ /** The owner did something that means `step`, while teaching. */
+ step(botId: BotId, step: Partial & Pick): void {
+ const lesson = this.fake.browser.state(botId).lesson;
+ if (lesson) {
+ this.fake.browser.set(botId, {
+ lesson: [...lesson, { role: null, secret: false, ...step }],
+ });
+ }
+ }
+
+ handlers(): Pick {
+ return {
+ "browser.teach": ({ botId, on }) => {
+ this.fake.browser.held(botId);
+ const state = this.fake.browser.state(botId);
+ if (on) {
+ const url = state.url?.split(/[?#]/)[0] ?? null;
+ const steps: LessonStep[] = url
+ ? [{ kind: "open", label: url, role: null, secret: false }]
+ : [];
+ this.fake.browser.set(botId, { lesson: steps });
+ return steps;
+ }
+ this.fake.browser.set(botId, { lesson: null });
+ return state.lesson ?? [];
+ },
+ };
+ }
+}
diff --git a/app/src/lib/protocol.gen.ts b/app/src/lib/protocol.gen.ts
index c9034d31..10b47a15 100644
--- a/app/src/lib/protocol.gen.ts
+++ b/app/src/lib/protocol.gen.ts
@@ -1233,6 +1233,23 @@ title: string, url: string,
*/
active: boolean, };
+export type LessonStepKind = "open" | "click" | "type" | "press";
+
+export type LessonStep = { kind: LessonStepKind,
+/**
+ * The address for `open`, the element's name for `click` and `type`,
+ * the key for `press`; at most 80 characters.
+ */
+label: string,
+/**
+ * The element's role (`button`, `link`, `textbox`...), when it has one.
+ */
+role: string | null,
+/**
+ * Typing in a password field: the bot asks the owner to type it.
+ */
+secret: boolean, };
+
export type BrowserState = { botId: BotId, status: BrowserStatus,
/**
* The active tab's address.
@@ -1269,6 +1286,11 @@ ask: string | null,
* `closed` meanwhile, and the bot's tools wait.
*/
window: boolean,
+/**
+ * The steps of the lesson the owner is giving, while they teach the
+ * bot a task (spec 21.13); `null` when nobody teaches.
+ */
+lesson: Array | null,
/**
* Unix time in milliseconds.
*/
@@ -1356,6 +1378,11 @@ export type BrowserTabParams = { botId: BotId, tabId: string, };
*/
export type BrowserOpenParams = { botId: BotId, url: string, };
+/**
+ * Starts or ends a lesson in the browser the connection has in its hands.
+ */
+export type BrowserTeachParams = { botId: BotId, on: boolean, };
+
/**
* The device a screen is drawn for.
*/
@@ -1477,6 +1504,7 @@ export interface RpcMethods {
"browser.take": { params: BrowserControlParams; result: BrowserState };
"browser.release": { params: BrowserControlParams; result: BrowserState };
"browser.window": { params: BrowserControlParams; result: BrowserState };
+ "browser.teach": { params: BrowserTeachParams; result: Array };
"browser.input": { params: BrowserInputParams; result: null };
"browser.reload": { params: BrowserControlParams; result: null };
"browser.newTab": { params: BrowserControlParams; result: null };
diff --git a/crates/botloft-core/src/protocol/browser.rs b/crates/botloft-core/src/protocol/browser.rs
index 48a6aff3..0360b8f5 100644
--- a/crates/botloft-core/src/protocol/browser.rs
+++ b/crates/botloft-core/src/protocol/browser.rs
@@ -3,7 +3,7 @@
use serde::{Deserialize, Serialize};
-use super::BrowserControl;
+use super::{BrowserControl, LessonStep};
use crate::ids::BotId;
/// Where a bot's browser is.
@@ -59,6 +59,10 @@ pub struct BrowserState {
/// site refuses a browser a program drives (spec 21.11). It is
/// `closed` meanwhile, and the bot's tools wait.
pub window: bool,
+ /// The steps of the lesson the owner is giving, while they teach the
+ /// bot a task (spec 21.13); `null` when nobody teaches.
+ #[serde(default)]
+ pub lesson: Option>,
/// Unix time in milliseconds.
pub updated_at: i64,
}
@@ -77,6 +81,7 @@ impl BrowserState {
resting: false,
ask: None,
window: false,
+ lesson: None,
updated_at: now,
}
}
diff --git a/crates/botloft-core/src/protocol/export_browser.rs b/crates/botloft-core/src/protocol/export_browser.rs
index d6c4fa92..e27d9910 100644
--- a/crates/botloft-core/src/protocol/export_browser.rs
+++ b/crates/botloft-core/src/protocol/export_browser.rs
@@ -9,6 +9,8 @@ pub(super) fn decls(out: &mut Out) {
out.decl::();
out.decl::();
out.decl::();
+ out.decl::();
+ out.decl::();
out.decl::();
out.decl::();
out.decl::();
@@ -23,6 +25,7 @@ pub(super) fn decls(out: &mut Out) {
out.decl::();
out.decl::();
out.decl::();
+ out.decl::();
out.decl::();
out.decl::();
out.decl::();
@@ -51,6 +54,11 @@ pub(super) fn methods(out: &mut Out) {
out.method(method::BROWSER_TAKE, &control, &state);
out.method(method::BROWSER_RELEASE, &control, &state);
out.method(method::BROWSER_WINDOW, &control, &state);
+ out.method(
+ method::BROWSER_TEACH,
+ &out.name::(),
+ &out.name::>(),
+ );
out.method(
method::BROWSER_INPUT,
&out.name::(),
diff --git a/crates/botloft-core/src/protocol/lesson.rs b/crates/botloft-core/src/protocol/lesson.rs
new file mode 100644
index 00000000..2cb33782
--- /dev/null
+++ b/crates/botloft-core/src/protocol/lesson.rs
@@ -0,0 +1,48 @@
+//! Teaching a bot a task (spec 21.13): while the owner has the bot's
+//! browser in their hands, what they do is kept as steps by what it means
+//! (the page, the button, the field), never by what they typed.
+
+use serde::{Deserialize, Serialize};
+
+use crate::ids::BotId;
+
+/// Most steps one lesson keeps; later ones are left out.
+pub const LESSON_STEPS_MAX: usize = 100;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+#[cfg_attr(test, derive(ts_rs::TS))]
+pub enum LessonStepKind {
+ /// A web address in the active tab: where the lesson began, or one
+ /// the owner typed.
+ Open,
+ /// A click on something with a role (a button, a link, a field...).
+ Click,
+ /// Typing in a field. What was typed is never kept.
+ Type,
+ /// A key that does something: Enter, Escape, Tab.
+ Press,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+#[cfg_attr(test, derive(ts_rs::TS))]
+pub struct LessonStep {
+ pub kind: LessonStepKind,
+ /// The address for `open`, the element's name for `click` and `type`,
+ /// the key for `press`; at most 80 characters.
+ pub label: String,
+ /// The element's role (`button`, `link`, `textbox`...), when it has one.
+ pub role: Option,
+ /// Typing in a password field: the bot asks the owner to type it.
+ pub secret: bool,
+}
+
+/// Starts or ends a lesson in the browser the connection has in its hands.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+#[cfg_attr(test, derive(ts_rs::TS))]
+pub struct BrowserTeachParams {
+ pub bot_id: BotId,
+ pub on: bool,
+}
diff --git a/crates/botloft-core/src/protocol/mod.rs b/crates/botloft-core/src/protocol/mod.rs
index f8b9a963..b3f19547 100644
--- a/crates/botloft-core/src/protocol/mod.rs
+++ b/crates/botloft-core/src/protocol/mod.rs
@@ -55,6 +55,7 @@ mod chat;
#[cfg(test)]
mod export;
mod files;
+mod lesson;
mod messaging;
mod methods;
mod model;
@@ -72,6 +73,7 @@ pub use browser::*;
pub use browser_input::*;
pub use chat::*;
pub use files::*;
+pub use lesson::*;
pub use messaging::*;
pub use methods::*;
pub use model::*;
@@ -146,6 +148,7 @@ pub mod method {
pub const BROWSER_SWITCH_TAB: &str = "browser.switchTab";
pub const BROWSER_OPEN: &str = "browser.open";
pub const BROWSER_WINDOW: &str = "browser.window";
+ pub const BROWSER_TEACH: &str = "browser.teach";
pub const SCREENS_LIST: &str = "screens.list";
pub const QUESTIONS_LIST: &str = "questions.list";
pub const QUESTIONS_ANSWER: &str = "questions.answer";
diff --git a/crates/botloftd/src/browser/actions.js b/crates/botloftd/src/browser/actions.js
index 0cd44651..e391b863 100644
--- a/crates/botloftd/src/browser/actions.js
+++ b/crates/botloftd/src/browser/actions.js
@@ -1,6 +1,8 @@
// What a bot does on the page (spec 21.4), beside the page reader: where to
// click an element, getting a text field ready for typing, choosing an
-// option and scrolling to an end. `reader.js` hands in its helpers.
+// option and scrolling to an end; and, for a lesson (spec 21.13), what is
+// at a point and which field has the focus. `reader.js` hands in its
+// helpers.
({ get, clean, styleOf, roleOf, nameOf, token, isTextField, next }) => {
// The element's box in the top page's coordinates, through same-site frames.
const topRect = (el) => {
@@ -90,5 +92,25 @@
return { top: window.scrollY };
};
- return { point, prepareType, select, scrollEnd };
+ // What the owner acts on while teaching a task (spec 21.13): the nearest
+ // element with a role, by its name. Never what a field holds.
+ const meaning = (el) => {
+ let shown = el;
+ while (shown && !roleOf(shown, styleOf(shown), "auto")) shown = shown.parentElement;
+ if (!shown) return { none: true };
+ return {
+ role: roleOf(shown, styleOf(shown), "auto"),
+ label: clean(nameOf(shown), 80),
+ secret:
+ shown.localName === "input" && (shown.getAttribute("type") || "").toLowerCase() === "password",
+ };
+ };
+ const at = (x, y) => meaning(document.elementFromPoint(x, y));
+ const focused = () => {
+ let el = document.activeElement;
+ while (el?.shadowRoot?.activeElement) el = el.shadowRoot.activeElement;
+ return el && el !== document.body && isTextField(el) ? meaning(el) : { none: true };
+ };
+
+ return { point, prepareType, select, scrollEnd, at, focused };
}
diff --git a/crates/botloftd/src/browser/hands.rs b/crates/botloftd/src/browser/hands.rs
index ea64e2c7..5ca72299 100644
--- a/crates/botloftd/src/browser/hands.rs
+++ b/crates/botloftd/src/browser/hands.rs
@@ -7,11 +7,12 @@ use std::sync::atomic::Ordering;
use std::time::{Duration, Instant};
use botloft_core::ids::{ApprovalId, BotId};
-use botloft_core::protocol::{BrowserControl, BrowserInput};
+use botloft_core::protocol::{BrowserControl, BrowserInput, LessonStep, LessonStepKind};
use tokio::sync::{broadcast, mpsc};
use tracing::debug;
use super::input::fits;
+use super::lesson::{self, Teacher};
use super::moves::{Move, feed};
use super::rest::wake;
use super::session::Session;
@@ -84,7 +85,13 @@ impl Browsers {
tokio::spawn(async move {
// A browser at rest wakes for the owner's hands.
wake(&page, &slots, &events, clock.as_ref(), &owner).await;
- feed(page, queue).await;
+ let teacher = Teacher {
+ slots,
+ events,
+ clock,
+ bot: owner,
+ };
+ feed(page, queue, teacher).await;
});
Ok(Hands {
slots: Arc::clone(&self.slots),
@@ -222,6 +229,40 @@ impl Hands {
self.make(Move::SwitchTab(id.to_owned()))
}
+ /// Starts a lesson (spec 21.13), from the page the active tab shows, or
+ /// ends it; the steps so far.
+ pub fn teach(&self, on: bool) -> Result, InputError> {
+ self.page()?;
+ let start = on
+ .then(|| self.session.lock().active().map(|tab| tab.url.clone()))
+ .flatten()
+ .and_then(|url| lesson::address(&url));
+ let mut steps = Vec::new();
+ update(
+ &self.slots,
+ &self.events,
+ self.clock.as_ref(),
+ &self.bot,
+ |state| {
+ if on {
+ steps = start
+ .map(|url| LessonStep {
+ kind: LessonStepKind::Open,
+ label: url,
+ role: None,
+ secret: false,
+ })
+ .into_iter()
+ .collect();
+ state.lesson = Some(steps.clone());
+ } else {
+ steps = state.lesson.take().unwrap_or_default();
+ }
+ },
+ );
+ Ok(steps)
+ }
+
/// Takes the active tab to the web address the owner typed.
pub fn open(&self, address: &str) -> Result<(), InputError> {
self.page()?;
@@ -245,7 +286,11 @@ impl Drop for Hands {
&self.events,
self.clock.as_ref(),
&self.bot,
- |state| state.control = BrowserControl::Bot,
+ |state| {
+ state.control = BrowserControl::Bot;
+ // Giving the browser back ends a lesson not finished.
+ state.lesson = None;
+ },
);
debug!(bot = %self.bot, "browser: the owner gave it back");
}
diff --git a/crates/botloftd/src/browser/lesson.rs b/crates/botloftd/src/browser/lesson.rs
new file mode 100644
index 00000000..d6207266
--- /dev/null
+++ b/crates/botloftd/src/browser/lesson.rs
@@ -0,0 +1,184 @@
+//! Teaching a bot a task (spec 21.13): while the owner has the browser in
+//! their hands and teaches, each thing they do becomes a step by what it
+//! means, read from the page before it reaches it: the button clicked, the
+//! field typed in, the address opened. What they type is never read.
+
+use std::sync::Arc;
+
+use botloft_core::ids::BotId;
+use botloft_core::protocol::{
+ BrowserInput, LESSON_STEPS_MAX, LessonStep, LessonStepKind, MouseAction, MouseButton, modifier,
+};
+use serde_json::Value;
+use tokio::sync::broadcast;
+
+use super::moves::Move;
+use super::session::Session;
+use super::{Slots, lock, update};
+use crate::clock::Clock;
+use crate::state::Event;
+
+/// Longest label a step keeps, in characters.
+const LABEL_MAX: usize = 80;
+
+/// Where a lesson's steps go: the bot's browser state, which the app
+/// follows.
+pub(super) struct Teacher {
+ pub slots: Slots,
+ pub events: broadcast::Sender,
+ pub clock: Arc,
+ pub bot: BotId,
+}
+
+impl Teacher {
+ fn teaching(&self) -> bool {
+ lock(&self.slots)
+ .get(&self.bot)
+ .is_some_and(|slot| slot.state.lesson.is_some())
+ }
+
+ /// Adds a step to the lesson going on, if one is. Typing on in the
+ /// field of the step before is the same step.
+ fn add(&self, step: LessonStep) {
+ update(
+ &self.slots,
+ &self.events,
+ self.clock.as_ref(),
+ &self.bot,
+ |state| {
+ let Some(steps) = state.lesson.as_mut() else {
+ return;
+ };
+ let same_field = |last: &LessonStep| {
+ last.kind == LessonStepKind::Type
+ && step.kind == LessonStepKind::Type
+ && last.label == step.label
+ };
+ if steps.last().is_some_and(same_field) || steps.len() >= LESSON_STEPS_MAX {
+ return;
+ }
+ steps.push(step);
+ },
+ );
+ }
+
+ /// What `step` adds to the lesson, read from the page before the step
+ /// reaches it.
+ pub(super) async fn observe(&self, session: &Session, step: &Move) {
+ if !self.teaching() {
+ return;
+ }
+ let found = match step {
+ Move::Open(url) => address(url).map(|url| plain(LessonStepKind::Open, url)),
+ Move::Input(BrowserInput::Mouse {
+ action: MouseAction::Down,
+ button: MouseButton::Left,
+ clicks,
+ x,
+ y,
+ ..
+ }) if *clicks <= 1 => {
+ read(session, &format!("at({x}, {y})"), LessonStepKind::Click).await
+ }
+ Move::Input(BrowserInput::Key { key, modifiers, .. }) => {
+ let plain_key = modifiers & (modifier::CTRL | modifier::ALT | modifier::META) == 0;
+ if plain_key && matches!(key.as_str(), "Enter" | "Escape" | "Tab") {
+ Some(plain(LessonStepKind::Press, key.clone()))
+ } else if plain_key && (key.chars().count() == 1 || key == "Backspace") {
+ read(session, "focused()", LessonStepKind::Type).await
+ } else {
+ None
+ }
+ }
+ Move::Input(BrowserInput::Text { .. }) => {
+ read(session, "focused()", LessonStepKind::Type).await
+ }
+ _ => None,
+ };
+ if let Some(found) = found {
+ self.add(found);
+ }
+ }
+}
+
+fn plain(kind: LessonStepKind, label: String) -> LessonStep {
+ LessonStep {
+ kind,
+ label,
+ role: None,
+ secret: false,
+ }
+}
+
+/// The element the page script found, as a step; `None` when nothing with
+/// a role is there.
+async fn read(session: &Session, call: &str, kind: LessonStepKind) -> Option {
+ let found = session.reader(call).await.ok()?;
+ meaning(&found, kind)
+}
+
+fn meaning(found: &Value, kind: LessonStepKind) -> Option {
+ if found["none"] == true {
+ return None;
+ }
+ let label: String = found["label"].as_str()?.chars().take(LABEL_MAX).collect();
+ Some(LessonStep {
+ kind,
+ label,
+ role: found["role"].as_str().map(str::to_owned),
+ secret: found["secret"] == true,
+ })
+}
+
+/// A web address as a lesson keeps it: its site and path, without the
+/// query or the fragment, where sign-in tokens and personal data travel.
+/// `None` for anything but `http` and `https`.
+pub(super) fn address(url: &str) -> Option {
+ let (scheme, rest) = url.split_once("://")?;
+ if !scheme.eq_ignore_ascii_case("http") && !scheme.eq_ignore_ascii_case("https") {
+ return None;
+ }
+ let rest = rest.split(['?', '#']).next()?;
+ let (authority, path) = rest.split_once('/').unwrap_or((rest, ""));
+ // Never a name and password written into the address.
+ let host = authority.rsplit('@').next()?;
+ if host.is_empty() {
+ return None;
+ }
+ let kept = format!("{}://{host}/{path}", scheme.to_ascii_lowercase());
+ Some(kept.chars().take(LABEL_MAX * 2).collect())
+}
+
+#[cfg(test)]
+mod tests {
+ use serde_json::json;
+
+ use super::*;
+
+ #[test]
+ fn an_address_keeps_its_site_and_path_only() {
+ assert_eq!(
+ address("https://mail.example.com/inbox?token=secret#msg-1").as_deref(),
+ Some("https://mail.example.com/inbox")
+ );
+ assert_eq!(
+ address("https://ana:hunter2@example.com").as_deref(),
+ Some("https://example.com/")
+ );
+ assert_eq!(address("about:blank"), None);
+ assert_eq!(address("file:///C:/notes.txt"), None);
+ }
+
+ #[test]
+ fn what_the_page_found_becomes_a_step_never_a_value() {
+ let field = json!({ "role": "textbox", "label": "Password", "secret": true });
+ let step = meaning(&field, LessonStepKind::Type).expect("step");
+ assert_eq!(step.label, "Password");
+ assert_eq!(step.role.as_deref(), Some("textbox"));
+ assert!(step.secret);
+ assert_eq!(
+ meaning(&json!({ "none": true }), LessonStepKind::Click),
+ None
+ );
+ }
+}
diff --git a/crates/botloftd/src/browser/mod.rs b/crates/botloftd/src/browser/mod.rs
index 18ff7868..c1a68fa0 100644
--- a/crates/botloftd/src/browser/mod.rs
+++ b/crates/botloftd/src/browser/mod.rs
@@ -11,6 +11,7 @@ mod hands;
mod input;
mod keys;
mod launch;
+mod lesson;
mod moves;
mod page;
mod program;
diff --git a/crates/botloftd/src/browser/moves.rs b/crates/botloftd/src/browser/moves.rs
index 0c9036f1..2b511469 100644
--- a/crates/botloftd/src/browser/moves.rs
+++ b/crates/botloftd/src/browser/moves.rs
@@ -12,6 +12,7 @@ use serde_json::json;
use tokio::sync::mpsc;
use tracing::debug;
+use super::lesson::Teacher;
use super::rest::wake;
use super::session::{Session, Tab};
use super::{BrowserError, Browsers};
@@ -173,7 +174,11 @@ impl Session {
/// Does what the owner did, one thing at a time, in order. Of several mouse
/// moves in a row only the newest matters. Each thing queued counts in
/// `owner_moves` until it is done or skipped.
-pub(super) async fn feed(session: Arc, mut queue: mpsc::UnboundedReceiver) {
+pub(super) async fn feed(
+ session: Arc,
+ mut queue: mpsc::UnboundedReceiver,
+ teacher: Teacher,
+) {
let done = || {
session
.owner_moves
@@ -190,6 +195,10 @@ pub(super) async fn feed(session: Arc, mut queue: mpsc::UnboundedReceiv
break;
}
}
+ if !session.is_closed() {
+ // A lesson reads the page before the step changes it.
+ teacher.observe(&session, &step).await;
+ }
if !session.is_closed()
&& let Err(err) = session.make(&step).await
{
diff --git a/crates/botloftd/src/rpc/mod.rs b/crates/botloftd/src/rpc/mod.rs
index c5e949c9..38f091f3 100644
--- a/crates/botloftd/src/rpc/mod.rs
+++ b/crates/botloftd/src/rpc/mod.rs
@@ -223,7 +223,8 @@ fn route(
| method::BROWSER_RELOAD
| method::BROWSER_NEW_TAB
| method::BROWSER_SWITCH_TAB
- | method::BROWSER_OPEN => watch.request(daemon, &request.method, request.params),
+ | method::BROWSER_OPEN
+ | method::BROWSER_TEACH => watch.request(daemon, &request.method, request.params),
// A window of its own outlives the request (spec 21.11).
method::BROWSER_WINDOW => watch.window(daemon, request.params),
name if ASIDE.contains(&name) => return Routed::Aside(request),
diff --git a/crates/botloftd/src/rpc/watching.rs b/crates/botloftd/src/rpc/watching.rs
index 0e2f9c5c..3eeb0c24 100644
--- a/crates/botloftd/src/rpc/watching.rs
+++ b/crates/botloftd/src/rpc/watching.rs
@@ -9,7 +9,8 @@ use std::sync::Arc;
use botloft_core::ids::BotId;
use botloft_core::protocol::{
ApprovalsAnswerParams, BrowserControlParams, BrowserInputParams, BrowserOpenParams,
- BrowserResizeParams, BrowserTabParams, BrowserWatchParams, error_code, method, notification,
+ BrowserResizeParams, BrowserTabParams, BrowserTeachParams, BrowserWatchParams, error_code,
+ method, notification,
};
use serde::Serialize;
use serde_json::Value;
@@ -47,6 +48,7 @@ impl Watch {
method::BROWSER_NEW_TAB => return self.new_tab(&parse(params)?),
method::BROWSER_SWITCH_TAB => return self.switch_tab(&parse(params)?),
method::BROWSER_OPEN => return self.open(&parse(params)?),
+ method::BROWSER_TEACH => return self.teach(&parse(params)?),
_ => {}
}
// Stop the one before, even if the new one is refused; the hands
@@ -188,6 +190,16 @@ impl Watch {
made(self.hands(¶ms.bot_id)?.open(¶ms.url))
}
+ /// Starts or ends a lesson (spec 21.13), with the browser in this
+ /// connection's hands.
+ fn teach(&self, params: &BrowserTeachParams) -> Result {
+ let steps = self
+ .hands(¶ms.bot_id)?
+ .teach(params.on)
+ .map_err(refused)?;
+ to_value(&steps)
+ }
+
/// The watched browser's next frame, as a notification. Never resolves
/// while nothing is watched.
pub(super) async fn next_frame(&mut self) -> Option {
@@ -220,12 +232,17 @@ fn helped(daemon: &Daemon, bot: &BotId) {
/// What the owner did is on its way to the browser, or why it is not.
fn made(queued: Result<(), InputError>) -> Result {
- queued.map_err(|err| match err {
+ queued.map_err(refused)?;
+ Ok(Value::Null)
+}
+
+fn refused(err: InputError) -> RpcError {
+ match err {
InputError::Invalid | InputError::Address => ApiError::validation(err.to_string()),
InputError::NoTab => ApiError::NotFound(err.to_string()),
InputError::NotHeld | InputError::Closed => ApiError::Conflict(err.to_string()),
- })?;
- Ok(Value::Null)
+ }
+ .into()
}
fn to_value(value: &impl Serialize) -> Result {
diff --git a/crates/botloftd/tests/browser_hands.rs b/crates/botloftd/tests/browser_hands.rs
index 5be6bc9f..25131719 100644
--- a/crates/botloftd/tests/browser_hands.rs
+++ b/crates/botloftd/tests/browser_hands.rs
@@ -8,38 +8,9 @@ mod common;
use std::time::Duration;
use common::Client;
-use common::browsing::{Browsing, answer_site, call, pending_approval, setup};
+use common::browsing::{Browsing, answer_site, call, click, pending_approval, press, send, setup};
use serde_json::{Value, json};
-async fn send(app: &mut Client, bot: &Value, input: Value) {
- app.call(
- "browser.input",
- json!({ "botId": bot["id"], "input": input }),
- )
- .await
- .expect("input");
-}
-
-async fn click(app: &mut Client, bot: &Value, x: f64, y: f64) {
- for (action, buttons) in [("move", 0), ("down", 1), ("up", 0)] {
- let input = json!({
- "kind": "mouse", "action": action, "x": x, "y": y, "button": "left",
- "buttons": buttons, "clicks": 1, "modifiers": 0,
- });
- send(app, bot, input).await;
- }
-}
-
-/// Presses the keys of `text`, one by one, as the app sends them.
-async fn press(app: &mut Client, bot: &Value, text: &str, modifiers: u32) {
- for key in text.chars() {
- let code = format!("Key{}", key.to_ascii_uppercase());
- let input =
- json!({ "kind": "key", "key": key.to_string(), "code": code, "modifiers": modifiers });
- send(app, bot, input).await;
- }
-}
-
/// The bot opens the sign-in page; the owner allows the site.
async fn sign_in_page(b: &mut Browsing) -> String {
let url = format!("{}/login", b.site);
diff --git a/crates/botloftd/tests/browser_lesson.rs b/crates/botloftd/tests/browser_lesson.rs
new file mode 100644
index 00000000..3622d5cd
--- /dev/null
+++ b/crates/botloftd/tests/browser_lesson.rs
@@ -0,0 +1,95 @@
+//! Teaching a bot a task with the real Microsoft Edge (spec 21.13): what
+//! the owner does becomes steps by what it means, never by what was typed.
+//! Skipped where Edge is not installed.
+
+mod common;
+
+use common::browsing::{Browsing, answer_site, call, click, press, send, setup};
+use serde_json::{Value, json};
+
+async fn sign_in_page(b: &mut Browsing) {
+ let url = format!("{}/login", b.site);
+ let opening = call(&b.mcp, "browser_open", json!({ "url": url }));
+ answer_site(&mut b.app, true, None).await;
+ opening.await.expect("task").expect("page");
+}
+
+async fn teach(b: &mut Browsing, on: bool) -> Value {
+ let params = json!({ "botId": b.bot["id"], "on": on });
+ b.app.call("browser.teach", params).await.expect("teach")
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn a_lesson_keeps_the_steps_and_never_what_was_typed() {
+ let Some(mut b) = setup().await else { return };
+ sign_in_page(&mut b).await;
+ let bot = b.bot.clone();
+ let id = json!({ "botId": bot["id"] });
+ b.app
+ .call("browser.watch", id.clone())
+ .await
+ .expect("watch");
+
+ // Only with the browser in the owner's hands.
+ let early = json!({ "botId": bot["id"], "on": true });
+ assert!(b.app.call("browser.teach", early).await.is_err());
+ b.app.call("browser.take", id.clone()).await.expect("take");
+
+ let started = teach(&mut b, true).await;
+ assert_eq!(started[0]["kind"], "open");
+ assert_eq!(started[0]["label"], format!("{}/login", b.site));
+
+ click(&mut b.app, &bot, 200.0, 115.0).await;
+ press(&mut b.app, &bot, "ana", 0).await;
+ click(&mut b.app, &bot, 200.0, 215.0).await;
+ send(
+ &mut b.app,
+ &bot,
+ json!({ "kind": "text", "text": "hunter2" }),
+ )
+ .await;
+ let enter = json!({ "kind": "key", "key": "Enter", "code": "Enter", "modifiers": 0 });
+ send(&mut b.app, &bot, enter).await;
+
+ // The steps reach the app as they happen.
+ let mut lesson = Value::Null;
+ for _ in 0..50 {
+ let list = b.app.call("browser.list", Value::Null).await.expect("list");
+ lesson = list[0]["lesson"].clone();
+ if lesson.as_array().is_some_and(|steps| steps.len() >= 6) {
+ break;
+ }
+ tokio::time::sleep(std::time::Duration::from_millis(100)).await;
+ }
+ let steps = teach(&mut b, false).await;
+ assert_eq!(steps, lesson);
+ let kinds: Vec<_> = steps
+ .as_array()
+ .expect("steps")
+ .iter()
+ .map(|step| {
+ format!(
+ "{} {}",
+ step["kind"].as_str().unwrap_or(""),
+ step["label"].as_str().unwrap_or("")
+ )
+ })
+ .collect();
+ assert_eq!(
+ kinds[1..],
+ [
+ "click User",
+ "type User",
+ "click Password",
+ "type Password",
+ "press Enter"
+ ]
+ );
+ assert_eq!(steps[4]["secret"], true);
+ assert!(!steps.to_string().contains("ana\""), "{steps}");
+ assert!(!steps.to_string().contains("hunter2"), "{steps}");
+
+ // Ended, nothing more is kept.
+ let list = b.app.call("browser.list", Value::Null).await.expect("list");
+ assert_eq!(list[0]["lesson"], Value::Null);
+}
diff --git a/crates/botloftd/tests/common/browsing.rs b/crates/botloftd/tests/common/browsing.rs
index 7db97165..87a593cb 100644
--- a/crates/botloftd/tests/common/browsing.rs
+++ b/crates/botloftd/tests/common/browsing.rs
@@ -80,3 +80,34 @@ pub fn ref_of(page: &str, what: &str) -> String {
.expect("ref")
.to_owned()
}
+
+/// One of the owner's events, as the app sends it.
+pub async fn send(app: &mut Client, bot: &Value, input: Value) {
+ app.call(
+ "browser.input",
+ json!({ "botId": bot["id"], "input": input }),
+ )
+ .await
+ .expect("input");
+}
+
+/// A left click at a point of the page.
+pub async fn click(app: &mut Client, bot: &Value, x: f64, y: f64) {
+ for (action, buttons) in [("move", 0), ("down", 1), ("up", 0)] {
+ let input = json!({
+ "kind": "mouse", "action": action, "x": x, "y": y, "button": "left",
+ "buttons": buttons, "clicks": 1, "modifiers": 0,
+ });
+ send(app, bot, input).await;
+ }
+}
+
+/// Presses the keys of `text`, one by one, as the app sends them.
+pub async fn press(app: &mut Client, bot: &Value, text: &str, modifiers: u32) {
+ for key in text.chars() {
+ let code = format!("Key{}", key.to_ascii_uppercase());
+ let input =
+ json!({ "kind": "key", "key": key.to_string(), "code": code, "modifiers": modifiers });
+ send(app, bot, input).await;
+ }
+}
diff --git a/docs/spec.md b/docs/spec.md
index b292f9ad..fb4995da 100644
--- a/docs/spec.md
+++ b/docs/spec.md
@@ -1377,6 +1377,7 @@ Esqueceu a senha? [e5 link "Recuperar acesso"]
| `browser.switchTab` | `botId`, `tabId` | `null`: essa aba vira a ativa. Só da conexão que controla; aba que não existe mais é `not_found` |
| `browser.open` | `botId`, `url` | `null`: leva a aba ativa a um endereço `http` ou `https`; sem esquema, vale `https://`. Só da conexão que controla; o que não é endereço da web é `validation` |
| `browser.window` | `botId` | `BrowserState`: abre o navegador numa janela própria, para o dono entrar numa conta (21.11). Da conexão que assiste esse bot; tira o controle dela, se o tinha. Já aberto numa janela, ou sem navegador no computador, é `conflict` |
+| `browser.teach` | `botId`, `on` | `LessonStep[]`: começa uma lição (21.13), com o primeiro passo, ou termina e devolve os passos. Só da conexão que controla |
- O que muda o navegador responde assim que o pedido é aceito; o resultado chega em `browser.changed` e nos quadros.
- `BrowserState`: `botId`, `status` (`closed`, `starting`, `open`, `failed`), `url`, `title` e `loading` (da aba ativa), `tabs` (as abas abertas, na ordem em que abriram: `BrowserTab {id, title, url, active}`, com `id` opaco), `error` (por que não abriu, em `failed`), `control` (`bot` ou `owner`, 21.10), `resting` (o navegador descansa, 21.2), `ask` (o que o bot pediu ao dono com `browser_ask_owner`, enquanto o pedido está aberto), `window` (aberto numa janela própria, 21.11: o estado é `closed` e `control` é `owner`) e `updatedAt`. `browser.list` traz também os `closed` com `window`.
@@ -1475,8 +1476,25 @@ O login do Google, e o de outros sites que olham mais que o user agent e `naviga
| **N3.2** Abas e recarregar | a lista de abas em `BrowserState`, `browser.reload`, `browser.newTab`, `browser.switchTab`, `browser.open`, o aviso ao bot quando o dono troca de aba, as abas e o botão de recarregar no painel, o endereço que o dono digita, textos nos três idiomas, testes com o Edge real | pelo app, o dono vê as abas que o bot abriu, recarrega a página, assume, abre uma aba nova num endereço, troca de aba e devolve; o bot lê que a aba mudou antes de agir |
| **N3.3** Descanso | o navegador descansa quando o turno do bot termina e acorda quando é usado, `resting` em `BrowserState`, fechar por falta de uso mesmo com o painel aberto, a etiqueta "Em descanso", textos nos três idiomas, testes com o Edge real | com um bot real parado numa página que se mexe, o Edge dele não gasta processador; o bot volta a usar a página sem perder nada |
| **N3.4** Janela de verdade | `browser.window`, `window` em `BrowserState`, o Edge com janela no perfil do bot, a espera das tools e do pedido de ajuda, o botão e a faixa no painel, textos nos três idiomas | com o Claude Code real, o dono entra numa conta do Google do bot pela janela, fecha, e o bot abre o Gmail já dentro da conta |
+| **N3.5** Ensinar uma tarefa | `browser.teach`, `lesson` em `BrowserState`, a leitura do que o dono faz em `actions.js`, o texto da lição, a faixa e o diálogo no painel, textos nos três idiomas, teste com o Edge real | 21.13 |
| **N4** Telas | o bot desenhando telas (HTML) que aparecem lado a lado numa área de design, atualizadas enquanto ele escreve | seção 22 |
+### 21.13 Ensinar uma tarefa (N3.5)
+
+O dono mostra ao bot, uma vez, como fazer uma tarefa no navegador dele; o que fez vira passos, e os passos viram uma rotina (20) ou uma message para o bot guardar na memória.
+
+- **Só nas mãos do dono.** Com o navegador no controle do dono (21.10), "Ensinar uma tarefa" embaixo da pílula chama `browser.teach {botId, on: true}`. A lição começa com um passo `open` com o endereço da aba ativa. `browser.teach {on: false}` a termina e devolve os passos. Devolver o navegador, fechar a conexão ou o navegador fechar descartam a lição.
+- **O que vira passo.** Cada coisa que o dono faz passa pela fila de 21.10; com uma lição aberta, o daemon lê a página antes de ela chegar lá, pelo script do mundo isolado (21.6, `actions.js`), que devolve só o papel, o nome e se é um campo de senha:
+ - um clique (botão esquerdo, não o segundo de um clique duplo) vira `click` com o nome do elemento com papel mais próximo do ponto (`at(x, y)`); sem nada com papel ali, não vira passo;
+ - teclas de texto, Backspace e texto colado ou composto viram `type` com o nome do campo que tem o foco (`focused()`), um passo por campo enquanto o dono escreve nele; o campo de senha marca `secret`;
+ - Enter, Escape e Tab sem Ctrl, Alt ou Meta viram `press`;
+ - um endereço que o dono digita (`browser.open`) vira `open`.
+- **O que nunca é guardado.** O que o dono digita, cola ou escolhe: o script não lê valor de campo. Um endereço guarda só o esquema, o site e o caminho, sem usuário e senha, query e fragmento, onde viajam tokens e dados pessoais. O rótulo de cada passo tem no máximo 80 caracteres, e uma lição, 100 passos. Os passos ficam só em memória, em `BrowserState.lesson`, e não vão para o log.
+- **No app.** Embaixo da pílula, "Ensinar uma tarefa" e a explicação de que o bot guarda cada passo, nunca o que é digitado. Gravando, uma faixa "Gravando a lição" mostra os passos ao vivo, numerados, com "Cancelar"; o botão da pílula vira "Terminar a lição". Terminada, um diálogo pede o nome da tarefa e mostra os passos, cada um com um X para tirar, e oferece:
+ - **Fazer virar rotina…**: abre o diálogo de rotina nova (20.9) com o nome e o pedido já escritos: a lição em palavras e "Faça isso agora.";
+ - **Mandar para guardar**: manda ao bot, como message do dono, a lição em palavras e o pedido de guardá-la na memória (5.1) para fazer quando o dono pedir.
+- **A lição em palavras** é montada pelo app, no idioma do dono, e pode ser editada na rotina: "Como fazer "" no navegador, do jeito que eu mostrei:", os passos numerados ("Abra ", "Clique em """, "Escreva em """, "Escreva a senha em "": peça para eu digitar", "Aperte Enter") e, se houve texto digitado, que o dono não mostra o quê: o bot usa o que a tarefa pede ou pergunta. Senhas, o bot pede com `browser_ask_owner` (21.10).
+
## 22. Telas
Status: **T1, T2 e T3 implementados** (22.8). É o N4 da seção 21.