From 3b216b6582dc1d72174b32b19ed9df6170385e7c Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Thu, 24 Sep 2026 16:23:45 +0100 Subject: [PATCH 01/30] Unify the Brunch conversation UI for chat and voice Add structured conversation turns, collapsible work, voice reply slots, and experiment result follow-ups. Refs FE-1793 Co-authored-by: Amp --- .changeset/brunch-conversation-turns.md | 5 + ...draft-experiment-interactive-tool.test.tsx | 174 ++++++++++- ...unch-draft-experiment-interactive-tool.tsx | 75 +++-- .../editor-drafts.ts | 2 + .../brunch-experiment-follow-up.tsx | 91 ++++++ .../local-storage-demo-app.test.tsx | 2 +- .../local-storage-demo-app.tsx | 5 + .../voice-history-continuity.test.tsx | 11 +- .../@hashintel/petrinaut/docs/ai-assistant.md | 52 ++-- .../horizontal/horizontal-tabs-container.tsx | 25 +- .../ui/types/ai-assistant-composer-control.ts | 2 + .../src/ui/types/ai-interactive-tool.ts | 4 + .../Editor/panels/ai-assistant-panel.test.tsx | 54 ++-- .../Editor/panels/ai-assistant-panel.tsx | 4 +- .../ai-assistant-contents.stories.tsx | 199 +++++++++++- .../ai-assistant-contents.test.tsx | 288 +++++++++++++++--- .../ai-assistant-contents.tsx | 242 ++++++++++----- .../brunch-work-fold.test.tsx | 47 +++ .../brunch-work-fold.tsx | 104 +++++++ .../experiment-execution-card.test.tsx | 7 + .../experiment-execution-card.tsx | 65 ++-- .../get-message-render-items.test.ts | 79 +++++ .../get-message-render-items.ts | 148 ++++----- .../ai-assistant-contents/prompt-chips.tsx | 6 +- .../ai-assistant-contents/reasoning.tsx | 17 +- .../shared/use-elapsed-time.test.ts | 33 ++ .../shared/use-elapsed-time.ts | 17 ++ .../ai-assistant-contents/tool-list.tsx | 244 +++++++-------- .../voice-input-provenance.tsx | 93 +++--- .../apply-auto-layout-widget.test.tsx | 10 +- .../apply-auto-layout-widget.tsx | 4 +- .../interactive-tools/registry.ts | 1 + .../interactive-tools/types.ts | 1 + 33 files changed, 1567 insertions(+), 544 deletions(-) create mode 100644 .changeset/brunch-conversation-turns.md create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/brunch-experiment-follow-up.tsx create mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.test.tsx create mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.tsx create mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/get-message-render-items.test.ts create mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/shared/use-elapsed-time.test.ts create mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/shared/use-elapsed-time.ts diff --git a/.changeset/brunch-conversation-turns.md b/.changeset/brunch-conversation-turns.md new file mode 100644 index 00000000000..998f9dbec90 --- /dev/null +++ b/.changeset/brunch-conversation-turns.md @@ -0,0 +1,5 @@ +--- +"@hashintel/petrinaut": patch +--- + +Unify Chat and Voice turns with collapsible work, tool status and timing, answer cards, optional voice briefs and replies, and clearer experiment progress. Preserve unsent composer text when hosts submit background results. diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-draft-experiment-interactive-tool.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-draft-experiment-interactive-tool.test.tsx index 1c39f254052..04f64e8c132 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-draft-experiment-interactive-tool.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-draft-experiment-interactive-tool.test.tsx @@ -31,6 +31,7 @@ import { describeBudget, describeExperiment, } from "./brunch-draft-experiment-interactive-tool/describe-draft"; +import { BrunchExperimentFollowUp } from "./brunch-experiment-follow-up"; // The `/ui` entry pulls in chart code that probes `matchMedia` at import time. vi.hoisted(() => { @@ -65,7 +66,10 @@ import type { SDCPN, } from "@hashintel/petrinaut-core"; import type { OptimizationsContextValue } from "@hashintel/petrinaut/react"; -import type { PetrinautAiInteractiveToolWidgetProps } from "@hashintel/petrinaut/ui"; +import type { + PetrinautAiInteractiveToolWidgetProps, + PetrinautAiComposerControlContext, +} from "@hashintel/petrinaut/ui"; import type { ReactNode } from "react"; // Distributive so the awaiting/submitted discriminant survives the Pick. @@ -753,9 +757,17 @@ describe("BrunchDraftExperimentWidget", () => { execution: { mode: "optimize", direction: "minimize" }, }); expect(seenSignal?.aborted).toBe(false); - await waitFor(() => expect(heading()).toEqual(["Running"])); + await waitFor(() => + expect(screen.getByRole("status").textContent).toBe("Optimizing"), + ); expect(screen.getByRole("status").textContent).toBe("Optimizing"); expect(screen.getByText("Step 1 of 3")).toBeTruthy(); + expect(screen.getAllByText("Staffing under peak demand")).toHaveLength(1); + expect(screen.queryByText("Metrics")).toBeNull(); + expect(screen.queryByText("Declared")).toBeNull(); + expect( + screen.queryByRole("region", { name: "Drafted experiment" }), + ).toBeNull(); expect(screen.getByRole("progressbar").getAttribute("aria-valuenow")).toBe( "5", ); @@ -771,7 +783,9 @@ describe("BrunchDraftExperimentWidget", () => { await act(async () => { resolveRun(finishedResult); }); - await waitFor(() => expect(heading()).toEqual(["Run complete"])); + await waitFor(() => + expect(screen.getByRole("status").textContent).toBe("Finished"), + ); expect(screen.getByRole("status").textContent).toBe("Finished"); expect(screen.getByText("15 runs")).toBeTruthy(); expect(runExperiment).toHaveBeenCalledTimes(1); @@ -794,14 +808,157 @@ describe("BrunchDraftExperimentWidget", () => { fireEvent.click(screen.getByRole("button", { name: "Run" })); - await waitFor(() => expect(heading()).toEqual(["Run failed"])); + await waitFor(() => + expect(screen.getByRole("status").textContent).toBe("Failed"), + ); expect(screen.getByRole("status").textContent).toBe("Failed"); expect(screen.getByText("Compilation failed")).toBeTruthy(); + expect(screen.queryByText("Metrics")).toBeNull(); + expect(screen.queryByText("Declared")).toBeNull(); fireEvent.click(screen.getByRole("button", { name: "Retry run" })); await waitFor(() => expect(runExperiment).toHaveBeenCalledTimes(2)); - await waitFor(() => expect(heading()).toEqual(["Run complete"])); + await waitFor(() => + expect(screen.getByRole("status").textContent).toBe("Finished"), + ); + }); + + it("sends completed results once, only to the originating idle conversation", async () => { + const definition = createReadableStore(makeDefinition()); + const { submit, wrap } = renderWidget({ + input: makeInput(), + toolCallId: "follow-up", + state: awaiting, + definition, + runExperiment: vi.fn().mockResolvedValue(finishedResult), + }); + await waitFor(() => expect(submit).toHaveBeenCalledOnce()); + fireEvent.click(screen.getByRole("button", { name: "Run" })); + await screen.findByText("Finished"); + const submitText = vi + .fn() + .mockRejectedValueOnce(new Error("Connection lost")) + .mockResolvedValue({ kind: "message", messageId: "completion" }); + const context = { + conversationId: "original", + messages: [ + { + id: "original-tool", + role: "assistant" as const, + parts: [ + { + type: "dynamic-tool" as const, + toolCallId: "follow-up", + toolName: "draft_petrinaut_experiment", + state: "output-available" as const, + input: {}, + output: {}, + }, + ], + }, + ], + status: "streaming" as const, + submitText, + stop: async () => {}, + }; + const followUp = render( + wrap(), + ); + expect(submitText).not.toHaveBeenCalled(); + followUp.rerender( + wrap( + , + ), + ); + expect(submitText).not.toHaveBeenCalled(); + followUp.rerender( + wrap( + , + ), + ); + expect(submitText).not.toHaveBeenCalled(); + followUp.rerender( + wrap( + , + ), + ); + await waitFor(() => expect(submitText).toHaveBeenCalledOnce()); + const submission = submitText.mock.calls[0]?.[0]; + expect(submission).toMatchObject({ + target: "message", + preserveDraft: true, + }); + expect(submission?.text).toContain(JSON.stringify(finishedResult)); + await screen.findByRole("button", { name: "Retry result summary" }); + followUp.rerender( + wrap( + , + ), + ); + expect(submitText).toHaveBeenCalledOnce(); + fireEvent.click( + screen.getByRole("button", { name: "Retry result summary" }), + ); + await waitFor(() => expect(submitText).toHaveBeenCalledTimes(2)); + followUp.unmount(); + render( + wrap( + , + ), + ); + expect(submitText).toHaveBeenCalledTimes(2); + expect(submit).toHaveBeenCalledOnce(); + }); + + it("shows model review when retrying a failed run against a changed model", async () => { + const definition = createReadableStore(makeDefinition()); + const runExperiment = vi + .fn() + .mockRejectedValueOnce(new Error("Unavailable")) + .mockResolvedValueOnce(finishedResult); + const { submit } = renderWidget({ + input: makeInput(), + toolCallId: "retry-review", + state: awaiting, + definition, + runExperiment, + }); + await waitFor(() => expect(submit).toHaveBeenCalledOnce()); + fireEvent.click(screen.getByRole("button", { name: "Run" })); + await screen.findByRole("button", { name: "Retry run" }); + definition.set({ + ...definition.get(), + places: [ + { + id: "new-place", + name: "New place", + colorId: null, + dynamicsEnabled: false, + differentialEquationId: null, + x: 0, + y: 0, + }, + ], + }); + fireEvent.click(screen.getByRole("button", { name: "Retry run" })); + expect( + await screen.findByRole("button", { name: "Accept current model" }), + ).toBeTruthy(); + expect(runExperiment).toHaveBeenCalledOnce(); + fireEvent.click( + screen.getByRole("button", { name: "Accept current model" }), + ); + fireEvent.click( + screen.getByRole("button", { name: "Retry against current model" }), + ); + await waitFor(() => expect(runExperiment).toHaveBeenCalledTimes(2)); }); it.each(["simulate", "optimize"] as const)( @@ -832,7 +989,9 @@ describe("BrunchDraftExperimentWidget", () => { fireEvent.click(screen.getByRole("button", { name: "Run" })); - await waitFor(() => expect(heading()).toEqual(["Run failed"])); + await waitFor(() => + expect(screen.getByRole("status").textContent).toBe("Failed"), + ); expect(screen.getByRole("status").textContent).toBe("Failed"); expect(screen.getByText("Compilation failed")).toBeTruthy(); }, @@ -1055,6 +1214,9 @@ describe("BrunchDraftExperimentWidget", () => { fireEvent.click(screen.getByRole("button", { name: "Dismiss" })); expect(heading()).toEqual(["Dismissed"]); + expect(screen.getByText("Staffing under peak demand")).toBeTruthy(); + expect(screen.getByText("Metrics")).toBeTruthy(); + expect(screen.getByText("Declared")).toBeTruthy(); expect(screen.queryByRole("button", { name: "Run" })).toBeNull(); expect(runExperiment).not.toHaveBeenCalled(); }); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-draft-experiment-interactive-tool.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-draft-experiment-interactive-tool.tsx index 8f83332c912..00a9b60995b 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-draft-experiment-interactive-tool.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-draft-experiment-interactive-tool.tsx @@ -54,13 +54,13 @@ const containerStyle = css({ padding: "3", borderWidth: "thin", borderStyle: "solid", - borderColor: "neutral.a30", + borderColor: "purple.a30", borderRadius: "lg", backgroundColor: "neutral.s00", }); const statusStyle = css({ - color: "neutral.s80", + color: "purple.s100", fontSize: "xs", fontWeight: "medium", letterSpacing: "wide", @@ -136,12 +136,12 @@ const primaryButtonStyle = css({ paddingX: "3", paddingY: "2", borderRadius: "md", - backgroundColor: "blue.a85", + backgroundColor: "purple.a85", color: "white", cursor: "pointer", fontSize: "sm", fontWeight: "medium", - _hover: { backgroundColor: "blue.a100" }, + _hover: { backgroundColor: "purple.a100" }, _disabled: { cursor: "not-allowed", opacity: 0.45 }, }); @@ -541,6 +541,45 @@ export const BrunchDraftExperimentWidget = ({ experimentId && experiments.some((experiment) => experiment.id === experimentId); + if ( + run && + run.phase !== "idle" && + draft.prepared && + !draft.dismissed && + !(run.phase === "failed" && (reviewed || runError)) + ) { + return ( + run.controller.abort() : undefined + } + onRetry={ + run.phase === "failed" && canRun ? () => void onRun() : undefined + } + onViewExperiment={ + canViewExperiment + ? () => { + navigate( + openPetrinautSimulationResource({ + type: "experiment", + id: experimentId, + }), + { cause: "user", action: "simulation-resource" }, + ); + } + : undefined + } + /> + ); + } + return (
) : null} - {run && run.phase !== "idle" && draft.prepared ? ( - run.controller.abort() : undefined - } - onViewExperiment={ - canViewExperiment - ? () => { - navigate( - openPetrinautSimulationResource({ - type: "experiment", - id: experimentId, - }), - { cause: "user", action: "simulation-resource" }, - ); - } - : undefined - } - /> - ) : null} {canAct ? (
+ ) : null; +}; diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx index 507299c5878..76f183eedd3 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx @@ -546,7 +546,7 @@ describe("local storage demo Brunch voice integration", () => { expect( (renderedPetrinaut.aiAssistant as PetrinautAiAssistant) .renderComposerControl, - ).toBeUndefined(); + ).toBeTypeOf("function"); rendered.unmount(); localPlaybackCancellation.mockRestore(); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx index 903e007e3b0..108c2334f45 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx @@ -39,6 +39,7 @@ import { import { DefaultChatTransport, Petrinaut, + type PetrinautAiComposerControlContext, type PetrinautAiMessage, type PetrinautAiStopResult, type PetrinautAiVoiceMode, @@ -79,6 +80,7 @@ import { createBrunchDraftExperimentInteractiveTool, resolveDraftAuthorityFromHistory, } from "./brunch-draft-experiment-interactive-tool"; +import { BrunchExperimentFollowUp } from "./brunch-experiment-follow-up"; import { BrunchPanelConversationTracker, type BrunchPanelAdmissionTarget, @@ -885,6 +887,9 @@ export const LocalStorageDemoApp = ({ primaryLabel: "Chat", resolveToolPresentation: resolveBrunchToolPresentation, workingLabel: "Brunch is working", + renderComposerControl: ( + context: PetrinautAiComposerControlContext, + ) => , } : {}), ...(conversationId === null ? {} : { conversationId }), diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-history-continuity.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-history-continuity.test.tsx index 89eb149ff41..a2cc2a6ed6d 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-history-continuity.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-history-continuity.test.tsx @@ -364,10 +364,15 @@ test("projects typed, in-band tool, and stopped fixture history after remount", ).queryByTestId("voice-input-provenance"), ).toBeNull(); expect( - secondMount.container.querySelector('[data-tool-call-id="voice-tool-1"]'), - ).not.toBeNull(); + within( + secondMount.container.querySelector( + '[data-tool-call-id="voice-tool-1"]', + )!, + ).queryByTestId("voice-input-provenance"), + ).toBeNull(); + expect(screen.getByText("voice-tool-1: The supervisor")).not.toBeNull(); expect(screen.getByText("Durably interrupted response")).not.toBeNull(); - expect(screen.getByText("Response stopped")).not.toBeNull(); + expect(screen.getByText("Brunch stopped")).not.toBeNull(); await act(async () => fireEvent.click(screen.getByRole("button", { name: "Start voice mode" })), diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index d00cc8cbf61..4f3fc2b3924 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -33,12 +33,18 @@ The header text is not selectable. Its icons animate on hover and click, respect ## The conversation -Type in the message field and press **Enter** or choose the **Send message** button with the upward arrow at its right edge. **Shift+Enter** adds a line. The field grows with your message while the send button stays at the bottom right. +Type in **Continue iterating...** and press **Enter** or choose the **Send message** button with the upward arrow at its right edge. **Shift+Enter** adds a line. The field grows with your message while the send button stays at the bottom right. Before the first message, suggestion chips wrap so all remain visible. + +The primary tab reads **Chat**, or **Voice** while Voice mode is selected. Your messages appear in right-aligned grey bubbles. Each assistant turn groups its work in a neutral disclosure, followed by a light-blue answer and any produced cards. In Voice, the written Brunch answer is inside the work disclosure instead. + +The work disclosure opens while **Brunch is working**, when **Brunch needs your approval**, or when **Brunch stopped**. Completed work collapses under **Brunch worked for Ns**; history without timing says **Brunch worked**. Expand it to inspect **Thought for Ns** and **Used N tools**, both initially collapsed. Stopped work says **Stopped after N tools**. Timing is shown when supplied or observed during this session; unavailable tool durations show a dash. Approval controls remain visible inside the work disclosure; **Allow** or **Deny** lets the assistant continue. + +When the host supplies them, Voice also shows a collapsed **Sent to Brunch** brief under your message, an immediate spoken-agent reply before the work, and a wrap-up after the produced cards. The brief says **Prepared from what you said** and lists its fields. These optional parts are absent in hosts that do not provide them. Messages have no per-message voice badge. While a response is streaming you can: -- Watch the model's text and reasoning appear live. A reasoning block uses **Thinking: _provider heading_** when the provider supplies a short heading, falling back to **Thinking** otherwise. It is collapsible; while streaming, it auto-opens, shows a shimmer effect, and (once attached timing information arrives) an elapsed timer. -- Follow tool operations as they run. Each call remains in chronological order as its own row. A spinner and **Preparing…** or **Running…** distinguish an unfinished operation from its completed or failed result. Preparing is available only when the host streams tool arguments. A host may also show one working label for the whole active turn before its first tool is admitted and through automatic continuations. Interactive questions remain waiting for your answer rather than showing a running spinner. +- Expand the work disclosure to watch reasoning and tool operations. The working label shimmers while active. Reasoning stays collapsed until you choose to inspect it. +- Expand **Used N tools** to inspect chronological tool rows, each with its name, duration and status dot: amber pending, green completed, or red error. Expand a row to read its arguments and result. **Preparing…** means streamed arguments are arriving; **Running…** means execution is pending. Interactive questions remain available for your answer. - Press **Stop AI response** (the send button turns into a stop icon) to halt the current response. A host with durable conversation execution can record that stop before Petrinaut cancels its local stream; without that host capability, Stop is local cancellation only. Stop also withholds browser tools that have not started. Brunch's integrated assistant returns browser results within the active reply rather than requiring a second message; other hosts may still use an automatic follow-up. Already-applied changes are not rolled back. If a browser result is lost after a change may have happened, its outcome is unknown and the assistant does not automatically retry it. A silently disconnected browser is detected after a bounded liveness wait, not immediately. - Type your next message in the composer -- it is queued for after the current response ends. @@ -49,7 +55,7 @@ If the host offers voice input, only a finalized transcript captured while Voice If an assistant request fails, Petrinaut shows the complete error in a persistent toast rather than adding it to the conversation. Long errors wrap, diagnostic details can be copied, and the toast stays open until you close it. Retry from the composer when the assistant is ready. -Hosts may provide canonical conversation rehydration. In that case, reopening the same assistant shows its settled and stopped turns without resubmitting a message or replaying Voice audio. Voice markers attached to client-tool results survive that history. A direct spoken user message remains in the transcript after reopening, but its **Voice** chip may not be restored by the current Brunch host. Durably aborted assistant entries retain their **Response stopped** label even after later completed replies. If a tool-call step had already completed when Stop withheld its browser follow-up, that local decision has no durable cancellation record: hosts using initial-history recovery can recover the tool as pending work. Do not treat that local withholding as a reload-safe cancellation. +Hosts may provide canonical conversation rehydration. In that case, reopening the same assistant shows its settled and stopped turns without resubmitting a message or replaying Voice audio. Spoken user messages remain in the transcript without voice badges. Durably aborted assistant entries retain their **Brunch stopped** label even after later completed replies. If a tool-call step had already completed when Stop withheld its browser follow-up, that local decision has no durable cancellation record: hosts using initial-history recovery can recover the tool as pending work. Do not treat that local withholding as a reload-safe cancellation. A host may also enable live history following, as the local Brunch panel does. Turns submitted elsewhere then appear in the open conversation without a reload. Your own in-progress response stays in place until the host confirms that its canonical history has caught up. In this mode, tools observed from another participant or restored after reopening are display-only: watching a pending tool does not execute it or resume that turn. Tools emitted in response to your own local submission still execute normally. A pending externally submitted tool needs its originating participant/operator to resolve it; reopening this following panel is not automatic recovery. @@ -219,8 +225,8 @@ Semantic voice detection finishes an answer automatically after a natural pause, so there is no required done-speaking action. Duplicate, empty, failed, or unavailable transcripts are not submitted. Provisional words remain display-only until the provider finalizes their transcript. Spoken turns then -appear in the conversation, and finalized spoken user messages carry a -**Voice** chip. Only finalized answers and canonical Brunch text become chat +appear in the same conversation without a per-message badge. +Only finalized answers and canonical Brunch text become chat history; provisional transcription and provider audio are ephemeral. Completed interruptions that strongly repeat the assistant's active speech may be silently discarded instead of sent as an answer. Short answers such as @@ -245,14 +251,14 @@ The delete button appears in the top right of the panel once the conversation co ## What the assistant can do -The assistant has tools for inspecting and modifying the current net. You'll see one card per tool call inline in the conversation. A failed tool card leads with its complete error instead of hiding it behind a hover tooltip: +The assistant has tools for inspecting and modifying the current net. Expand the work disclosure and its tool group to see one row per call. A failed tool row exposes its error in the expanded result rather than only in a hover tooltip: -- **Pending tools** (gold/yellow, with a spinner) -- a host can mark an in-progress tool this way so it is not mistaken for a completed success. The stock assistant keeps its existing default for unfinished rows. +- **Pending tools** (amber dot) -- distinguish unfinished work from completed results. - **Read tools** (neutral, expandable) –– for checking the current net state and active Petrinaut extensions at any point, for compilation errors, and for reading the user guide. - **Applied mutation tools** (green for additions/updates, red for deletions) -- "Added place X", "Updated transition Y", "Removed metric Z", and so on. Successive tools remain visible as individual chronological rows. -- **Not applied** (neutral, with a dash) -- a completed tool that explicitly reports no change shows its actual reason rather than a successful summary of the requested edit. This includes blocked, declined, unchanged, and host-refused mutations. A host can instead show a compact correctable refusal whose detailed reason stays collapsed. Execution errors remain red, with a close mark, and show the error. +- **Not applied** (neutral summary) -- a completed tool that explicitly reports no change shows its actual reason rather than a successful summary of the requested edit. This includes blocked, declined, unchanged, and host-refused mutations. A completed status dot indicates execution finished, not that a mutation was applied. A host can instead show a compact correctable refusal whose detailed reason stays collapsed. Execution errors show a red dot and the error. - **`setNetTitle`** -- renames the net when the host supplies title editing. -- **`applyAutoLayout`** -- rearranges places and transitions on the canvas. If the assistant calls this on a net you've already arranged, it asks you first via an inline widget with **Yes, auto-layout** / **No, keep current layout** buttons. Otherwise it'll run it without asking. +- **`applyAutoLayout`** -- rearranges places and transitions on the canvas. If the assistant calls this on a net you've already arranged, it asks you first via an inline widget with **Allow** / **Deny** buttons. Otherwise it'll run it without asking. - **Host-specific questions and actions** -- an application embedding Petrinaut may add interactive widgets. For example, an elicitation assistant can ask a structured question inline and continue after you submit the answer. The @@ -281,9 +287,8 @@ For example: "Run 100 simulations of this scenario and show the completed orders metric." The assistant can also search numeric scenario parameter ranges to minimize or maximize a metric. -The experiment appears in a compact card with its status, run count, and -results. Simulation cards use blue; optimization cards use purple and glow -while running. Select **View +The experiment appears in a white card with purple accents, its status, run count, and +results. Optimization cards glow while running. Select **View experiment** to inspect metric distributions in the Experiments panel. The heatmap shows how values spread across runs; click a time step to see its histogram. Select **Cancel** @@ -323,13 +328,20 @@ active indicator. Choosing **Run** starts execution and uses the normal **1 active** indicator; when it completes, that indicator disappears and the result remains under **Simulate → Experiments**. -After **Run**, the chat card shows the same blue simulation or purple -optimization progress, **Cancel** action, final metrics and errors as the -built-in assistant. Optimization reports search steps and refinement. Drafting -and starting a run keep your current tab selected; choose **View experiment** -to open its results while the experiment is still available. The proposal's -disclosures remain visible. Run results are shown to you, not sent back to -Brunch for interpretation, and are not retained after reloading the session. +**Dismiss** retains the proposal as a **Dismissed** record. After **Run**, the +draft is replaced in place by execution progress: **Validating**, simulation +or optimization steps and runs, then **Finished** with metric cells. **Cancel** +leaves a **Cancelled** record. A failed run offers **Retry run**; model changes +still require review before retrying. Drafting and starting a run keep your +current tab selected; choose **View experiment** to open its results while the +experiment is still available. + +Completed local results are sent to Brunch as a new message in the originating +conversation when it is ready, without clearing your unsent draft. Brunch can +then interpret them in a new answer. A stopped response is not automatically +resumed. If submitting the result fails, choose **Retry result summary**. +The local run is not retained after reloading the session; the submitted result +message follows the host's conversation-history policy. ## Read-only behaviour diff --git a/libs/@hashintel/petrinaut/src/ui/components/sub-view/horizontal/horizontal-tabs-container.tsx b/libs/@hashintel/petrinaut/src/ui/components/sub-view/horizontal/horizontal-tabs-container.tsx index 8138a662c4c..6e10bba0aff 100644 --- a/libs/@hashintel/petrinaut/src/ui/components/sub-view/horizontal/horizontal-tabs-container.tsx +++ b/libs/@hashintel/petrinaut/src/ui/components/sub-view/horizontal/horizontal-tabs-container.tsx @@ -99,6 +99,7 @@ const liveRegionStyle = css({ export type HorizontalTabView = Pick & { attention?: { count?: number; marker?: boolean }; + mark?: React.ReactNode; }; const contentStyle = cva({ @@ -145,13 +146,33 @@ const TabButton: React.FC = ({ role="tab" > + {subView.mark && ( + + )} {subView.title} {subView.attention?.count ? ( -
)} - {messages.map((message) => ( + {messages.map((message, index) => ( ))} - {stopped && !error && !messages.at(-1)?.metadata?.stopped && ( -
Response stopped
+ {isBusy && messages.at(-1)?.role !== "assistant" && ( + + Waiting for Brunch… + )} @@ -1150,7 +1228,7 @@ export const AiAssistantContents = ({ {isBusy && workingLabel && (
diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.test.tsx new file mode 100644 index 00000000000..709036bec8b --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.test.tsx @@ -0,0 +1,47 @@ +/** @vitest-environment jsdom */ +import { + cleanup, + fireEvent, + render, + screen, + waitFor, +} from "@testing-library/react"; +import { afterEach, expect, test } from "vitest"; + +import { BrunchWorkFold } from "./brunch-work-fold"; + +afterEach(cleanup); + +test.each([ + ["streaming", "Brunch is working", "true"], + ["settled", "Brunch worked for 7s", "false"], + ["approval", "Brunch needs your approval", "true"], + ["stopped", "Brunch stopped", "true"], +] as const)( + "%s fold label and default disclosure", + (status, label, expanded) => { + render( + + Work details + , + ); + expect( + screen.getByRole("button", { name: label }).getAttribute("aria-expanded"), + ).toBe(expanded); + }, +); + +test("settles after approval and permits manual reopening", async () => { + const { rerender } = render( + Allow or deny, + ); + rerender(Approved); + const trigger = screen.getByRole("button", { name: "Brunch worked" }); + await waitFor(() => + expect(trigger.getAttribute("aria-expanded")).toBe("false"), + ); + fireEvent.click(trigger); + await waitFor(() => + expect(trigger.getAttribute("aria-expanded")).toBe("true"), + ); +}); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.tsx new file mode 100644 index 00000000000..f4b1a57848d --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.tsx @@ -0,0 +1,104 @@ +import { Collapsible } from "@ark-ui/react/collapsible"; +import { type ReactNode, useState } from "react"; + +import { Icon } from "@hashintel/ds-components"; +import { css } from "@hashintel/ds-helpers/css"; + +import { collapsibleContentStyle } from "./shared/collapsible-content-style"; +import { useElapsedTime } from "./shared/use-elapsed-time"; + +export type BrunchWorkStatus = "streaming" | "settled" | "approval" | "stopped"; + +const foldStyle = css({ + backgroundColor: "neutral.a20", + borderRadius: "lg", + padding: "1", + minWidth: "[0]", +}); +const triggerStyle = css({ + display: "flex", + alignItems: "center", + gap: "2", + width: "full", + padding: "2", + fontSize: "sm", + fontWeight: "medium", + color: "neutral.s100", + textAlign: "left", + cursor: "pointer", + "& [data-chevron]": { + marginLeft: "auto", + transition: "[transform 150ms ease]", + }, + "&[data-state=closed] [data-chevron]": { transform: "[rotate(180deg)]" }, + "&[data-working=true] [data-label]": { + backgroundImage: + "[linear-gradient(110deg, {colors.neutral.s100} 35%, {colors.neutral.s60} 50%, {colors.neutral.s100} 65%)]", + backgroundSize: "[200% 100%]", + backgroundClip: "text", + color: "[transparent]", + animation: "[shimmer 2.4s linear infinite]", + "@media (prefers-reduced-motion: reduce)": { animation: "none" }, + }, +}); + +export const BrunchWorkFold = ({ + status, + elapsedMs, + children, +}: { + status: BrunchWorkStatus; + elapsedMs?: number; + children: ReactNode; +}) => { + const observedElapsed = useElapsedTime(status === "streaming"); + const duration = observedElapsed ?? elapsedMs; + const defaultOpen = status !== "settled"; + const [disclosure, setDisclosure] = useState({ status, open: defaultOpen }); + // Reset only on a lifecycle transition, not on each streamed delta. + if (disclosure.status !== status) + setDisclosure({ status, open: defaultOpen }); + const label = + status === "streaming" + ? "Brunch is working" + : status === "approval" + ? "Brunch needs your approval" + : status === "stopped" + ? "Brunch stopped" + : duration === undefined + ? "Brunch worked" + : `Brunch worked for ${Math.floor(duration / 1_000)}s`; + return ( + setDisclosure({ status, open })} + data-work-status={status} + > + + + {label} + + + +
+ {children} +
+ + + ); +}; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-execution-card.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-execution-card.test.tsx index b4d0d09e18c..a8059cb6a88 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-execution-card.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-execution-card.test.tsx @@ -89,6 +89,9 @@ it.each(["simulate", "optimize"] as const)( expect(screen.getByRole("progressbar").getAttribute("aria-valuemax")).toBe( "6", ); + expect(screen.getByRole("progressbar").className).toContain("h_[3px]"); + expect(card.className).toContain("bg-c_neutral.s00"); + expect(card.className).toContain("bd-c_purple.a30"); if (mode === "optimize") { expect(screen.getByText("Step 3 of 4")).toBeTruthy(); view.rerender( @@ -159,16 +162,20 @@ it.each(["simulate", "optimize"] as const)( ); it("distinguishes an execution error from an unavailable result", () => { + const onRetry = vi.fn(); const view = render( , ); expect(screen.getByRole("status").textContent).toBe("Failed"); expect(screen.getByText("Compilation failed")).toBeTruthy(); expect(screen.queryByText(/No result is available/)).toBeNull(); + fireEvent.click(screen.getByRole("button", { name: "Retry run" })); + expect(onRetry).toHaveBeenCalledTimes(1); view.rerender(); expect(screen.getByRole("status").textContent).toBe("Not running"); expect(screen.getByText(/No result is available/)).toBeTruthy(); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-execution-card.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-execution-card.tsx index 2054094bfe3..4d424a0136f 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-execution-card.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-execution-card.tsx @@ -15,6 +15,8 @@ export type ExperimentExecutionCardProps = { result?: PetrinautExperimentResult; error?: string; onCancel?: () => void; + /** Supply for a failed execution that can be attempted again. */ + onRetry?: () => void; /** Supply only while the caller can open the experiment's record. */ onViewExperiment?: () => void; }; @@ -27,23 +29,14 @@ const cardStyle = css({ padding: "2.5", borderRadius: "xl", border: "[1px solid]", - borderColor: "blue.a30", - backgroundColor: "blue.s10", - backgroundImage: - "[linear-gradient(135deg, var(--colors-blue-a10), transparent 70%)]", + borderColor: "purple.a30", + backgroundColor: "neutral.s00", fontSize: "sm", color: "neutral.s90", transition: "[border-color 200ms ease, box-shadow 200ms ease]", "&[data-pending=true]": { boxShadow: - "[0 0 0 1px var(--colors-blue-a15), 0 0 14px var(--colors-blue-a20)]", - }, - "&[data-tone=optimization]": { - borderColor: "purple.a30", - backgroundColor: "purple.s10", - backgroundImage: - "[linear-gradient(135deg, var(--colors-purple-a10), transparent 70%)]", - boxShadow: "[0 2px 12px var(--colors-purple-a10)]", + "[0 0 0 1px var(--colors-purple-a15), 0 0 14px var(--colors-purple-a20)]", }, "&[data-tone=optimization][data-pending=true]::after": { content: '""', @@ -81,19 +74,14 @@ const iconStyle = css({ marginTop: "0.5", flexShrink: "0", borderRadius: "lg", - color: "blue.s100", - backgroundColor: "blue.a15", - "[data-tone=optimization] &": { - color: "purple.s100", - backgroundColor: "purple.a15", - }, + color: "purple.s100", + backgroundColor: "purple.a15", }); const kindStyle = css({ fontSize: "xs", fontWeight: "medium", - color: "blue.s100", + color: "purple.s100", lineHeight: "[16px]", - "[data-tone=optimization] &": { color: "purple.s100" }, }); const titleStyle = css({ display: "block", @@ -126,8 +114,8 @@ const statusStyle = css({ fontSize: "xs", fontWeight: "medium", lineHeight: "[14px]", - color: "blue.s110", - backgroundColor: "blue.a10", + color: "purple.s110", + backgroundColor: "purple.a10", "&[data-status=pending]": { "[data-tone=optimization] &": { color: "purple.s110", @@ -162,18 +150,16 @@ const detailStyle = css({ overflowWrap: "anywhere", }); const progressTrackStyle = css({ - height: "[5px]", + height: "[3px]", borderRadius: "full", overflow: "hidden", - backgroundColor: "blue.a15", - "[data-tone=optimization] &": { backgroundColor: "purple.a15" }, + backgroundColor: "purple.a15", }); const progressFillStyle = css({ height: "[100%]", borderRadius: "full", - backgroundColor: "blue.s90", + backgroundColor: "purple.s90", transition: "[width 280ms ease-out]", - "[data-tone=optimization] &": { backgroundColor: "purple.s90" }, "@media (prefers-reduced-motion: reduce)": { transition: "[none]" }, }); const metricsStyle = css({ @@ -217,6 +203,7 @@ export const ExperimentExecutionCard = ({ result, error: errorText, onCancel, + onRetry, onViewExperiment, }: ExperimentExecutionCardProps) => { const name = result?.name ?? progress?.name ?? request?.name ?? "Experiment"; @@ -360,6 +347,7 @@ export const ExperimentExecutionCard = ({ )} {result?.metrics.length || available || + (errorText && onRetry) || (pending && onCancel) || result?.experimentId ? (
)} - {(available || (pending && onCancel) || result?.experimentId) && ( + {(available || + (errorText && onRetry) || + (pending && onCancel) || + result?.experimentId) && (
+ {errorText && onRetry && ( + + )} {available && ( ); - if (!expandable) { - return button; - } - return ( {button}
+ {tool.toolName} + {href && !errored && ( + + Open user guide + + )} + Arguments + 
+            {tool.input === undefined
+              ? "Not available"
+              : JSON.stringify(tool.input, null, 2)}
+
+ Result + 
+            {tool.errorText ??
+              (tool.output === undefined
+                ? "Pending"
+                : JSON.stringify(tool.output, null, 2))}
+
{children.map((item, index) => ( // oxlint-disable-next-line react/no-array-index-key
@@ -769,16 +696,22 @@ export const AiAssistantToolList = ({ onInteractiveToolSubmit, onSelectToolTarget, tools, + active = false, + stopped = false, + producedCard = false, }: { onInteractiveToolSubmit?: OnInteractiveToolSubmit; onSelectToolTarget?: (target: AiToolTarget) => void; tools: ToolRenderItem[]; + active?: boolean; + stopped?: boolean; + producedCard?: boolean; }) => { if (tools.length === 0) { return null; } - return ( + const content = (
); + + if (producedCard) return content; + return ( + <> + + + + {stopped ? "Stopped after" : "Used"} {tools.length}{" "} + {tools.length === 1 ? "tool" : "tools"} + + + +
+ {tools + .filter( + (tool) => !tool.interactive || tool.state !== "input-available", + ) + .map((tool) => ( + + ))} +
+ + + {tools + .filter((tool) => tool.interactive && tool.state === "input-available") + .map((tool) => ( + + ))} + + ); }; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-input-provenance.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-input-provenance.tsx index 08b71415555..5cc6375637f 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-input-provenance.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-input-provenance.tsx @@ -1,53 +1,46 @@ import { css } from "@hashintel/ds-helpers/css"; -import { - VoiceIcon, - useExperimentalIconPackEnabled, -} from "../../../../../experimental-icons"; +import type { VoiceBrief } from "./get-message-render-items"; -// Sits in the text flow ahead of the words it belongs to, so a spoken turn is -// marked where reading starts rather than by something trailing the bubble. -const chipStyle = css({ - display: "inline-flex", - alignItems: "center", - gap: "[3px]", - marginRight: "[6px]", - padding: "[1px 6px 1px 4px]", - borderRadius: "full", - backgroundColor: "blue.a30", - color: "blue.s110", - fontSize: "[10px]", - fontWeight: "semibold", - letterSpacing: "[0.03em]", - textTransform: "uppercase", - verticalAlign: "[1px]", - whiteSpace: "nowrap", -}); - -export const VoiceInputProvenance = () => { - const experimentalIcons = useExperimentalIconPackEnabled(); - return ( - - {experimentalIcons ? ( - - ) : ( - - )} - Voice - - ); -}; +/** The prepared brief is optional; ordinary spoken messages carry no marker. */ +export const VoiceInputProvenance = ({ brief }: { brief: VoiceBrief }) => ( +
+ + Sent to Brunch + +
+

Prepared from what you said

+
+ {Object.entries(brief.fields).map(([field, value]) => ( +
+
{field.replace(/([A-Z])/gu, " $1")}
+
{value}
+
+ ))} +
+
+
+); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/apply-auto-layout-widget.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/apply-auto-layout-widget.test.tsx index 496c8e7654a..0a8cd80d789 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/apply-auto-layout-widget.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/apply-auto-layout-widget.test.tsx @@ -28,7 +28,7 @@ describe("ApplyAutoLayoutWidget", () => { />, ); - fireEvent.click(screen.getByRole("button", { name: /Yes, auto-layout/i })); + fireEvent.click(screen.getByRole("button", { name: "Allow" })); expect(submit).toHaveBeenCalledTimes(1); expect(submit.mock.calls[0]![0]).toEqual({ @@ -49,9 +49,7 @@ describe("ApplyAutoLayoutWidget", () => { />, ); - fireEvent.click( - screen.getByRole("button", { name: /No, keep current layout/i }), - ); + fireEvent.click(screen.getByRole("button", { name: "Deny" })); expect(submit).toHaveBeenCalledTimes(1); expect(submit.mock.calls[0]![0]).toEqual({ @@ -72,9 +70,7 @@ describe("ApplyAutoLayoutWidget", () => { ); expect(screen.getByText("Auto-laid out 3 nodes")).toBeDefined(); - expect( - screen.queryByRole("button", { name: /Yes, auto-layout/i }), - ).toBeNull(); + expect(screen.queryByRole("button", { name: "Allow" })).toBeNull(); }); }); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/apply-auto-layout-widget.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/apply-auto-layout-widget.tsx index f8b1056ed49..e107ccb8d7d 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/apply-auto-layout-widget.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/apply-auto-layout-widget.tsx @@ -71,7 +71,7 @@ const ApplyAutoLayoutWidget = ({ }) } > - Yes, auto-layout + Allow
diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/registry.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/registry.ts index 6f75651ceca..379f34da8e5 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/registry.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/registry.ts @@ -59,6 +59,7 @@ export const getInteractiveTool = ( : hostDefinition ? { toolName: hostDefinition.toolName, + placement: hostDefinition.placement, shouldHandle: () => true, parseInput: hostDefinition.parseInput, parseOutput: hostDefinition.parseOutput, diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/types.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/types.ts index 5de09166f9a..598ef0cb0f3 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/types.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/types.ts @@ -18,6 +18,7 @@ export type InteractiveToolWidgetProps = */ export type InteractiveToolDefinition = { toolName: string; + placement?: "work" | "card"; /** * Whether this tool call should be handled interactively. Lets a single * tool branch between interactive and non-interactive paths based on its From d7ab86c1d916edeb5f0b46b59d09454e449564b4 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Thu, 24 Sep 2026 16:55:48 +0100 Subject: [PATCH 02/30] Align Brunch conversation details with the prototype Keep stock Petrinaut tabs unchanged, refine work and voice disclosures, and add completed-answer actions with draft-preserving retry. Refs FE-1793 Co-authored-by: Amp --- .../@hashintel/petrinaut/docs/ai-assistant.md | 4 +- .../horizontal/horizontal-tabs-container.tsx | 25 +- .../Editor/panels/ai-assistant-panel.test.tsx | 45 ++++ .../Editor/panels/ai-assistant-panel.tsx | 11 +- .../ai-assistant-contents.stories.tsx | 39 +++- .../ai-assistant-contents.test.tsx | 214 +++++++++++++++++- .../ai-assistant-contents.tsx | 159 +++++++++++-- .../ai-assistant-contents/brunch-tabs.tsx | 138 +++++++++++ .../brunch-work-fold.tsx | 25 +- .../experiment-execution-card.tsx | 20 +- .../ai-assistant-contents/reasoning.tsx | 27 +-- .../ai-assistant-contents/tool-list.tsx | 98 +++++--- .../voice-input-provenance.tsx | 41 +++- 13 files changed, 730 insertions(+), 116 deletions(-) create mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-tabs.tsx diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index 4f3fc2b3924..27c905c1b4f 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -37,7 +37,9 @@ Type in **Continue iterating...** and press **Enter** or choose the **Send messa The primary tab reads **Chat**, or **Voice** while Voice mode is selected. Your messages appear in right-aligned grey bubbles. Each assistant turn groups its work in a neutral disclosure, followed by a light-blue answer and any produced cards. In Voice, the written Brunch answer is inside the work disclosure instead. -The work disclosure opens while **Brunch is working**, when **Brunch needs your approval**, or when **Brunch stopped**. Completed work collapses under **Brunch worked for Ns**; history without timing says **Brunch worked**. Expand it to inspect **Thought for Ns** and **Used N tools**, both initially collapsed. Stopped work says **Stopped after N tools**. Timing is shown when supplied or observed during this session; unavailable tool durations show a dash. Approval controls remain visible inside the work disclosure; **Allow** or **Deny** lets the assistant continue. +Completed Chat answers offer **Copy** and **Retry**. Copy keeps the answer's Markdown. Retry sends that answer's original prompt as a new turn, keeping the previous answer and any unsent draft. It can lead to new tool calls, just like sending the prompt yourself. Retry is unavailable while another response or voice handoff is active. + +The work disclosure opens while **Brunch is working**, when **Brunch needs your approval**, or when **Brunch stopped**. Completed work collapses under **Brunch worked for Ns**, even while the answer is still streaming; history without timing says **Brunch worked**. Expand it to inspect **Thought for Ns** and **Used N tools**. Reasoning starts collapsed; **Running tools** opens automatically during execution and collapses when finished. Stopped work says **Stopped after N tools** and retains a **Response stopped** note. Unfinished tools show **Cancelled** while completed rows keep their results. Timing is shown when supplied or observed during this session; unavailable tool durations show a dash. Approval controls remain visible inside the work disclosure; **Allow** or **Deny** lets the assistant continue. When the host supplies them, Voice also shows a collapsed **Sent to Brunch** brief under your message, an immediate spoken-agent reply before the work, and a wrap-up after the produced cards. The brief says **Prepared from what you said** and lists its fields. These optional parts are absent in hosts that do not provide them. Messages have no per-message voice badge. diff --git a/libs/@hashintel/petrinaut/src/ui/components/sub-view/horizontal/horizontal-tabs-container.tsx b/libs/@hashintel/petrinaut/src/ui/components/sub-view/horizontal/horizontal-tabs-container.tsx index 6e10bba0aff..8138a662c4c 100644 --- a/libs/@hashintel/petrinaut/src/ui/components/sub-view/horizontal/horizontal-tabs-container.tsx +++ b/libs/@hashintel/petrinaut/src/ui/components/sub-view/horizontal/horizontal-tabs-container.tsx @@ -99,7 +99,6 @@ const liveRegionStyle = css({ export type HorizontalTabView = Pick & { attention?: { count?: number; marker?: boolean }; - mark?: React.ReactNode; }; const contentStyle = cva({ @@ -146,33 +145,13 @@ const TabButton: React.FC = ({ role="tab" > - {subView.mark && ( - - )} {subView.title} {subView.attention?.count ? ( -
); }, @@ -710,6 +814,7 @@ export const AiAssistantContents = ({ onInteractiveToolSubmit, onSelectToolTarget, onSendPrompt, + onRetryPrompt, onStop, onSubmit, onVoiceDockCollapsedChange, @@ -934,12 +1039,31 @@ export const AiAssistantContents = ({ // so memoised children never re-render due to handler changes — but we // refresh `.current` in an effect so any new closure capture is picked up // by the next event. + const firstUserIndex = messages.findIndex( + (message) => message.role === "user", + ); + const onRetryMessage = (messageId: string) => { + if (isBusy || voiceHandoffPending) return; + const index = messages.findIndex((message) => message.id === messageId); + const userMessage = messages + .slice(0, index) + .findLast((message) => message.role === "user"); + const prompt = userMessage?.parts + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + .join("\n\n"); + if (prompt) onRetryPrompt?.(prompt); + }; const handlersRef = useRef({ onInteractiveToolSubmit, onSelectToolTarget, + onRetryMessage, }); useEffect(() => { - handlersRef.current = { onInteractiveToolSubmit, onSelectToolTarget }; + handlersRef.current = { + onInteractiveToolSubmit, + onSelectToolTarget, + onRetryMessage, + }; }); useEffect(() => { @@ -1075,7 +1199,7 @@ export const AiAssistantContents = ({
{additionalTab && ( - = 0 && + index > firstUserIndex + } active={ isBusy && index === messages.length - 1 && diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-tabs.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-tabs.tsx new file mode 100644 index 00000000000..015e316e503 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-tabs.tsx @@ -0,0 +1,138 @@ +import { css } from "@hashintel/ds-helpers/css"; + +import type { ReactNode } from "react"; + +type BrunchTab = { + id: string; + title: string; + mark: ReactNode; + attention?: { count?: number; marker?: boolean }; +}; + +const tabStyle = css({ + display: "inline-flex", + alignItems: "center", + gap: "1.5", + minWidth: "[0]", + padding: "[4px 8px]", + border: "none", + borderRadius: "md", + backgroundColor: "[transparent]", + color: "neutral.s90", + fontSize: "xs", + fontWeight: "medium", + cursor: "pointer", + whiteSpace: "nowrap", + "& [data-mark]": { display: "inline-flex", color: "neutral.s80" }, + "&[aria-selected=true]": { + backgroundColor: "neutral.bg.subtle", + color: "neutral.s120", + "& [data-mark]": { color: "neutral.s90" }, + }, + _hover: { backgroundColor: "neutral.bg.subtle" }, +}); + +/** Conversation-only tabs; stock Petrinaut subview tabs retain their defaults. */ +export const BrunchTabs = ({ + subViews, + activeTabId, + onTabChange, + announcement, +}: { + subViews: BrunchTab[]; + activeTabId: string; + onTabChange: (tabId: string) => void; + announcement?: string; +}) => ( + <> +
{ + const index = subViews.findIndex((tab) => tab.id === activeTabId); + const nextIndex = + event.key === "ArrowRight" + ? (index + 1) % subViews.length + : event.key === "ArrowLeft" + ? (index - 1 + subViews.length) % subViews.length + : event.key === "Home" + ? 0 + : event.key === "End" + ? subViews.length - 1 + : undefined; + if (nextIndex === undefined) return; + const next = subViews[nextIndex]; + if (!next) return; + event.preventDefault(); + onTabChange(next.id); + event.currentTarget + .querySelectorAll('[role="tab"]') + [nextIndex]?.focus(); + }} + > + {subViews.map((tab) => ( + + ))} +
+ + {announcement} + + +); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.tsx index f4b1a57848d..ed11305e9b6 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.tsx @@ -10,27 +10,28 @@ import { useElapsedTime } from "./shared/use-elapsed-time"; export type BrunchWorkStatus = "streaming" | "settled" | "approval" | "stopped"; const foldStyle = css({ - backgroundColor: "neutral.a20", - borderRadius: "lg", - padding: "1", + display: "flex", + flexDirection: "column", minWidth: "[0]", }); const triggerStyle = css({ - display: "flex", + display: "inline-flex", + alignSelf: "flex-start", alignItems: "center", - gap: "2", - width: "full", - padding: "2", - fontSize: "sm", + gap: "1", + padding: "[2px 6px 2px 2px]", + borderRadius: "md", + _hover: { backgroundColor: "neutral.a20" }, + fontSize: "[13px]", fontWeight: "medium", color: "neutral.s100", textAlign: "left", cursor: "pointer", "& [data-chevron]": { - marginLeft: "auto", transition: "[transform 150ms ease]", }, - "&[data-state=closed] [data-chevron]": { transform: "[rotate(180deg)]" }, + "&[data-state=closed] [data-chevron]": { transform: "[rotate(90deg)]" }, + "&[data-state=open] [data-chevron]": { transform: "[rotate(180deg)]" }, "&[data-working=true] [data-label]": { backgroundImage: "[linear-gradient(110deg, {colors.neutral.s100} 35%, {colors.neutral.s60} 50%, {colors.neutral.s100} 65%)]", @@ -93,7 +94,9 @@ export const BrunchWorkFold = ({ display: "flex", flexDirection: "column", gap: "2", - padding: "1", + margin: "[4px 0 2px 5px]", + paddingLeft: "3", + borderLeft: "[2px solid {colors.neutral.a30}]", })} > {children} diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-execution-card.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-execution-card.tsx index 4d424a0136f..27bb3fa2fde 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-execution-card.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/experiment-execution-card.tsx @@ -80,6 +80,8 @@ const iconStyle = css({ const kindStyle = css({ fontSize: "xs", fontWeight: "medium", + textTransform: "uppercase", + letterSpacing: "wide", color: "purple.s100", lineHeight: "[16px]", }); @@ -163,11 +165,10 @@ const progressFillStyle = css({ "@media (prefers-reduced-motion: reduce)": { transition: "[none]" }, }); const metricsStyle = css({ - display: "flex", - flexWrap: "wrap", - columnGap: "4", - rowGap: "2", - flex: "[1 1 100px]", + display: "grid", + gridTemplateColumns: "[repeat(auto-fit, minmax(88px, 1fr))]", + gap: "1.5", + width: "full", minWidth: "[0]", margin: "0", }); @@ -175,6 +176,9 @@ const metricStyle = css({ display: "flex", flexDirection: "column", minWidth: "[0]", + padding: "2", + border: "[1px solid {colors.neutral.a30}]", + borderRadius: "lg", overflowWrap: "anywhere", "& > dt": { fontSize: "xs", color: "neutral.s90", lineHeight: "[16px]" }, "& > dd": { @@ -253,11 +257,11 @@ export const ExperimentExecutionCard = ({
+ + {optimization ? "Optimization" : "Simulation"} + {name}
- - {optimization ? "Optimization" : "Simulation"} - {(result || pending) && ( {result diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/reasoning.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/reasoning.tsx index ed487e0ee2a..e8d9dc45004 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/reasoning.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/reasoning.tsx @@ -16,9 +16,6 @@ const reasoningGroupStyle = css({ display: "flex", flexDirection: "column", gap: "1", - borderRadius: "lg", - backgroundColor: "neutral.bg.subtle", - padding: "1", }); const reasoningHeaderStyle = css({ @@ -26,23 +23,26 @@ const reasoningHeaderStyle = css({ alignItems: "center", gap: "2", width: "full", - height: "8", + minHeight: "6", paddingX: "1", border: "none", borderRadius: "lg", backgroundColor: "[transparent]", color: "neutral.s90", cursor: "pointer", - fontSize: "sm", + fontSize: "xs", fontWeight: "medium", textAlign: "left", _hover: { - backgroundColor: "white.a60", + backgroundColor: "neutral.a20", }, "& svg[data-chevron]": { transition: "[transform 150ms ease-out]", }, "&[data-state=closed] svg[data-chevron]": { + transform: "[rotate(90deg)]", + }, + "&[data-state=open] svg[data-chevron]": { transform: "[rotate(180deg)]", }, }); @@ -70,12 +70,9 @@ const reasoningBodyStyle = cva({ base: { position: "relative", overflow: "hidden", - borderWidth: "thin", - borderStyle: "solid", - borderColor: "neutral.a30", - borderRadius: "md", - backgroundColor: "neutral.s10", - padding: "2", + borderLeft: "[2px solid {colors.neutral.a30}]", + marginLeft: "2", + paddingLeft: "3", color: "neutral.s90", fontSize: "sm", fontWeight: "medium", @@ -243,7 +240,11 @@ export const AiAssistantReasoning = ({ className={css({ color: "orange.s90" })} /> - {elapsedTime === undefined ? "Thought" : `Thought for ${elapsedTime}`} + {isStreaming + ? "Thinking" + : elapsedTime === undefined + ? "Thought" + : `Thought for ${elapsedTime}`} {elapsedTime !== undefined && ( void; tool: ToolRenderItem; active?: boolean; + stopped?: boolean; }) => { const inProgress = tool.state === "input-streaming" || tool.state === "input-available"; + const cancelled = stopped && inProgress; const duration = useElapsedTime(active && inProgress && !tool.interactive); - if (tool.interactive) { + if (tool.interactive && !cancelled) { return ( { if (target) { onSelectToolTarget?.(target); @@ -599,8 +605,24 @@ const ToolItem = ({ > {title} @@ -656,7 +678,9 @@ const ToolItem = ({ > {tool.errorText ?? (tool.output === undefined - ? "Pending" + ? cancelled + ? "Cancelled before a result was received" + : "Pending" : JSON.stringify(tool.output, null, 2))} {children.map((item, index) => ( @@ -707,6 +731,16 @@ export const AiAssistantToolList = ({ stopped?: boolean; producedCard?: boolean; }) => { + const running = + active && + !stopped && + tools.some( + (tool) => + !tool.interactive && + (tool.state === "input-streaming" || tool.state === "input-available"), + ); + const [disclosure, setDisclosure] = useState({ running, open: running }); + if (disclosure.running !== running) setDisclosure({ running, open: running }); if (tools.length === 0) { return null; } @@ -724,11 +758,19 @@ export const AiAssistantToolList = ({ if (producedCard) return content; return ( <> - + setDisclosure({ running, open })} + > - - {stopped ? "Stopped after" : "Used"} {tools.length}{" "} - {tools.length === 1 ? "tool" : "tools"} + {running ? ( + @@ -742,6 +784,7 @@ export const AiAssistantToolList = ({ key={tool.id} tool={tool} active={active && !stopped} + stopped={stopped} onInteractiveToolSubmit={onInteractiveToolSubmit} onSelectToolTarget={onSelectToolTarget} /> @@ -755,6 +798,7 @@ export const AiAssistantToolList = ({ diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-input-provenance.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-input-provenance.tsx index 5cc6375637f..0ff87e4d545 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-input-provenance.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-input-provenance.tsx @@ -1,3 +1,4 @@ +import { Icon } from "@hashintel/ds-components"; import { css } from "@hashintel/ds-helpers/css"; import type { VoiceBrief } from "./get-message-render-items"; @@ -7,35 +8,55 @@ export const VoiceInputProvenance = ({ brief }: { brief: VoiceBrief }) => (
summary [data-chevron]": { transform: "[rotate(180deg)]" }, })} aria-busy={brief.state === "streaming"} > - + + Sent to Brunch +

Prepared from what you said

{Object.entries(brief.fields).map(([field, value]) => ( -
+
{field.replace(/([A-Z])/gu, " $1")}
{value}
From dc0930f7802f421da07da61fe3a0ad9a1396a6c0 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Thu, 24 Sep 2026 17:25:38 +0100 Subject: [PATCH 03/30] Use neutral activity labels and stock Petrinaut styling Reuse standard buttons and the existing blue palette, keep disclosure icons neutral, and remove purple experiment accents. Refs FE-1793 Co-authored-by: Amp --- ...unch-draft-experiment-interactive-tool.tsx | 54 ++++++------------- .../voice-history-continuity.test.tsx | 2 +- .../@hashintel/petrinaut/docs/ai-assistant.md | 8 +-- .../Editor/panels/ai-assistant-panel.test.tsx | 16 +++--- .../ai-assistant-contents.test.tsx | 12 ++--- .../ai-assistant-contents.tsx | 5 +- .../brunch-work-fold.test.tsx | 10 ++-- .../brunch-work-fold.tsx | 34 ++++++------ .../experiment-execution-card.test.tsx | 5 +- .../experiment-execution-card.tsx | 52 +++--------------- .../ai-assistant-contents/reasoning.tsx | 6 +-- .../voice-input-provenance.tsx | 2 +- 12 files changed, 71 insertions(+), 135 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-draft-experiment-interactive-tool.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-draft-experiment-interactive-tool.tsx index 00a9b60995b..734020a7187 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-draft-experiment-interactive-tool.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-draft-experiment-interactive-tool.tsx @@ -10,6 +10,7 @@ import { parseClientToolResultMetadata, } from "@hashintel/brunch-agent-plugin-sdcpn"; import { brunchTools } from "@hashintel/brunch-agent/constants"; +import { Button } from "@hashintel/ds-components"; import { css } from "@hashintel/ds-helpers/css"; import { ExperimentHostContext, @@ -54,13 +55,13 @@ const containerStyle = css({ padding: "3", borderWidth: "thin", borderStyle: "solid", - borderColor: "purple.a30", + borderColor: "neutral.a30", borderRadius: "lg", backgroundColor: "neutral.s00", }); const statusStyle = css({ - color: "purple.s100", + color: "neutral.s90", fontSize: "xs", fontWeight: "medium", letterSpacing: "wide", @@ -132,34 +133,6 @@ const actionsStyle = css({ marginTop: "1", }); -const primaryButtonStyle = css({ - paddingX: "3", - paddingY: "2", - borderRadius: "md", - backgroundColor: "purple.a85", - color: "white", - cursor: "pointer", - fontSize: "sm", - fontWeight: "medium", - _hover: { backgroundColor: "purple.a100" }, - _disabled: { cursor: "not-allowed", opacity: 0.45 }, -}); - -const secondaryButtonStyle = css({ - paddingX: "3", - paddingY: "2", - borderWidth: "thin", - borderStyle: "solid", - borderColor: "neutral.a30", - borderRadius: "md", - backgroundColor: "neutral.s00", - color: "neutral.s90", - cursor: "pointer", - fontSize: "sm", - fontWeight: "medium", - _hover: { backgroundColor: "neutral.a10" }, -}); - /** Forget every draft, as a reload would. For tests that share the module. */ export const resetBrunchEditorDrafts = resetEditorDrafts; @@ -702,26 +675,29 @@ export const BrunchDraftExperimentWidget = ({ ) : null} {canAct ? (
- + {canRun ? ( reviewed && !reviewAccepted ? ( - + ) : ( - + ) ) : null}
diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-history-continuity.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-history-continuity.test.tsx index a2cc2a6ed6d..a68b1176f5f 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-history-continuity.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-history-continuity.test.tsx @@ -372,7 +372,7 @@ test("projects typed, in-band tool, and stopped fixture history after remount", ).toBeNull(); expect(screen.getByText("voice-tool-1: The supervisor")).not.toBeNull(); expect(screen.getByText("Durably interrupted response")).not.toBeNull(); - expect(screen.getByText("Brunch stopped")).not.toBeNull(); + expect(screen.getByText("Stopped")).not.toBeNull(); await act(async () => fireEvent.click(screen.getByRole("button", { name: "Start voice mode" })), diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index 27c905c1b4f..7fdec34a7dd 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -39,7 +39,7 @@ The primary tab reads **Chat**, or **Voice** while Voice mode is selected. Your Completed Chat answers offer **Copy** and **Retry**. Copy keeps the answer's Markdown. Retry sends that answer's original prompt as a new turn, keeping the previous answer and any unsent draft. It can lead to new tool calls, just like sending the prompt yourself. Retry is unavailable while another response or voice handoff is active. -The work disclosure opens while **Brunch is working**, when **Brunch needs your approval**, or when **Brunch stopped**. Completed work collapses under **Brunch worked for Ns**, even while the answer is still streaming; history without timing says **Brunch worked**. Expand it to inspect **Thought for Ns** and **Used N tools**. Reasoning starts collapsed; **Running tools** opens automatically during execution and collapses when finished. Stopped work says **Stopped after N tools** and retains a **Response stopped** note. Unfinished tools show **Cancelled** while completed rows keep their results. Timing is shown when supplied or observed during this session; unavailable tool durations show a dash. Approval controls remain visible inside the work disclosure; **Allow** or **Deny** lets the assistant continue. +The activity disclosure opens while **Working…**, when **Approval required**, or when **Stopped**. Completed work collapses under **Activity · Ns**, even while the answer is still streaming; history without timing says **Activity**. Expand it to inspect **Thought for Ns** and **Used N tools**. Reasoning starts collapsed; **Running tools** opens automatically during execution and collapses when finished. Stopped work says **Stopped after N tools** and retains a **Response stopped** note. Unfinished tools show **Cancelled** while completed rows keep their results. Timing is shown when supplied or observed during this session; unavailable tool durations show a dash. Approval controls remain visible inside the disclosure; **Allow** or **Deny** lets the assistant continue. Disclosure icons are neutral; status dots distinguish pending, completed, and failed tools. When the host supplies them, Voice also shows a collapsed **Sent to Brunch** brief under your message, an immediate spoken-agent reply before the work, and a wrap-up after the produced cards. The brief says **Prepared from what you said** and lists its fields. These optional parts are absent in hosts that do not provide them. Messages have no per-message voice badge. @@ -57,7 +57,7 @@ If the host offers voice input, only a finalized transcript captured while Voice If an assistant request fails, Petrinaut shows the complete error in a persistent toast rather than adding it to the conversation. Long errors wrap, diagnostic details can be copied, and the toast stays open until you close it. Retry from the composer when the assistant is ready. -Hosts may provide canonical conversation rehydration. In that case, reopening the same assistant shows its settled and stopped turns without resubmitting a message or replaying Voice audio. Spoken user messages remain in the transcript without voice badges. Durably aborted assistant entries retain their **Brunch stopped** label even after later completed replies. If a tool-call step had already completed when Stop withheld its browser follow-up, that local decision has no durable cancellation record: hosts using initial-history recovery can recover the tool as pending work. Do not treat that local withholding as a reload-safe cancellation. +Hosts may provide canonical conversation rehydration. In that case, reopening the same assistant shows its settled and stopped turns without resubmitting a message or replaying Voice audio. Spoken user messages remain in the transcript without voice badges. Durably aborted assistant entries retain their **Stopped** label even after later completed replies. If a tool-call step had already completed when Stop withheld its browser follow-up, that local decision has no durable cancellation record: hosts using initial-history recovery can recover the tool as pending work. Do not treat that local withholding as a reload-safe cancellation. A host may also enable live history following, as the local Brunch panel does. Turns submitted elsewhere then appear in the open conversation without a reload. Your own in-progress response stays in place until the host confirms that its canonical history has caught up. In this mode, tools observed from another participant or restored after reopening are display-only: watching a pending tool does not execute it or resume that turn. Tools emitted in response to your own local submission still execute normally. A pending externally submitted tool needs its originating participant/operator to resolve it; reopening this following panel is not automatic recovery. @@ -289,8 +289,8 @@ For example: "Run 100 simulations of this scenario and show the completed orders metric." The assistant can also search numeric scenario parameter ranges to minimize or maximize a metric. -The experiment appears in a white card with purple accents, its status, run count, and -results. Optimization cards glow while running. Select **View +The experiment appears in a neutral card with a blue progress bar, its status, run count, and +results. Select **View experiment** to inspect metric distributions in the Experiments panel. The heatmap shows how values spread across runs; click a time step to see its histogram. Select **Cancel** diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx index 35e29c26ad9..eab2b765d4b 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx @@ -3431,7 +3431,7 @@ describe("AiAssistantPanel composer submissions", () => { ); await waitFor(() => expect(aborted).toHaveBeenCalledOnce()); - expect(await screen.findByText("Brunch stopped")).not.toBeNull(); + expect(await screen.findByText("Stopped")).not.toBeNull(); await waitFor(() => expect( screen @@ -3483,7 +3483,7 @@ describe("AiAssistantPanel composer submissions", () => { expect(requestStop.mock.invocationCallOrder[0]).toBeLessThan( localCancellation.mock.invocationCallOrder[0]!, ); - expect(await screen.findByText("Brunch stopped")).not.toBeNull(); + expect(await screen.findByText("Stopped")).not.toBeNull(); }); test("keeps a response completed before the durable Stop race", async () => { @@ -3537,7 +3537,7 @@ describe("AiAssistantPanel composer submissions", () => { ).toHaveProperty("disabled", true), ); expect(localCancellation).not.toHaveBeenCalled(); - expect(screen.queryByText("Brunch stopped")).toBeNull(); + expect(screen.queryByText("Stopped")).toBeNull(); }); test("withholds the client-tool follow-up when a durable Stop lands after a tool-calls step", async () => { @@ -3592,7 +3592,7 @@ describe("AiAssistantPanel composer submissions", () => { fireEvent.click(screen.getByRole("button", { name: "Stop AI response" })); await waitFor(() => expect(requestStop).toHaveBeenCalledOnce()); - expect(await screen.findByText("Brunch stopped")).not.toBeNull(); + expect(await screen.findByText("Stopped")).not.toBeNull(); // Let both deferred execution and any follow-up drain: withholding only // the send is insufficient if the mutation already ran after Stop. await act(() => new Promise((resolve) => setTimeout(resolve, 20))); @@ -3672,7 +3672,7 @@ describe("AiAssistantPanel composer submissions", () => { expect(latest?.status).toBe("error"); expect(latest?.stopped).toBe(false); - expect(screen.queryByText("Brunch stopped")).toBeNull(); + expect(screen.queryByText("Stopped")).toBeNull(); expect( screen.getAllByText(/Durable stop failed/u).length, ).toBeGreaterThan(0); @@ -3805,7 +3805,7 @@ describe("AiAssistantPanel composer submissions", () => { "PlaceOne", ); expect(sendMessages).not.toHaveBeenCalled(); - expect(screen.getByText("Brunch stopped")).not.toBeNull(); + expect(screen.getByText("Stopped")).not.toBeNull(); }); test("keeps hosts seeing a busy conversation between a tool-calls step and its follow-up", async () => { @@ -4021,7 +4021,7 @@ describe("AiAssistantPanel composer submissions", () => { await waitFor(() => expect(screen.getByTestId("host-status").textContent).toBe("ready"), ); - expect(screen.queryByText("Brunch stopped")).toBeNull(); + expect(screen.queryByText("Stopped")).toBeNull(); }); test("does not let a late durable Stop cancel a newer turn", async () => { @@ -4135,7 +4135,7 @@ describe("AiAssistantPanel composer submissions", () => { await waitFor(() => expect(screen.getByTestId("host-status").textContent).toBe("ready"), ); - expect(screen.queryByText("Brunch stopped")).toBeNull(); + expect(screen.queryByText("Stopped")).toBeNull(); }); test("does not surface a late durable Stop failure on a newer turn", async () => { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx index 3bad8e7d877..44df56ed237 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx @@ -49,7 +49,7 @@ vi.mock("react-markdown", async (importOriginal) => { const noop = () => {}; const expandWork = async () => { for (const fold of screen.queryAllByRole("button", { - name: /^Brunch worked/u, + name: /^Activity/u, })) { if (fold.getAttribute("aria-expanded") === "false") fireEvent.click(fold); await waitFor(() => @@ -992,7 +992,7 @@ describe("AiAssistantContents", () => { />, ); expect(container.textContent).toMatch( - /Sent to Brunch[\s\S]*I’ll ask Brunch\.[\s\S]*Brunch worked[\s\S]*Written answer[\s\S]*Experiment draft[\s\S]*Your draft is ready\./u, + /Sent to Brunch[\s\S]*I’ll ask Brunch\.[\s\S]*Activity[\s\S]*Written answer[\s\S]*Experiment draft[\s\S]*Your draft is ready\./u, ); expect( screen @@ -1151,7 +1151,7 @@ describe("AiAssistantContents", () => { await waitFor(() => expect( screen - .getByRole("button", { name: /^Brunch worked/u }) + .getByRole("button", { name: /^Activity/u }) .getAttribute("aria-expanded"), ).toBe("false"), ); @@ -1176,7 +1176,7 @@ describe("AiAssistantContents", () => { await waitFor(() => expect( screen - .getByRole("button", { name: "Brunch is working" }) + .getByRole("button", { name: "Working…" }) .getAttribute("aria-expanded"), ).toBe("true"), ); @@ -1679,7 +1679,7 @@ describe("AiAssistantContents", () => { ); expect(screen.getByText("Partial reply")).not.toBeNull(); expect(screen.getByText("Later completed reply")).not.toBeNull(); - expect(screen.getAllByText("Brunch stopped")).toHaveLength(1); + expect(screen.getAllByText("Stopped")).toHaveLength(1); }); test("keeps non-Voice assistant errors in global notifications", () => { @@ -3312,7 +3312,7 @@ describe("AiAssistantContents", () => { expect(screen.getByText("Created")).not.toBeNull(); expect( screen - .getByRole("button", { name: "Brunch worked" }) + .getByRole("button", { name: "Activity" }) .getAttribute("aria-expanded"), ).toBe("false"); await expandWork(); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx index 7b7ed7250a6..c3f7edbd97f 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx @@ -1189,10 +1189,7 @@ export const AiAssistantContents = ({ } {...(isFloating ? handleProps : {})} > - + {!additionalTab && ( {inputMode === "voice" ? "Voice" : "Chat"} )} diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.test.tsx index 709036bec8b..63664a0cec3 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.test.tsx @@ -13,10 +13,10 @@ import { BrunchWorkFold } from "./brunch-work-fold"; afterEach(cleanup); test.each([ - ["streaming", "Brunch is working", "true"], - ["settled", "Brunch worked for 7s", "false"], - ["approval", "Brunch needs your approval", "true"], - ["stopped", "Brunch stopped", "true"], + ["streaming", "Working…", "true"], + ["settled", "Activity · 7s", "false"], + ["approval", "Approval required", "true"], + ["stopped", "Stopped", "true"], ] as const)( "%s fold label and default disclosure", (status, label, expanded) => { @@ -36,7 +36,7 @@ test("settles after approval and permits manual reopening", async () => { Allow or deny, ); rerender(Approved); - const trigger = screen.getByRole("button", { name: "Brunch worked" }); + const trigger = screen.getByRole("button", { name: "Activity" }); await waitFor(() => expect(trigger.getAttribute("aria-expanded")).toBe("false"), ); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.tsx index ed11305e9b6..3dabce76bc8 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.tsx @@ -1,7 +1,7 @@ import { Collapsible } from "@ark-ui/react/collapsible"; import { type ReactNode, useState } from "react"; -import { Icon } from "@hashintel/ds-components"; +import { Button, Icon } from "@hashintel/ds-components"; import { css } from "@hashintel/ds-helpers/css"; import { collapsibleContentStyle } from "./shared/collapsible-content-style"; @@ -61,14 +61,14 @@ export const BrunchWorkFold = ({ setDisclosure({ status, open: defaultOpen }); const label = status === "streaming" - ? "Brunch is working" + ? "Working…" : status === "approval" - ? "Brunch needs your approval" + ? "Approval required" : status === "stopped" - ? "Brunch stopped" + ? "Stopped" : duration === undefined - ? "Brunch worked" - : `Brunch worked for ${Math.floor(duration / 1_000)}s`; + ? "Activity" + : `Activity · ${Math.floor(duration / 1_000)}s`; return ( setDisclosure({ status, open })} data-work-status={status} > - - - {label} - + +
setOpen(details.open)} > - + {isStreaming ? "Thinking" diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-input-provenance.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-input-provenance.tsx index 0ff87e4d545..448f4428b2c 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-input-provenance.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-input-provenance.tsx @@ -28,7 +28,7 @@ export const VoiceInputProvenance = ({ brief }: { brief: VoiceBrief }) => ( "& [data-chevron]": { transform: "[rotate(90deg)]" }, })} > - + Sent to Brunch
From 47c88882a90e7452ef2ced8eb243f39894f38fe1 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Thu, 24 Sep 2026 19:04:18 +0100 Subject: [PATCH 04/30] =?UTF-8?q?Restore=20the=20prototype=E2=80=99s=20com?= =?UTF-8?q?pact=20conversation=20disclosures?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep work, reasoning, and tools as flat inline disclosures with adjacent chevrons and nested guide lines. Preserve neutral icons, Petrinaut blue, and stock components. Refs FE-1793 Co-authored-by: Amp --- .../@hashintel/petrinaut/docs/ai-assistant.md | 2 ++ .../ai-assistant-contents.stories.tsx | 20 +++++++++++ .../ai-assistant-contents.test.tsx | 12 +++++-- .../brunch-work-fold.tsx | 23 ++++++------ .../ai-assistant-contents/reasoning.tsx | 35 +++++++++---------- .../ai-assistant-contents/tool-list.tsx | 32 ++++++++++++++--- 6 files changed, 87 insertions(+), 37 deletions(-) diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index 7fdec34a7dd..6a4cc61f938 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -43,6 +43,8 @@ The activity disclosure opens while **Working…**, when **Approval required**, When the host supplies them, Voice also shows a collapsed **Sent to Brunch** brief under your message, an immediate spoken-agent reply before the work, and a wrap-up after the produced cards. The brief says **Prepared from what you said** and lists its fields. These optional parts are absent in hosts that do not provide them. Messages have no per-message voice badge. +Work, reasoning, and tools use compact inline disclosures. Expand **Thought for Ns** to read the reasoning heading and details; expand the tool group to inspect its indented rows. Keyboard focus uses Petrinaut's blue outline. + While a response is streaming you can: - Expand the work disclosure to watch reasoning and tool operations. The working label shimmers while active. Reasoning stays collapsed until you choose to inspect it. diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx index 87efe601837..e2e04b1201e 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx @@ -1628,6 +1628,26 @@ export const ChatTurn: Story = { messages={[supportDeskUser, conversationTurn]} /> ), + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + const activity = canvas.getByRole("button", { name: "Activity" }); + await userEvent.click(activity); + // Disclosures are inline labels, not raised action buttons or full rows. + await expect(getComputedStyle(activity).boxShadow).toBe("none"); + await expect(getComputedStyle(activity).borderTopWidth).toBe("0px"); + for (const name of ["Thought for 7s", "Used 1 tool"]) { + const disclosure = canvas.getByRole("button", { name }); + await expect(disclosure.getBoundingClientRect().width).toBeLessThan(180); + await userEvent.click(disclosure); + await expect(disclosure).toHaveAttribute("aria-expanded", "true"); + } + await waitFor(() => + expect(canvas.getByText("Compare capacity")).toBeVisible(), + ); + await userEvent.click(activity); + await userEvent.keyboard("{Enter}"); + await expect(activity).toHaveAttribute("aria-expanded", "true"); + }, }; export const VoiceMediatedTurn: Story = { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx index 44df56ed237..ae5d7eae357 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx @@ -3318,12 +3318,20 @@ describe("AiAssistantContents", () => { await expandWork(); expect( screen - .getByRole("button", { name: /Thought for 4s/u }) + .getByRole("button", { name: "Thought for 4s" }) .getAttribute("aria-expanded"), ).toBe("false"); expect(screen.getByText("Thought for 4s")).not.toBeNull(); expect(screen.queryByTestId("reasoning-status")).toBeNull(); - expect(screen.getByLabelText(/Reasoning time/u)).not.toBeNull(); + fireEvent.click(screen.getByRole("button", { name: "Thought for 4s" })); + await waitFor(() => + expect( + screen + .getByRole("button", { name: "Thought for 4s" }) + .getAttribute("aria-expanded"), + ).toBe("true"), + ); + expect(screen.getByText("Planning the net")).not.toBeNull(); }); test("calls the clear handler from the header", () => { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.tsx index 3dabce76bc8..9d296bd0e1a 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/brunch-work-fold.tsx @@ -1,7 +1,7 @@ import { Collapsible } from "@ark-ui/react/collapsible"; import { type ReactNode, useState } from "react"; -import { Button, Icon } from "@hashintel/ds-components"; +import { Icon } from "@hashintel/ds-components"; import { css } from "@hashintel/ds-helpers/css"; import { collapsibleContentStyle } from "./shared/collapsible-content-style"; @@ -20,8 +20,11 @@ const triggerStyle = css({ alignItems: "center", gap: "1", padding: "[2px 6px 2px 2px]", + border: "none", borderRadius: "md", + backgroundColor: "[transparent]", _hover: { backgroundColor: "neutral.a20" }, + _focusVisible: { outline: "[2px solid {colors.blue.s90}]" }, fontSize: "[13px]", fontWeight: "medium", color: "neutral.s100", @@ -76,17 +79,13 @@ export const BrunchWorkFold = ({ onOpenChange={({ open }) => setDisclosure({ status, open })} data-work-status={status} > - - + + + {label} +
`. This helper hoists the heading so the - * collapsible card trigger can preview what the model is thinking about. + * item as `**Heading**\n\n`. Keep the heading with the expanded body, + * leaving the compact disclosure label for status and duration. * * If the convention is not matched (different provider, OpenAI changes the * format, or the model just produced an unheaded summary), we fall back to - * returning the original text as the body and let the trigger render the - * plain "Thinking" label. + * returning the original text as the body. */ const reasoningHeadingPattern = /^\s*(?:\*\*([^*\n]+?)\*\*|#+\s+([^\n]+))\s*(?:\n|$)/u; @@ -234,7 +231,7 @@ export const AiAssistantReasoning = ({ onOpenChange={(details) => setOpen(details.open)} > - + {isStreaming ? "Thinking" @@ -242,15 +239,15 @@ export const AiAssistantReasoning = ({ ? "Thought" : `Thought for ${elapsedTime}`} - {elapsedTime !== undefined && ( + {isStreaming && elapsedTime !== undefined && ( - {heading} + {elapsedTime} )} - +
diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/tool-list.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/tool-list.tsx index d7b84382703..73afd831fe1 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/tool-list.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/tool-list.tsx @@ -166,6 +166,19 @@ const toolItemStyle = cva({ color: "neutral.s90", }, }, + group: { + true: { + display: "inline-flex", + width: "[fit-content]", + minHeight: "[0]", + padding: "[2px 6px 2px 2px]", + gap: "1", + color: "neutral.s80", + _hover: { color: "neutral.s90" }, + _focusVisible: { outline: "[2px solid {colors.blue.s90}]" }, + "& svg[data-chevron]": { marginLeft: "0" }, + }, + }, link: { true: { cursor: "pointer", @@ -762,19 +775,30 @@ export const AiAssistantToolList = ({ open={disclosure.open} onOpenChange={({ open }) => setDisclosure({ running, open })} > - + {running ? ( -
+
{tools .filter( (tool) => !tool.interactive || tool.state !== "input-available", From f868c1c02d1c1aaad66e9d196263d9583d5658ea Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 25 Sep 2026 12:13:16 +0100 Subject: [PATCH 05/30] Align voice brief spacing and controls with the conversation design Mark spoken user messages in Chat with neutral voice bars. Match compact brief disclosures, conversation spacing, and mode-specific Stop controls without changing shared UI defaults. Refs FE-1793 Co-authored-by: Amp --- .../@hashintel/petrinaut/docs/ai-assistant.md | 4 +- .../ai-assistant-contents.stories.tsx | 87 ++++++++++++++++++- .../ai-assistant-contents.test.tsx | 32 +++++-- .../ai-assistant-contents.tsx | 52 +++++++++-- .../ai-assistant-contents/voice-dock.tsx | 6 ++ .../voice-dock/session-action-icons.tsx | 16 ++-- .../voice-input-provenance.tsx | 53 ++++++++--- 7 files changed, 210 insertions(+), 40 deletions(-) diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index 6a4cc61f938..b2947c6a27d 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -41,7 +41,7 @@ Completed Chat answers offer **Copy** and **Retry**. Copy keeps the answer's Mar The activity disclosure opens while **Working…**, when **Approval required**, or when **Stopped**. Completed work collapses under **Activity · Ns**, even while the answer is still streaming; history without timing says **Activity**. Expand it to inspect **Thought for Ns** and **Used N tools**. Reasoning starts collapsed; **Running tools** opens automatically during execution and collapses when finished. Stopped work says **Stopped after N tools** and retains a **Response stopped** note. Unfinished tools show **Cancelled** while completed rows keep their results. Timing is shown when supplied or observed during this session; unavailable tool durations show a dash. Approval controls remain visible inside the disclosure; **Allow** or **Deny** lets the assistant continue. Disclosure icons are neutral; status dots distinguish pending, completed, and failed tools. -When the host supplies them, Voice also shows a collapsed **Sent to Brunch** brief under your message, an immediate spoken-agent reply before the work, and a wrap-up after the produced cards. The brief says **Prepared from what you said** and lists its fields. These optional parts are absent in hosts that do not provide them. Messages have no per-message voice badge. +When the host supplies them, Voice also shows a collapsed **Sent to Brunch** brief directly under your message, an immediate spoken-agent reply before the work, and a wrap-up after the produced cards. Expand the brief to see **Prepared from what you said** and its right-aligned fields. These optional parts are absent in hosts that do not provide them. In Chat, a small neutral voice-bars icon marks user messages sent using Voice; typed messages have no icon. In Voice, those per-message icons are hidden. Work, reasoning, and tools use compact inline disclosures. Expand **Thought for Ns** to read the reasoning heading and details; expand the tool group to inspect its indented rows. Keyboard focus uses Petrinaut's blue outline. @@ -59,7 +59,7 @@ If the host offers voice input, only a finalized transcript captured while Voice If an assistant request fails, Petrinaut shows the complete error in a persistent toast rather than adding it to the conversation. Long errors wrap, diagnostic details can be copied, and the toast stays open until you close it. Retry from the composer when the assistant is ready. -Hosts may provide canonical conversation rehydration. In that case, reopening the same assistant shows its settled and stopped turns without resubmitting a message or replaying Voice audio. Spoken user messages remain in the transcript without voice badges. Durably aborted assistant entries retain their **Stopped** label even after later completed replies. If a tool-call step had already completed when Stop withheld its browser follow-up, that local decision has no durable cancellation record: hosts using initial-history recovery can recover the tool as pending work. Do not treat that local withholding as a reload-safe cancellation. +Hosts may provide canonical conversation rehydration. In that case, reopening the same assistant shows its settled and stopped turns without resubmitting a message or replaying Voice audio. Spoken user messages remain in the transcript and show the voice-bars icon in Chat when their voice origin is retained. Durably aborted assistant entries retain their **Stopped** label even after later completed replies. If a tool-call step had already completed when Stop withheld its browser follow-up, that local decision has no durable cancellation record: hosts using initial-history recovery can recover the tool as pending work. Do not treat that local withholding as a reload-safe cancellation. A host may also enable live history following, as the local Brunch panel does. Turns submitted elsewhere then appear in the open conversation without a reload. Your own in-progress response stays in place until the host confirms that its canonical history has caught up. In this mode, tools observed from another participant or restored after reopening are display-only: watching a pending tool does not execute it or resume that turn. Tools emitted in response to your own local submission still execute normally. A pending externally submitted tool needs its originating participant/operator to resolve it; reopening this following panel is not automatic recovery. diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx index e2e04b1201e..389925c074c 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx @@ -1119,10 +1119,23 @@ export const VoiceSessionThinking: Story = { ), + play: async ({ canvasElement }) => { + const stop = within(canvasElement).getByRole("button", { + name: "Stop AI response", + }); + await expect(stop.getBoundingClientRect().width).toBe(28); + await expect(stop.getBoundingClientRect().height).toBe(28); + await expect(stop.querySelector("svg")).toHaveAttribute( + "viewBox", + "0 0 24 24", + ); + await expect(stop.querySelector("rect")).toHaveAttribute("width", "11"); + }, }; export const VoiceSessionMuted: Story = { @@ -1192,6 +1205,16 @@ export const StreamingReasoning: Story = { status="streaming" /> ), + play: async ({ canvasElement }) => { + const stop = within(canvasElement).getByRole("button", { + name: "Stop AI response", + }); + await expect(stop.getBoundingClientRect().width).toBe(28); + await expect(stop.getBoundingClientRect().height).toBe(28); + await expect( + parseFloat(getComputedStyle(stop).borderRadius), + ).toBeGreaterThanOrEqual(14); + }, }; export const SingleCompletedToolCall: Story = { @@ -1631,6 +1654,12 @@ export const ChatTurn: Story = { play: async ({ canvasElement }) => { const canvas = within(canvasElement); const activity = canvas.getByRole("button", { name: "Activity" }); + const assistantTurn = activity.closest("[data-role=assistant]")!; + await expect(getComputedStyle(assistantTurn).paddingTop).toBe("4px"); + await expect( + getComputedStyle(assistantTurn.querySelector("[data-answer=brunch]")!) + .marginTop, + ).toBe("4px"); await userEvent.click(activity); // Disclosures are inline labels, not raised action buttons or full rows. await expect(getComputedStyle(activity).boxShadow).toBe("none"); @@ -1658,10 +1687,11 @@ export const VoiceMediatedTurn: Story = { voiceModeAvailable voiceSession={liveSession({ phase: "speaking" })} messages={[ - { ...supportDeskUser, metadata: { source: "voice" } }, { - ...conversationTurn, + ...supportDeskUser, + metadata: { source: "voice" }, parts: [ + ...supportDeskUser.parts, { type: "data-brief", data: { @@ -1673,6 +1703,11 @@ export const VoiceMediatedTurn: Story = { }, }, }, + ], + }, + { + ...conversationTurn, + parts: [ { type: "data-voiceAgentReply", data: { @@ -1693,6 +1728,54 @@ export const VoiceMediatedTurn: Story = { ]} /> ), + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + const summary = canvas.getByText("Sent to Brunch").closest("summary")!; + const userTurn = summary.closest("[data-role=user]")!; + const assistantTurn = canvasElement.querySelector("[data-role=assistant]")!; + await expect(getComputedStyle(userTurn).gap).toBe("2px"); + await expect(getComputedStyle(assistantTurn).gap).toBe("4px"); + await expect(getComputedStyle(assistantTurn).paddingTop).toBe("6px"); + await expect(getComputedStyle(summary).padding).toBe("2px 6px 2px 2px"); + await userEvent.click(summary); + await expect(summary.closest("details")).toHaveAttribute("open"); + await expect( + getComputedStyle(canvas.getByText("Prepared from what you said")) + .fontSize, + ).toBe("11px"); + await expect( + getComputedStyle(canvas.getByText("Arrival rate")).fontSize, + ).toBe("13px"); + await userEvent.click(summary); + }, +}; + +export const ChatVoiceOrigin: Story = { + render: () => ( + + ), + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect( + canvas.getAllByRole("img", { name: "Sent using voice" }), + ).toHaveLength(1); + await expect( + canvas + .getByRole("img", { name: "Sent using voice" }) + .querySelector("svg"), + ).toHaveAttribute("width", "12"); + }, }; export const VoicePreparing: Story = { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx index ae5d7eae357..cb7d8b1c30a 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx @@ -2085,7 +2085,9 @@ describe("AiAssistantContents", () => { ]) { const button = within(dock).getByRole("button", { name: label }); const icon = button.querySelector("svg"); - expect(icon?.getAttribute("viewBox")).toBe("0 0 20 20"); + expect(icon?.getAttribute("viewBox")).toBe( + label === "Stop AI response" ? "0 0 24 24" : "0 0 20 20", + ); expect(icon?.getAttribute("width")).toBe("16"); expect(icon?.getAttribute("height")).toBe("16"); expect(within(button).queryByText(label)).toBeNull(); @@ -2666,10 +2668,11 @@ describe("AiAssistantContents", () => { expect(onInputModeChange).toHaveBeenCalledWith("voice"); }); - test("renders spoken and typed messages without per-message voice markers", () => { - render( + test("marks only spoken user messages when switching from Voice to Chat", () => { + const contents = (inputMode: "voice" | "text") => ( { role: "user", parts: [{ type: "text", text: "Typed follow-up" }], }, + { + id: "voice-assistant", + metadata: { source: "voice" }, + role: "assistant", + parts: [{ type: "text", text: "Assistant reply" }], + }, ]} onClose={noop} onInputChange={noop} onStop={noop} onSubmit={noop} status="ready" - />, + /> ); - + const { rerender } = render(contents("voice")); + expect(screen.queryByRole("img", { name: "Sent using voice" })).toBeNull(); + rerender(contents("text")); expect( within( screen.getByText("Spoken workflow").closest("[data-role]")!, - ).queryByTestId("voice-input-provenance"), - ).toBeNull(); + ).getByRole("img", { name: "Sent using voice" }), + ).not.toBeNull(); + expect( + screen.getAllByRole("img", { name: "Sent using voice" }), + ).toHaveLength(1); expect( within( screen.getByText("Typed follow-up").closest("[data-role]")!, - ).queryByTestId("voice-input-provenance"), + ).queryByRole("img", { name: "Sent using voice" }), ).toBeNull(); + rerender(contents("voice")); + expect(screen.queryByRole("img", { name: "Sent using voice" })).toBeNull(); }); test("retains spoken tool answers without per-message voice markers", () => { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx index c3f7edbd97f..023226aec99 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx @@ -330,6 +330,7 @@ const messagesStyle = css({ minHeight: "[0]", overflowY: "auto", padding: "3", + paddingBottom: "4", overscrollBehavior: "contain", }); @@ -365,11 +366,17 @@ const messageStyle = cva({ role: { assistant: { alignSelf: "stretch", - paddingX: "[0]", + gap: "1", + padding: "[6px 0]", + '&[data-input-mode="text"]': { + paddingY: "1", + '& > [data-answer="brunch"]:not(:first-child)': { marginTop: "1" }, + }, }, user: { alignSelf: "flex-end", maxWidth: "[92%]", + gap: "0.5", padding: "[0]", }, }, @@ -380,6 +387,9 @@ const messageStyle = cva({ // `*`, `_`, `#`, etc. and collapse the single newlines they typed. Render it // verbatim with preserved whitespace instead. const userTextStyle = css({ + display: "flex", + alignItems: "flex-start", + gap: "1.5", alignSelf: "flex-end", backgroundColor: "neutral.a20", borderRadius: "lg", @@ -450,6 +460,12 @@ const composerActionButtonStyle = css({ height: "[30px]", minWidth: "[30px]", borderRadius: "[calc({radii.lg} - {spacing.1})]", + "&[data-stop=true]": { + width: "[28px]", + height: "[28px]", + minWidth: "[28px]", + borderRadius: "full", + }, }); const composerStyle = css({ @@ -630,13 +646,31 @@ const AiAssistantMessage = memo(
{role === "user" && (
- {answers.map((item) => ( -
{item.part.text}
- ))} + {!voice && message.metadata?.source === "voice" && ( + + + + )} +
+ {answers.map((item) => ( +
{item.part.text}
+ ))} +
)} {brief && } @@ -860,9 +894,9 @@ export const AiAssistantContents = ({ isSubmit: false, label: "Stop AI response", onClick: onStop, - tone: "neutral", + tone: "brand", type: "button", - variant: "subtle", + variant: "solid", } : canSubmit ? { @@ -1481,6 +1515,7 @@ export const AiAssistantContents = ({
{devicesExpanded && (
( ); -/** Uses the same line weight and canvas as the other Voice action icons. */ +/** Hang-up handset from the Voice dock design, distinct from closing the panel. */ export const EndIcon = () => ( - + diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-input-provenance.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-input-provenance.tsx index 3ef28125b79..3823999e11e 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-input-provenance.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-input-provenance.tsx @@ -46,7 +46,11 @@ export const VoiceInputProvenance = ({ brief }: { brief: VoiceBrief }) => ( > - Sent to Brunch + {brief.state === "streaming" + ? Object.keys(brief.fields).length === 0 + ? "Preparing for Brunch" + : "Sending to Brunch" + : "Sent to Brunch"}
( })} >

- Prepared from what you said + {brief.state === "streaming" && Object.keys(brief.fields).length === 0 + ? "Preparing from what you said" + : "Prepared from what you said"}

Date: Mon, 28 Sep 2026 17:29:42 +0200 Subject: [PATCH 07/30] Complete the conversation prototype controls and feedback Co-authored-by: Amp --- .changeset/brunch-conversation-turns.md | 2 +- .changeset/experiment-candidate-progress.md | 5 + AGENTS.md | 12 + .../brunch-conversation-id.ts | 24 ++ ...draft-experiment-interactive-tool.test.tsx | 53 +++ .../brunch-experiment-follow-up.tsx | 15 + ...nch-mutation-approval.integration.test.tsx | 363 ++++++++++++++++++ .../brunch-mutation-approval.test.tsx | 130 +++++++ .../brunch-mutation-approval.tsx | 250 ++++++++++++ .../live-conversation-control.test.tsx | 28 ++ .../live-conversation-control.tsx | 35 +- .../petrinaut-core/src/experiments/host.ts | 2 + .../@hashintel/petrinaut/docs/ai-assistant.md | 36 +- .../experiment-host/run-experiment.test.ts | 74 ++++ .../react/experiment-host/run-experiment.ts | 26 +- .../ui/types/ai-assistant-composer-control.ts | 2 + .../src/ui/types/ai-interactive-tool.ts | 7 + .../Editor/panels/ai-assistant-panel.tsx | 3 + .../ai-assistant-contents.stories.tsx | 36 +- .../ai-assistant-contents.test.tsx | 296 +++++++++++--- .../ai-assistant-contents.tsx | 226 ++++++++--- .../experiment-execution-card.test.tsx | 9 + .../experiment-execution-card.tsx | 6 + .../ai-assistant-contents/reasoning.tsx | 6 +- .../voice-alerts.test.tsx | 114 ++++++ .../ai-assistant-contents/voice-alerts.tsx | 81 +++- .../ai-assistant-contents/voice-dock.tsx | 65 +++- .../voice-dock/audio-popover/settings.tsx | 101 +---- .../voice-dock/voice-controls.test.tsx | 125 ++++++ .../interactive-tools/registry.test.tsx | 18 + .../interactive-tools/registry.ts | 2 +- 31 files changed, 1925 insertions(+), 227 deletions(-) create mode 100644 .changeset/experiment-candidate-progress.md create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/brunch-mutation-approval.integration.test.tsx create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/brunch-mutation-approval.test.tsx create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/brunch-mutation-approval.tsx create mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-alerts.test.tsx create mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock/voice-controls.test.tsx diff --git a/.changeset/brunch-conversation-turns.md b/.changeset/brunch-conversation-turns.md index 998f9dbec90..14576f4013d 100644 --- a/.changeset/brunch-conversation-turns.md +++ b/.changeset/brunch-conversation-turns.md @@ -2,4 +2,4 @@ "@hashintel/petrinaut": patch --- -Unify Chat and Voice turns with collapsible work, tool status and timing, answer cards, optional voice briefs and replies, and clearer experiment progress. Preserve unsent composer text when hosts submit background results. +Unify Chat and Voice turns with collapsible work, conditional host approvals, answer cards, voice briefs, and experiment candidate progress. Preserve drafts and reading position, and improve composer, audio settings, and recovery feedback. diff --git a/.changeset/experiment-candidate-progress.md b/.changeset/experiment-candidate-progress.md new file mode 100644 index 00000000000..f27b4ac188a --- /dev/null +++ b/.changeset/experiment-candidate-progress.md @@ -0,0 +1,5 @@ +--- +"@hashintel/petrinaut-core": patch +--- + +Allow experiment progress to report the parameter values of the optimization candidate currently being computed. diff --git a/AGENTS.md b/AGENTS.md index 4c9cf33d11f..30c2f57a39b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -157,3 +157,15 @@ Fix PostgreSQL integration type reference issues - Yarn workspaces: install a dependency in the relevant workspace, not at the root. File placement for TypeScript modules is covered by the `fractal-file-structuring` skill. + + + +# This is NOT the Turborepo you know + +Turborepo configuration, task behavior, and CLI commands can vary between installed versions and may differ from your training data. Resolve the `turbo` package from this file's directory or relevant workspace; in monorepos, it may not be visible from the repository root. For example, run `node -p "require.resolve('turbo/package.json')"` from a workspace that depends on `turbo`. + +Read `docs/README.md` inside that installed package first, then read the relevant pages from its `docs/` directory before changing Turborepo configuration or commands. Heed deprecation notices. These bundled docs match the installed package version and are available without network access. + +This block is written and re-added by `turbo` before repository-scoped commands when an AI agent is detected. In the Turborepo source repository, its template is defined in `crates/turborepo-cli/src/cli/agent_guidance.rs`. Removing the managed block while updates are enabled means a later qualifying invocation will add it again. Set `"agentGuidance": false` in the root `turbo.json` or `turbo.jsonc` to opt out; this does not remove an existing block. Keep the block committed with your work to avoid an uncommitted change on the next agent invocation. + + diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-conversation-id.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-conversation-id.ts index f1832ce1f6e..4219bafccd7 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-conversation-id.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-conversation-id.ts @@ -17,6 +17,30 @@ interface ConversationStorage { setItem(key: string, value: string): void; } +/** Change only the pointer; old conversation history and the model are retained. */ +export const replaceBrunchConversationId = ( + netId: string, + conversationId: string, + storage: ConversationStorage = window.localStorage, +): void => { + let stored: Record = {}; + try { + stored = JSON.parse( + storage.getItem(conversationStorageKey) ?? "{}", + ) as Record; + } catch { + // A broken pointer cache must not prevent starting a new conversation. + } + try { + storage.setItem( + conversationStorageKey, + JSON.stringify({ ...stored, [netId]: conversationId }), + ); + } catch { + // The host retains the new id for this page load. + } +}; + export const getOrCreateBrunchConversationId = ( netId: string, storage: ConversationStorage = window.localStorage, diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-draft-experiment-interactive-tool.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-draft-experiment-interactive-tool.test.tsx index 04f64e8c132..1a8e53e69da 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-draft-experiment-interactive-tool.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-draft-experiment-interactive-tool.test.tsx @@ -824,6 +824,59 @@ describe("BrunchDraftExperimentWidget", () => { ); }); + it("reports active runs only to their conversation and clears the hint on unmount", async () => { + const result = Promise.withResolvers(); + const { submit, wrap } = renderWidget({ + input: makeInput(), + toolCallId: "running-hint", + state: awaiting, + definition: createReadableStore(makeDefinition()), + runExperiment: vi.fn().mockReturnValue(result.promise), + }); + await waitFor(() => expect(submit).toHaveBeenCalledOnce()); + const reportExperimentRunning = vi.fn(); + const context: PetrinautAiComposerControlContext = { + conversationId: "original", + messages: [ + { + id: "draft", + role: "assistant", + parts: [ + { + type: "dynamic-tool", + toolName: "draft_petrinaut_experiment", + toolCallId: "running-hint", + state: "output-available", + input: {}, + output: {}, + }, + ], + }, + ], + status: "streaming", + submitText: vi.fn(), + stop: async () => {}, + reportExperimentRunning, + }; + const followUp = render( + wrap(), + ); + expect(reportExperimentRunning).toHaveBeenLastCalledWith(false); + fireEvent.click(screen.getByRole("button", { name: "Run" })); + await waitFor(() => + expect(reportExperimentRunning).toHaveBeenLastCalledWith(true), + ); + followUp.rerender( + wrap(), + ); + expect(reportExperimentRunning).toHaveBeenLastCalledWith(false); + followUp.rerender(wrap()); + expect(reportExperimentRunning).toHaveBeenLastCalledWith(true); + followUp.unmount(); + expect(reportExperimentRunning).toHaveBeenLastCalledWith(false); + await act(async () => result.resolve(finishedResult)); + }); + it("sends completed results once, only to the originating idle conversation", async () => { const definition = createReadableStore(makeDefinition()); const { submit, wrap } = renderWidget({ diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-experiment-follow-up.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-experiment-follow-up.tsx index 20a8f0ab146..c8f15c166ba 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-experiment-follow-up.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-experiment-follow-up.tsx @@ -44,6 +44,21 @@ export const BrunchExperimentFollowUp = ({ const instance = usePetrinautInstance(); const drafts = sessionDraftsFor(instance.definition); const snapshot = useSyncExternalStore(drafts.subscribe, drafts.get); + const running = [...snapshot.drafts.values()].some( + (draft) => + draft.run.phase === "running" && + context.messages.some((message) => + message.parts.some( + (part) => + "toolCallId" in part && part.toolCallId === draft.toolCallId, + ), + ), + ); + const reportRunning = context.reportExperimentRunning; + useEffect(() => { + reportRunning?.(running); + return () => reportRunning?.(false); + }, [reportRunning, running]); const pending = [...snapshot.drafts.values()].some( (draft) => draft.followUp === "pending", ); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-mutation-approval.integration.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-mutation-approval.integration.test.tsx new file mode 100644 index 00000000000..6c8226478fd --- /dev/null +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-mutation-approval.integration.test.tsx @@ -0,0 +1,363 @@ +/** @vitest-environment jsdom */ +import { + act, + cleanup, + fireEvent, + render, + screen, + waitFor, +} from "@testing-library/react"; +import { afterEach, beforeAll, expect, test, vi } from "vitest"; + +import { + mutatePetrinetInputSchema, + mutatePetrinetOutputSchema, + mutatePetrinautNetToolName, +} from "@hashintel/brunch-agent-plugin-sdcpn"; +import { createJsonDocHandle } from "@hashintel/petrinaut-core"; +import { + compileHirArtifacts, + lowerScenarioToHir, +} from "@hashintel/petrinaut-core/hir"; +import { createInProcessMonteCarloWorker } from "@hashintel/petrinaut-core/workers/monte-carlo"; +import { UserSettingsProvider } from "@hashintel/petrinaut/react"; +import { Petrinaut } from "@hashintel/petrinaut/ui"; + +import { + batchedConstructionClientToolNames, + brunchPetrinautDynamicToolNames, +} from "./brunch-client-tools"; +import { + createBrunchMutationApprovalCoordinator, + createBrunchMutationApprovalInteractiveTool, +} from "./brunch-mutation-approval"; +import { + BrunchPanelConversationTracker, + createBrunchPanelTransport, +} from "./brunch-panel-transport"; +import { createBrunchPetrinautTools } from "./brunch-petrinaut-tools"; +import { + createJoinedBrowserMutationRecorder, + observeBrowserDefinition, +} from "./mutation-record"; + +import type { AgentSendResult, FlueClient } from "@flue/sdk"; +import type { LspWorkerFactory, SDCPN } from "@hashintel/petrinaut-core"; + +vi.hoisted(() => { + window.matchMedia = (media) => ({ + media, + matches: false, + onchange: null, + addListener() {}, + removeListener() {}, + addEventListener() {}, + removeEventListener() {}, + dispatchEvent: () => true, + }); + Object.defineProperty(document, "queryCommandSupported", { + configurable: true, + value: () => false, + }); + class ClipboardItem { + constructor(readonly items: Record>) {} + } + Object.defineProperty(window, "ClipboardItem", { + configurable: true, + value: ClipboardItem, + }); + Object.defineProperty(navigator, "clipboard", { + configurable: true, + value: { + write: (items: ClipboardItem[]) => + Promise.all( + items.flatMap((item) => + Object.values(item.items).map((value) => Promise.resolve(value)), + ), + ).then(() => undefined), + }, + }); + Object.defineProperty(window, "CSS", { + configurable: true, + value: { + ...window.CSS, + escape: (value: string) => value.replace(/[^a-zA-Z0-9_-]/g, "\\$&"), + }, + }); +}); + +beforeAll(async () => { + await import("monaco-editor"); +}, 30_000); + +type LspWorker = Awaited>; +type LspWorkerMessage = Parameters[0]; +type LspWorkerListener = Parameters[1]; + +const cleanDiagnosticsWorker: LspWorkerFactory = () => { + const listeners = new Set(); + return { + postMessage(message: LspWorkerMessage) { + if (!("id" in message)) return; + let result: unknown; + if (message.method === "sdcpn/diagnostics") result = []; + else if (message.method === "sdcpn/compileHirArtifacts") + result = compileHirArtifacts( + message.params.sdcpn, + message.params.extensions, + message.params.options, + ); + else if (message.method === "sdcpn/lowerScenario") + result = lowerScenarioToHir(message.params.scenario, { + adHocContext: message.params.adHocContext, + }); + else return; + queueMicrotask(() => { + for (const listener of listeners) + listener({ data: { jsonrpc: "2.0", id: message.id, result } }); + }); + }, + addEventListener(_type, listener) { + listeners.add(listener); + }, + removeEventListener(_type, listener) { + listeners.delete(listener); + }, + terminate() { + listeners.clear(); + }, + }; +}; + +const initialNet: SDCPN = { + places: [ + { + id: "queue", + name: "Queue", + colorId: null, + dynamicsEnabled: false, + differentialEquationId: null, + x: 0, + y: 0, + }, + ], + transitions: [], + types: [], + differentialEquations: [], + parameters: [], +}; + +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); + vi.unstubAllGlobals(); +}); + +test.each(["Allow", "Deny"])( + "the real panel applies %s and submits the mutation result once", + async (choice) => { + vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null); + vi.stubGlobal( + "ResizeObserver", + class { + observe() {} + unobserve() {} + disconnect() {} + }, + ); + const handle = createJsonDocHandle({ + id: "approval-integration", + initial: initialNet, + }); + const mutationInput = mutatePetrinetInputSchema.parse({ + observation: { + toolCallId: "read-1", + baseHash: observeBrowserDefinition(handle).sha256, + }, + bases: [ + { + basisId: "basis-1", + basis: { kind: "absent", reason: "Integration test" }, + }, + ], + operations: [ + { + operationId: "remove-queue", + basisId: "basis-1", + type: "removePlace", + input: { placeId: "queue" }, + }, + ], + }); + const releaseServerValidation = Promise.withResolvers(); + const settleInitialStream = Promise.withResolvers(); + const tracker = new BrunchPanelConversationTracker(); + const approval = createBrunchMutationApprovalCoordinator(); + const send = vi.fn( + async (): Promise => ({ + submissionId: `submission-${send.mock.calls.length}`, + uid: "uid", + offset: "0", + streamUrl: "http://local.test/agents/chat/test/stream", + }), + ); + const wait = vi.fn(async (admission, options) => { + const submissionId = (admission as AgentSendResult).submissionId; + const messageId = `assistant-${submissionId}`; + let index = 0; + const position = () => ({ batch: 1, index: index++ }); + await options?.onEvent?.({ + type: "message-started", + conversationId: "test", + submissionId, + messageId, + turnId: messageId, + position: position(), + }); + if (submissionId === "submission-1") { + await options?.onEvent?.({ + type: "tool-input", + conversationId: "test", + messageId, + toolCallId: "mutation-1", + toolName: mutatePetrinautNetToolName, + input: mutationInput, + position: position(), + }); + await releaseServerValidation.promise; + await options?.onEvent?.({ + type: "tool-output", + conversationId: "test", + toolCallId: "mutation-1", + output: { awaiting: "client" }, + position: position(), + }); + await settleInitialStream.promise; + } else { + await options?.onEvent?.({ + type: "message-delta", + conversationId: "test", + messageId, + kind: "text", + delta: "The requested edit has been handled.", + position: position(), + }); + } + await options?.onEvent?.({ + type: "message-completed", + conversationId: "test", + messageId, + position: position(), + }); + await options?.onEvent?.({ + type: "submission-settled", + conversationId: "test", + submissionId, + outcome: "completed", + position: position(), + }); + }); + const client = { send, wait } as Pick< + FlueClient, + "send" | "wait" + > as FlueClient; + const binding = { + conversationId: "test", + documentId: handle.id, + incarnationId: "incarnation", + }; + const recorder = createJoinedBrowserMutationRecorder({ handle, binding }); + const approvalTool = + createBrunchMutationApprovalInteractiveTool(approval).interactiveTool; + + render( + + "Queue", + mutation: { + approval, + binding, + retainAttempt: recorder.retainAttempt, + }, + }), + interactiveTools: [approvalTool], + conversationId: "test", + requestStop: async () => "already-settled", + transport: createBrunchPanelTransport( + Promise.resolve(client), + tracker, + { + clientToolNames: batchedConstructionClientToolNames, + dynamicClientToolNames: brunchPetrinautDynamicToolNames, + validatedClientToolNames: recorder.validatedClientToolNames, + }, + ), + }} + /> + , + ); + + const show = await screen.findByRole("button", { + name: "Show AI assistant", + }); + await act(async () => fireEvent.click(show)); + const composer = await screen.findByRole("textbox", { + name: "Message AI assistant", + }); + fireEvent.change(composer, { target: { value: "Remove the queue." } }); + await act(async () => + fireEvent.click(screen.getByRole("button", { name: "Send message" })), + ); + + // Browser execution starts only after server validation releases the call. + expect(handle.doc()?.places.map(({ id }) => id)).toEqual(["queue"]); + await act(async () => releaseServerValidation.resolve()); + await act(async () => settleInitialStream.resolve()); + const approvalRegion = await screen.findByRole("region", { + name: "Approve destructive edits", + }); + expect(screen.getByRole("button", { name: "Allow" })).toBeTruthy(); + expect(screen.getByRole("button", { name: "Always allow" })).toBeTruthy(); + expect(screen.getByRole("button", { name: "Deny" })).toBeTruthy(); + expect(approvalRegion.textContent).toContain("Remove place — queue"); + expect(handle.doc()?.places.map(({ id }) => id)).toEqual(["queue"]); + expect(send).toHaveBeenCalledTimes(1); + + await act(async () => + fireEvent.click(screen.getByRole("button", { name: choice })), + ); + await waitFor(() => + expect(handle.doc()?.places.map(({ id }) => id)).toEqual( + choice === "Allow" ? [] : ["queue"], + ), + ); + await waitFor(() => expect(send).toHaveBeenCalledTimes(2)); + const followUp = send.mock.calls[1]![0].message; + expect(followUp.kind).toBe("signal"); + const results = JSON.parse((followUp as { body: string }).body) as Array<{ + toolCallId: string; + toolName: string; + output: unknown; + }>; + expect(results).toHaveLength(1); + expect(results[0]?.toolCallId).toBe("mutation-1"); + expect(results[0]?.toolName).toBe(mutatePetrinautNetToolName); + const output = mutatePetrinetOutputSchema.parse(results[0]?.output); + expect(output.outcomes).toMatchObject([ + { + operationId: "remove-queue", + status: choice === "Allow" ? "applied" : "failed", + }, + ]); + expect(results[0]?.output).not.toEqual({ decision: "allow" }); + + await screen.findByText("The requested edit has been handled."); + expect(send).toHaveBeenCalledTimes(2); + }, + 30_000, +); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-mutation-approval.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-mutation-approval.test.tsx new file mode 100644 index 00000000000..487e4154068 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-mutation-approval.test.tsx @@ -0,0 +1,130 @@ +/** + * @vitest-environment jsdom + */ +import { fireEvent, render, screen } from "@testing-library/react"; +import { describe, expect, test, vi } from "vitest"; + +import { mutatePetrinetInputSchema } from "@hashintel/brunch-agent-plugin-sdcpn"; + +import { + createBrunchMutationApprovalCoordinator, + createBrunchMutationApprovalInteractiveTool, +} from "./brunch-mutation-approval"; + +vi.hoisted(() => { + window.matchMedia = (media) => ({ + media, + matches: false, + onchange: null, + addListener() {}, + removeListener() {}, + addEventListener() {}, + removeEventListener() {}, + dispatchEvent: () => true, + }); +}); + +const destructiveInput = mutatePetrinetInputSchema.parse({ + observation: { toolCallId: "read-1", baseHash: "0".repeat(64) }, + bases: [{ basisId: "basis-1", basis: { kind: "absent", reason: "test" } }], + operations: [ + { + operationId: "remove-queue", + basisId: "basis-1", + type: "removePlace", + input: { placeId: "queue" }, + }, + { + operationId: "remove-wire", + basisId: "basis-1", + type: "removeArc", + input: { + transitionId: "serve", + placeId: "queue", + arcDirection: "input", + }, + }, + ], +}); + +describe("Brunch destructive mutation approval", () => { + test("renders itemized removals and resolves Allow without submitting a tool result", async () => { + const coordinator = createBrunchMutationApprovalCoordinator(); + const decision = coordinator.request({ + toolCallId: "mutation-1", + input: destructiveInput, + signal: new AbortController().signal, + }); + const { interactiveTool, Widget: ApprovalWidget } = + createBrunchMutationApprovalInteractiveTool(coordinator); + const submit = vi.fn(); + render( + , + ); + + expect(screen.getByText(/Remove place.*queue/u)).not.toBeNull(); + expect(screen.getByText(/Remove input arc.*serve.*queue/u)).not.toBeNull(); + expect( + screen.getByText(/associated arcs or references may also be removed/iu), + ).not.toBeNull(); + fireEvent.click(screen.getByRole("button", { name: "Allow" })); + + await expect(decision).resolves.toEqual({ decision: "allow" }); + expect(submit).not.toHaveBeenCalled(); + expect(interactiveTool.toolName).toBe("mutate_petrinaut_net"); + }); + + test("Always allow is scoped to one coordinator and abort revokes stale UI authority", async () => { + const coordinator = createBrunchMutationApprovalCoordinator(); + const first = coordinator.request({ + toolCallId: "mutation-1", + input: destructiveInput, + signal: new AbortController().signal, + }); + coordinator.resolve("mutation-1", "always-allow"); + await expect(first).resolves.toEqual({ decision: "allow" }); + await expect( + coordinator.request({ + toolCallId: "mutation-2", + input: destructiveInput, + signal: new AbortController().signal, + }), + ).resolves.toEqual({ decision: "allow" }); + + const fresh = createBrunchMutationApprovalCoordinator(); + const controller = new AbortController(); + const pending = fresh.request({ + toolCallId: "mutation-3", + input: destructiveInput, + signal: controller.signal, + }); + controller.abort(); + await expect(pending).resolves.toEqual({ + decision: "deny", + reason: "The mutation request was stopped before approval.", + }); + expect(fresh.resolve("mutation-3", "allow")).toBe(false); + expect(fresh.hasPending("mutation-3")).toBe(false); + }); + + test("historical rendering cannot create approval authority", () => { + const coordinator = createBrunchMutationApprovalCoordinator(); + const { Widget: ApprovalWidget } = + createBrunchMutationApprovalInteractiveTool(coordinator); + const { container } = render( + , + ); + expect(container.innerHTML).toBe(""); + expect(coordinator.resolve("historical-call", "allow")).toBe(false); + }); +}); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-mutation-approval.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-mutation-approval.tsx new file mode 100644 index 00000000000..a5d911a9620 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-mutation-approval.tsx @@ -0,0 +1,250 @@ +import { useSyncExternalStore } from "react"; + +import { + mutatePetrinetInputSchema, + mutatePetrinetOutputSchema, + mutatePetrinautNetToolName, + type MutatePetrinetInput, +} from "@hashintel/brunch-agent-plugin-sdcpn"; +import { Button } from "@hashintel/ds-components"; +import { css } from "@hashintel/ds-helpers/css"; +import { + definePetrinautAiInteractiveTool, + type PetrinautAiInteractiveToolWidgetProps, +} from "@hashintel/petrinaut/ui"; + +export type BrunchMutationApprovalChoice = "allow" | "always-allow" | "deny"; + +export type BrunchMutationApprovalDecision = + | { readonly decision: "allow" } + | { readonly decision: "deny"; readonly reason: string }; + +type PendingApproval = { + readonly resolve: (decision: BrunchMutationApprovalDecision) => void; + readonly removeAbortListener: () => void; +}; + +export interface BrunchMutationApprovalCoordinator { + request(params: { + readonly toolCallId: string; + readonly input: unknown; + readonly signal: AbortSignal; + }): Promise; + resolve(toolCallId: string, choice: BrunchMutationApprovalChoice): boolean; + hasPending(toolCallId: string): boolean; + subscribe: (listener: () => void) => () => void; + dispose(): void; +} + +const stoppedReason = "The mutation request was stopped before approval."; +const deniedReason = "The user denied this destructive mutation batch."; + +/** One ephemeral approval authority. Create once per mounted conversation. */ +export const createBrunchMutationApprovalCoordinator = + (): BrunchMutationApprovalCoordinator => { + const pending = new Map(); + const listeners = new Set<() => void>(); + let alwaysAllow = false; + let disposed = false; + const notify = () => listeners.forEach((listener) => listener()); + const settle = ( + toolCallId: string, + decision: BrunchMutationApprovalDecision, + ) => { + const approval = pending.get(toolCallId); + if (!approval) return false; + pending.delete(toolCallId); + approval.removeAbortListener(); + approval.resolve(decision); + notify(); + return true; + }; + + return { + request: ({ toolCallId, input, signal }) => { + mutatePetrinetInputSchema.parse(input); + if (disposed || signal.aborted) + return Promise.resolve({ decision: "deny", reason: stoppedReason }); + if (alwaysAllow) return Promise.resolve({ decision: "allow" }); + return new Promise((resolve) => { + const onAbort = () => { + settle(toolCallId, { decision: "deny", reason: stoppedReason }); + }; + signal.addEventListener("abort", onAbort, { once: true }); + pending.set(toolCallId, { + resolve, + removeAbortListener: () => + signal.removeEventListener("abort", onAbort), + }); + notify(); + }); + }, + resolve: (toolCallId, choice) => { + if (!pending.has(toolCallId) || disposed) return false; + if (choice === "always-allow") alwaysAllow = true; + return settle( + toolCallId, + choice === "deny" + ? { decision: "deny", reason: deniedReason } + : { decision: "allow" }, + ); + }, + hasPending: (toolCallId) => pending.has(toolCallId), + subscribe: (listener) => { + listeners.add(listener); + return () => listeners.delete(listener); + }, + dispose: () => { + disposed = true; + alwaysAllow = false; + for (const toolCallId of pending.keys()) + settle(toolCallId, { decision: "deny", reason: stoppedReason }); + }, + }; + }; + +const destructiveOperations = (input: MutatePetrinetInput) => + input.operations.filter(({ type }) => type.startsWith("remove")); + +export const requiresBrunchMutationApproval = (input: unknown): boolean => + destructiveOperations(mutatePetrinetInputSchema.parse(input)).length > 0; + +const operationDescription = ( + operation: MutatePetrinetInput["operations"][number], +): string => { + const values = operation.input as Record; + switch (operation.type) { + case "removePlace": + return `Remove place — ${String(values.placeId)}`; + case "removeTransition": + return `Remove transition — ${String(values.transitionId)}`; + case "removeArc": + return `Remove ${String(values.arcDirection)} arc — ${String(values.transitionId)} ↔ ${String(values.placeId)}`; + case "removeType": + return `Remove type — ${String(values.typeId)}`; + case "removeTypeElement": + return `Remove type element — ${String(values.typeId)} / ${String(values.elementId)}`; + case "removeParameter": + return `Remove parameter — ${String(values.parameterId)}`; + case "removeDifferentialEquation": + return `Remove differential equation — ${String(values.differentialEquationId)}`; + case "removeScenario": + return `Remove scenario — ${String(values.scenarioId)}`; + case "removeMetric": + return `Remove metric — ${String(values.metricId)}`; + default: + return operation.operationId; + } +}; + +const containerStyle = css({ + display: "flex", + flexDirection: "column", + gap: "2", + padding: "3", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a20", + borderRadius: "lg", + backgroundColor: "neutral.s00", +}); +const actionsStyle = css({ display: "flex", gap: "2", flexWrap: "wrap" }); + +type WidgetProps = PetrinautAiInteractiveToolWidgetProps< + MutatePetrinetInput, + unknown +>; + +const createWidget = (coordinator: BrunchMutationApprovalCoordinator) => { + const Widget = ({ + input, + toolCallId, + state, + submittedOutput, + }: WidgetProps) => { + const pending = useSyncExternalStore( + coordinator.subscribe, + () => coordinator.hasPending(toolCallId), + () => false, + ); + if (state === "submitted") { + const output = mutatePetrinetOutputSchema.parse(submittedOutput); + const failure = output.outcomes.find( + (outcome) => + outcome.status === "failed" || outcome.status === "unknown", + ); + return

{failure?.error ?? "Model edits completed."}

; + } + if (!pending) return null; + return ( +
+ Allow these destructive edits? +
    + {destructiveOperations(input).map((operation) => ( +
  • + {operationDescription(operation)} +
    • + ))} +
+

+ Associated arcs or references may also be removed. Always allow + applies to destructive edits in this conversation only, until you + leave or reload. +

+
+ + + +
+
+ ); + }; + return Widget; +}; + +export const createBrunchMutationApprovalInteractiveTool = ( + coordinator: BrunchMutationApprovalCoordinator, +) => { + const Widget = createWidget(coordinator); + return { + Widget, + interactiveTool: definePetrinautAiInteractiveTool({ + toolName: mutatePetrinautNetToolName, + inputSchema: mutatePetrinetInputSchema, + outputSchema: mutatePetrinetOutputSchema, + shouldHandle: requiresBrunchMutationApproval, + component: Widget, + }), + }; +}; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx index e38b377d0a8..1cbab4799e0 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx @@ -86,6 +86,34 @@ const config = { provider: "live" as const, connectionTimeoutMs: 15_000, }; +test("checks the Live microphone locally, releases tracks and never starts a session", async () => { + const stopTrack = vi.fn(); + const getUserMedia = vi.fn(async () => ({ + getTracks: () => [{ stop: stopTrack }], + })); + const previous = Object.getOwnPropertyDescriptor(navigator, "mediaDevices"); + Object.defineProperty(navigator, "mediaDevices", { + configurable: true, + value: { getUserMedia }, + }); + try { + render(); + fireEvent.click( + await screen.findByRole("button", { name: "Test microphone" }), + ); + await screen.findByText("Microphone ready. No audio was sent."); + expect(getUserMedia).toHaveBeenCalledWith({ audio: true }); + expect(stopTrack).toHaveBeenCalledOnce(); + expect(createLiveConversation).not.toHaveBeenCalled(); + expect(screen.getByRole("checkbox").getAttribute("aria-checked")).not.toBe( + "true", + ); + } finally { + if (previous) Object.defineProperty(navigator, "mediaDevices", previous); + else Reflect.deleteProperty(navigator, "mediaDevices"); + } +}); + const start = async () => { fireEvent.click(screen.getByRole("checkbox")); await waitFor(() => diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx index fd6cf010d7b..0e7b1d5e1b0 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx @@ -74,6 +74,8 @@ export const LiveConversationControl = ({ audioSettingsStore.getSnapshot, ); const [consented, setConsented] = useState(false); + const [checkingMicrophone, setCheckingMicrophone] = useState(false); + const [microphoneCheck, setMicrophoneCheck] = useState(""); const [warningMessage, setWarningMessage] = useState(null); const [microphoneMuted, setMicrophoneMutedState] = useState(false); const [speakerMuted, setSpeakerMutedState] = useState(false); @@ -262,7 +264,6 @@ export const LiveConversationControl = ({ handledVoiceSelection.current = true; return; } - // eslint-disable-next-line react-hooks-js/set-state-in-effect -- input mode synchronizes persisted disclosure state with the Live session handledVoiceSelection.current = tryStartLiveConversation(); }, [ disclosureAcknowledged, @@ -424,10 +425,36 @@ export const LiveConversationControl = ({ experimental consented={consented} onConsentChange={setConsented} - startDisabled={phase === "stopping"} - microphoneCheck={phase === "error" ? (state.message ?? "") : ""} + checkingMicrophone={checkingMicrophone} + startDisabled={phase === "stopping" || checkingMicrophone} + microphoneCheck={ + phase === "error" ? (state.message ?? "") : microphoneCheck + } + onCheckMicrophone={() => { + if (checkingMicrophone) return; + setCheckingMicrophone(true); + setMicrophoneCheck(""); + void (async () => { + try { + const microphoneId = audioSettings.devices.microphoneId; + const stream = await navigator.mediaDevices.getUserMedia({ + audio: microphoneId + ? { deviceId: { exact: microphoneId } } + : true, + }); + for (const track of stream.getTracks()) track.stop(); + setMicrophoneCheck("Microphone ready. No audio was sent."); + } catch { + setMicrophoneCheck( + "Microphone access was not available. Check your browser permissions and try again.", + ); + } finally { + setCheckingMicrophone(false); + } + })(); + }} onStart={() => { - if (!consented) return; + if (!consented || checkingMicrophone) return; acknowledgeDisclosure(); setDisclosureAcknowledged(true); tryStartLiveConversation(); diff --git a/libs/@hashintel/petrinaut-core/src/experiments/host.ts b/libs/@hashintel/petrinaut-core/src/experiments/host.ts index 21aedfe5aef..9e4a886af51 100644 --- a/libs/@hashintel/petrinaut-core/src/experiments/host.ts +++ b/libs/@hashintel/petrinaut-core/src/experiments/host.ts @@ -125,6 +125,8 @@ export type PetrinautExperimentProgress = { runsTarget: number; step?: number; steps?: number; + /** Parameter values being evaluated by the active optimization trial. */ + candidate?: Readonly>; }; export const petrinautExperimentResultSchema = z.strictObject({ diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index f26b36990bd..e11b0f2db78 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -33,23 +33,25 @@ The header text is not selectable. Its icons animate on hover and click, respect ## The conversation -Type in **Continue iterating...** and press **Enter** or choose the **Send message** button with the upward arrow at its right edge. **Shift+Enter** adds a line. The field grows with your message while the send button stays at the bottom right. Before the first message, suggestion chips wrap so all remain visible. +Type in **Continue iterating...** and press **Enter** or choose **Send message**. **Shift+Enter** adds a line. The field grows with your message. In Brunch Chat, the microphone stays on the left below the field, with Send or Stop on the right. **Waiting for your decision** marks an open approval; **Experiment running** marks an active run. Suggestion chips hide while Brunch is working and return when idle. Scrolling up keeps your reading position; new text follows automatically only when you are within 96 pixels of the bottom. The primary tab reads **Chat**, or **Voice** while Voice mode is selected. Your messages appear in right-aligned grey bubbles. Each assistant turn groups its work in a neutral disclosure, followed by a light-blue answer and any produced cards. In Voice, the written Brunch answer is inside the work disclosure instead. -Completed Chat answers offer **Copy** and **Retry**. The latest answer keeps these controls visible; older answers reveal them on hover or keyboard focus. Touch screens keep them visible. Copy keeps the answer's Markdown. Retry sends that answer's original prompt as a new turn, keeping the previous answer and any unsent draft. It can lead to new tool calls, just like sending the prompt yourself. Retry is unavailable while another response or voice handoff is active. +Completed Chat answers offer **Copy** and **Retry**. The latest answer keeps these controls visible; older answers reveal them on hover or keyboard focus. Touch screens keep them visible. Copy keeps the answer's Markdown and briefly shows a check mark after copying succeeds. Retry sends that answer's original prompt as a new turn, keeping the previous answer and any unsent draft. It can lead to new tool calls, just like sending the prompt yourself. Retry is unavailable while another response or voice handoff is active. -The activity disclosure opens while **Working…**, when **Approval required**, or when **Stopped**. Completed work collapses under **Activity · Ns**, even while the answer is still streaming; history without timing says **Activity**. Expand it to inspect **Thought for Ns** and **Used N tools**. Reasoning starts collapsed; **Running tools** opens automatically during execution and collapses when finished. Stopped work says **Stopped after N tools** and retains a **Response stopped** note. Unfinished tools show **Cancelled** while completed rows keep their results. Timing is shown when supplied or observed during this session; unavailable tool durations show a dash. Approval controls remain visible inside the disclosure; **Allow** or **Deny** lets the assistant continue. Disclosure icons are neutral; status dots distinguish pending, completed, and failed tools. +The activity disclosure opens while **Working…**, when **Approval required**, or when **Stopped**. Completed work collapses under **Activity · Ns**, even while the answer is still streaming; history without timing says **Activity**. Expand it to inspect **Thought for Ns** and **Used N tools**. Brunch Chat opens streaming reasoning automatically; you can collapse it while it continues. Other hosts retain their collapsed reasoning default. **Running tools** opens automatically during execution and collapses when finished. Stopped work says **Stopped after N tools** and retains a **Response stopped** note. Unfinished tools show **Cancelled** while completed rows keep their results. Timing is shown when supplied or observed during this session; unavailable tool durations show a dash. Approval controls remain visible inside the disclosure. Disclosure icons are neutral; status dots distinguish pending, completed, and failed tools. + +Before Brunch removes model elements, an approval lists the requested removals. Associated arcs or references may also be removed. **Allow** applies this batch; **Deny** prevents the entire batch, including any additions or updates it contains. **Always allow** permits later destructive batches only in the current mounted conversation, until you leave or reload. It does not grant permission for another conversation or browser session. Stop cancels a pending approval. A model changed while approval was open must be read again before the batch can apply. Stock auto-layout approval is unchanged. When the host supplies them, Voice also shows a collapsed brief directly under your message, an immediate spoken-agent reply before the work, and a wrap-up after the produced cards. The brief says **Preparing for Brunch** while its fields are being prepared, **Sending to Brunch** once the fields are ready but not yet accepted, and **Sent to Brunch** after acceptance. Expand a prepared brief to see **Prepared from what you said** and its right-aligned fields. These optional parts are absent in hosts that do not provide them. In Chat, a small neutral voice-bars icon marks user messages sent using Voice; typed messages have no icon. In Voice, those per-message icons are hidden. -Hosts that provide live input captions can show your words while you speak. This partial text is display-only: it does not submit work or start preparing a brief. The finalized transcript replaces it in the same bubble before preparation starts. +Hosts that provide live input captions can show your words while you speak. This partial text is display-only: it does not submit work or start preparing a brief. The finalized transcript replaces it in the same bubble before preparation starts. New spoken words and status labels fade in; reduced-motion preferences disable these effects. Work, reasoning, and tools use compact inline disclosures. Expand **Thought for Ns** to read the reasoning heading and details; expand the tool group to inspect its indented rows. Keyboard focus uses Petrinaut's blue outline. While a response is streaming you can: -- Expand the work disclosure to watch reasoning and tool operations. The working label shimmers while active. Reasoning stays collapsed until you choose to inspect it. +- Expand the work disclosure to watch reasoning and tool operations. The working label shimmers while active. Brunch Chat opens streaming reasoning; you can close it without interrupting the response. - Expand **Used N tools** to inspect chronological tool rows, each with its name, duration and status dot: amber pending, green completed, or red error. Expand a row to read its arguments and result. **Preparing…** means streamed arguments are arriving; **Running…** means execution is pending. Interactive questions remain available for your answer. - Press **Stop AI response** (the send button turns into a stop icon) to halt the current response. A host with durable conversation execution can record that stop before Petrinaut cancels its local stream; without that host capability, Stop is local cancellation only. Stop also withholds browser tools that have not started. Brunch's integrated assistant returns browser results within the active reply rather than requiring a second message; other hosts may still use an automatic follow-up. Already-applied changes are not rolled back. If a browser result is lost after a change may have happened, its outcome is unknown and the assistant does not automatically retry it. A silently disconnected browser is detected after a bounded liveness wait, not immediately. - Type your next message in the composer -- it is queued for after the current response ends. @@ -82,9 +84,9 @@ The document is mirrored to browser local storage automatically; there is no sep Wait for the status panel to report a settled bundle before reopening the same fixture in another tab. A refused status leaves the previous coherent bundle selected and names the failed history, workpiece, or document check instead of claiming that partial state settled. -When the Brunch voice preview is enabled and available, an empty composer shows a waveform action -titled **Start voice mode**. Typing non-whitespace text replaces it with **Send**. The same dynamic -action appears in the first-run prompt and the assistant panel; if voice is unavailable, the empty +When the Brunch voice preview is enabled and available, the composer shows a separate waveform action +titled **Start voice mode**. Typing keeps that action available beside **Send**. The first-run +prompt swaps its trailing Voice action for Send when you type; if voice is unavailable, the empty composer retains a disabled **Send** action. Starting Voice mode keeps the transcript in place and opens the existing one-time disclosure. Voice selected from the first-run prompt starts compact: the disclosure and microphone check appear in a card immediately above a **Voice setup** dock, while the @@ -106,7 +108,9 @@ best-effort spoken delivery. Before the first Live session, the permission panel explains both OpenAI audio streams and text retention, with a permission checkbox, **Start voice**, and **Cancel**. Petrinaut remembers this Live-specific versioned acknowledgement in browser storage, so later Live -sessions start directly. A failed or ended acknowledged session offers +sessions start directly. **Test microphone** checks access to the selected microphone locally +and immediately releases it; it does not start a provider session or send audio. +A failed or ended acknowledged session offers **Retry voice** without showing the consent prompt again. Browser microphone permission remains separate. **Cancel** returns to text without starting one. If the browser blocks remote @@ -168,14 +172,14 @@ change it with the switch beside the hand icon. When the host provides extended audio settings, Audio options also contains **Voice** when supported. **Devices** starts collapsed; expand it to choose a microphone or speaker. Providers with numeric speed control also show a -collapsed **Real-time** section immediately below **Devices**. The panel fits +visible **Speed** slider immediately below **Devices**. The panel fits its content and scrolls when it exceeds the available screen height. Opening Audio options refreshes the device list. Voice warnings appear after Audio options in the dock. - **Voice** saves a preference in this browser for the selected provider and applies it to the next Voice session, without restarting the current session. - The information icon at the right explains when changes apply and how to preview. + The guidance **Applies next session. Mute your mic to preview.** stays visible below the selector. You can select a voice at any time, including before connecting. In Brunch, selection also previews a short sample when the session is connected, your microphone is muted, and the agent is idle. Otherwise, it saves silently for @@ -191,10 +195,10 @@ options in the dock. - **Show status text** is on by default. Turn it off to hide ordinary Listening, Thinking, and Speaking labels beside the voice indicator. The waveform, controls, microphone mute and connection notices, and screen-reader announcements remain. -- Expand **Real-time** to use **Speed**. Its compact horizontal slider follows +- **Speed** stays visible when the provider supports it. Its compact horizontal slider follows the volume control's layout, with the multiplier at the right. It changes the next response, not speech already playing. - Realtime offers 0.25×–1.5× and resets to 1× for a new session. Live does not + Realtime offers 0.25×–1.5× in 0.05 steps and resets to 1× for a new session. Live does not offer numeric speed control. - **Microphone** and **Speaker** choose devices for the current session. They start at **System default**. If a selected device disconnects, Voice attempts @@ -256,7 +260,9 @@ Voice ends when the panel closes. If Realtime Voice is interrupted, allow microphone access or check the connection, then select **Reconnect voice mode**. **Clear AI chat** is unavailable while a Voice session is active. -The delete button appears in the top right of the panel once the conversation contains messages. When no interview is active and the host permits clearing, **Clear AI chat** wipes the local conversation, stops any in-flight stream, and tells the host app to forget the messages if it persists them. Hosts with canonical history may disable this control. The Brunch panel disables it because clearing only the browser view would not delete Flue history and the conversation would return on rehydration. +The delete button appears in the top right once the conversation contains messages. When Voice is inactive, **Clear AI chat** in ordinary Brunch starts a fresh conversation and resets conversation-only approvals. It preserves the model and the old saved history; it is not a history-deletion action. Reopening the page returns to the new conversation. Fixed prepared fixtures may still disable clearing. Other hosts can clear local messages or disable this control. + +An interrupted Voice session shows a gentle red waveform without a visible status label. Recovery controls remain available and screen readers still announce the interruption. Open **Voice issues** for a short title and explanation. **Copy details** becomes **Copied** after success; **Dismiss** clears the displayed issues without ending Voice. ## What the assistant can do @@ -339,7 +345,7 @@ disappears and the result remains under **Simulate → Experiments**. **Dismiss** retains the proposal as a **Dismissed** record. After **Run**, the draft is replaced in place by execution progress: **Validating**, simulation -or optimization steps and runs, then **Finished** with metric cells. **Cancel** +or optimization steps and runs, then **Finished** with metric cells. While an optimization candidate is computing, the step also shows its parameter values, such as `agent_count = 3`; no candidate is shown when it is unknown or between runs. **Cancel** leaves a **Cancelled** record. A failed run offers **Retry run**; model changes still require review before retrying. Drafting and starting a run keep your current tab selected; choose **View experiment** to open its results while the diff --git a/libs/@hashintel/petrinaut/src/react/experiment-host/run-experiment.test.ts b/libs/@hashintel/petrinaut/src/react/experiment-host/run-experiment.test.ts index ec3fb67ef68..5a93bfa9407 100644 --- a/libs/@hashintel/petrinaut/src/react/experiment-host/run-experiment.test.ts +++ b/libs/@hashintel/petrinaut/src/react/experiment-host/run-experiment.test.ts @@ -283,6 +283,80 @@ describe("runExperiment", () => { await pending; }); + it("publishes the active sweep candidate without retaining a finished candidate", async () => { + const harness = createHarness(true); + const onProgress = vi.fn(); + const pending = runExperiment(harness.dependencies, makeRequest(true), { + onProgress, + }); + await vi.waitFor(() => + expect(harness.actions.createOptimization).toHaveBeenCalledOnce(), + ); + onProgress.mockClear(); + + harness.optimizations.set([ + { + ...harness.study, + completedTrials: 1, + trials: [ + { + type: "trial", + trial: 0, + parameters: { rate: 0.2 }, + objective: 1, + state: "complete", + best: null, + seq: 1, + }, + ], + }, + ]); + harness.experiments.set([ + { + ...harness.record, + sweep: { + ...harness.record.sweep!, + computing: true, + selection: { rate: { from: 40, to: 40 } }, + }, + }, + ]); + await Promise.resolve(); + expect(onProgress).toHaveBeenLastCalledWith( + expect.objectContaining({ candidate: { rate: 0.8 } }), + ); + + onProgress.mockClear(); + harness.experiments.set([ + { + ...harness.record, + sweep: { + ...harness.record.sweep!, + computing: true, + selection: { rate: { from: 30, to: 30 } }, + }, + }, + ]); + await Promise.resolve(); + expect(onProgress).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ candidate: { rate: 0.6 } }), + ); + + onProgress.mockClear(); + harness.experiments.set([ + { + ...harness.record, + sweep: { ...harness.record.sweep!, computing: false }, + }, + ]); + await Promise.resolve(); + expect(onProgress).toHaveBeenCalledOnce(); + expect(onProgress.mock.lastCall?.[0]?.candidate).toBeUndefined(); + + harness.createOptions?.ownership?.cancel(); + await pending; + }); + it("captures the final distribution mean and releases its controls after completion", async () => { const harness = createHarness(); const onProgress = vi.fn(); diff --git a/libs/@hashintel/petrinaut/src/react/experiment-host/run-experiment.ts b/libs/@hashintel/petrinaut/src/react/experiment-host/run-experiment.ts index f54a31fe7e0..e1891006fb8 100644 --- a/libs/@hashintel/petrinaut/src/react/experiment-host/run-experiment.ts +++ b/libs/@hashintel/petrinaut/src/react/experiment-host/run-experiment.ts @@ -130,6 +130,27 @@ export const runExperiment = async ( const study = dependencies.optimizations .get() .find((record) => record.id === optimizationId); + const candidateEntries = + phase === "optimizing" && + study?.status === "running" && + experiment.sweep?.computing + ? experiment.parameterAxes.flatMap((axis) => { + const selection = experiment.sweep?.selection[axis.identifier]; + return selection && selection.from === selection.to + ? [ + [ + axis.identifier, + axisValueAt(axis, selection.from), + ] as const, + ] + : []; + }) + : []; + const candidate = + candidateEntries.length > 0 && + candidateEntries.length === experiment.parameterAxes.length + ? Object.fromEntries(candidateEntries) + : undefined; const progress: PetrinautExperimentProgress = { experimentId: experiment.id, name: experiment.name, @@ -149,6 +170,7 @@ export const runExperiment = async ( 1, ), steps: study.requestedTrials, + ...(candidate ? { candidate } : {}), } : {}), }; @@ -159,7 +181,9 @@ export const runExperiment = async ( lastProgress.runsCompleted === progress.runsCompleted && lastProgress.runsTarget === progress.runsTarget && lastProgress.step === progress.step && - lastProgress.steps === progress.steps + lastProgress.steps === progress.steps && + JSON.stringify(lastProgress.candidate) === + JSON.stringify(progress.candidate) ) { return; } diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts index 65de155fec4..5db75f0b012 100644 --- a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts +++ b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts @@ -50,6 +50,8 @@ export type PetrinautAiComposerControlContext = { stop: () => Promise; /** Call from an event handler or effect, never while rendering. */ submitText: PetrinautAiComposerSubmitText; + /** Report a host-owned experiment's activity from an effect; clear it on unmount. */ + reportExperimentRunning?: (running: boolean) => void; }; /** Render callback for a host-owned control inside the assistant composer. */ diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-interactive-tool.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-interactive-tool.ts index 79c72a90930..208b3553840 100644 --- a/libs/@hashintel/petrinaut/src/ui/types/ai-interactive-tool.ts +++ b/libs/@hashintel/petrinaut/src/ui/types/ai-interactive-tool.ts @@ -46,6 +46,8 @@ export type PetrinautAiInteractiveToolDefinition = { inputSchema: PetrinautAiInteractiveToolSchema; /** Runtime contract for the widget's submitted output. */ outputSchema: PetrinautAiInteractiveToolSchema; + /** Render an interaction only for matching validated inputs. Defaults to all. */ + shouldHandle?: (input: Input) => boolean; /** * Optionally map text submitted through the assistant composer to this * tool's output. Petrinaut validates both the pending input and mapped @@ -63,6 +65,7 @@ type ErasedInteractiveToolDefinition = { placement?: "work" | "card"; parseInput: (value: unknown) => unknown; parseOutput: (value: unknown) => unknown; + shouldHandle?: (input: unknown) => boolean; fromComposerText?: (params: { input: unknown; text: string }) => unknown; component: ComponentType< PetrinautAiInteractiveToolWidgetProps @@ -85,6 +88,7 @@ export const definePetrinautAiInteractiveTool = ( definition: PetrinautAiInteractiveToolDefinition, ): PetrinautAiInteractiveTool => { const fromComposerText = definition.fromComposerText; + const shouldHandle = definition.shouldHandle; return { toolName: definition.toolName, @@ -93,6 +97,9 @@ export const definePetrinautAiInteractiveTool = ( placement: definition.placement, parseInput: (value) => definition.inputSchema.parse(value), parseOutput: (value) => definition.outputSchema.parse(value), + shouldHandle: shouldHandle + ? (value) => shouldHandle(definition.inputSchema.parse(value)) + : undefined, fromComposerText: fromComposerText ? ({ input, text }) => definition.outputSchema.parse( diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index 1fb5c0d8124..667907046b5 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -815,6 +815,7 @@ const ConversationAiAssistantPanel = ({ kind: "stopped" | "failed"; } | null>(null); const automaticToolAbortsRef = useRef(new Set()); + const [hostExperimentRunning, setHostExperimentRunning] = useState(false); const abortAutomaticTools = () => { for (const controller of automaticToolAbortsRef.current) { controller.abort(); @@ -2293,6 +2294,7 @@ const ConversationAiAssistantPanel = ({ stopped, stop: stopComposer, submitText, + reportExperimentRunning: setHostExperimentRunning, }; const composerControl = aiAssistant.renderComposerControl?.( composerControlContext, @@ -2326,6 +2328,7 @@ const ConversationAiAssistantPanel = ({ composerControl={composerControl} error={streamError ?? error} experimentStates={experimentStates} + hostExperimentRunning={hostExperimentRunning} onCancelExperiment={cancelExperiment} input={input} inputMode={interactionMode} diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx index c256b2cdeef..7f86796efc0 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx @@ -364,6 +364,7 @@ const createStoryVoiceSessionStore = ( const Frame = ({ additionalTab, + composerControl, error, experimentStates, onCancelExperiment, @@ -384,6 +385,9 @@ const Frame = ({ workingLabel, }: { additionalTab?: ComponentProps["additionalTab"]; + composerControl?: ComponentProps< + typeof AiAssistantContents + >["composerControl"]; error?: Error; experimentStates?: ComponentProps< typeof AiAssistantContents @@ -436,6 +440,7 @@ const Frame = ({ > ((resolve) => @@ -815,7 +817,7 @@ export const ExtendedAudioSettings: Story = { ).toEqual(["Hide conversation", "Show 1 Voice issue"]); speed.focus(); await userEvent.keyboard("{ArrowLeft}"); - await expect(speed).toHaveAttribute("aria-valuenow", "1"); + await expect(speed).toHaveAttribute("aria-valuenow", "1.2"); await expect(getComputedStyle(speed).cursor).toBe("pointer"); await expect( getComputedStyle(canvas.getByRole("slider", { name: "Speaker volume" })) @@ -1194,6 +1196,22 @@ export const StreamingMarkdown: Story = { ), }; +/** Brunch Chat keeps voice entry on the left and the text action on the right. */ +export const BrunchComposerExperimentRunning: Story = { + render: () => ( + + ), +}; + export const ReasoningCollapsed: Story = { render: () => , }; @@ -1201,6 +1219,7 @@ export const ReasoningCollapsed: Story = { export const StreamingReasoning: Story = { render: () => ( @@ -1214,6 +1233,14 @@ export const StreamingReasoning: Story = { await expect( parseFloat(getComputedStyle(stop).borderRadius), ).toBeGreaterThanOrEqual(14); + const composer = within(canvasElement) + .getByRole("textbox", { + name: "Message AI assistant", + }) + .getBoundingClientRect(); + await expect(stop.getBoundingClientRect().left).toBeGreaterThan( + composer.left + composer.width / 2, + ); }, }; @@ -1961,6 +1988,7 @@ const ExperimentExample = ({ finished = false }: { finished?: boolean }) => { runsTarget: 12, step: 3, steps: 8, + candidate: { agents: 4 }, }, }, }} diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx index 6747d8c7e41..da9a2bee04a 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx @@ -556,7 +556,7 @@ test.each(["listening", "thinking", "speaking", "paused"] as const)( }, ); -test("collapses Real-time speed settings until expanded", async () => { +test("shows the speed slider immediately with fine keyboard steps", async () => { const setSpeed = vi.fn(); render( { ); fireEvent.click(screen.getByRole("button", { name: "Audio options" })); - const realTime = await screen.findByRole("button", { name: "Real-time" }); - expect(realTime.getAttribute("aria-expanded")).toBe("false"); - expect(screen.queryByRole("slider", { name: "Speed" })).toBeNull(); - - fireEvent.click(realTime); - expect(realTime.getAttribute("aria-expanded")).toBe("true"); - const speed = screen.getByRole("slider", { name: "Speed" }); + const speed = await screen.findByRole("slider", { name: "Speed" }); expect(speed.getAttribute("aria-valuemin")).toBe("0.25"); expect(speed.getAttribute("aria-valuemax")).toBe("1.5"); expect(speed.getAttribute("aria-valuenow")).toBe("1"); - expect(screen.getByText("1×")).not.toBeNull(); + expect(screen.getByText("1.00×")).not.toBeNull(); expect(screen.queryByText("Next reply")).toBeNull(); speed.focus(); fireEvent.keyDown(speed, { key: "ArrowRight" }); - await waitFor(() => expect(setSpeed).toHaveBeenCalledExactlyOnceWith(1.25)); + await waitFor(() => expect(setSpeed).toHaveBeenCalledExactlyOnceWith(1.05)); }); test("keeps voice visible while toggling devices and refreshes devices when opened", async () => { @@ -699,36 +693,12 @@ test("keeps voice visible while toggling devices and refreshes devices when open expect(devicesToggle.getAttribute("aria-expanded")).toBe("false"); expect(screen.queryByRole("combobox", { name: "Microphone" })).toBeNull(); expect(screen.queryByRole("combobox", { name: "Speaker" })).toBeNull(); - expect(screen.queryByText(/^The voice applies next time/)).toBeNull(); - fireEvent.click( - screen.getByRole("button", { name: "About voice selection" }), - ); - const voiceInfo = await screen.findByText(/^The voice applies next time/); - expect(voiceInfo.textContent).toBe( - "The voice applies next time the agent is connected.Preview when your mic is muted and the agent is idle.", - ); - expect(voiceInfo.querySelector("br")).not.toBeNull(); - await waitFor(() => - expect( - screen - .getByText(/^The voice applies next time/) - .closest('[data-part="content"]') - ?.getAttribute("data-state"), - ).toBe("open"), - ); - // Ark installs the nested dismissable layer on the next animation frame. - await act( - async () => - new Promise((resolve) => - requestAnimationFrame(() => requestAnimationFrame(() => resolve())), - ), - ); - fireEvent.keyDown(screen.getByText(/^The voice applies next time/), { - key: "Escape", - }); - await waitFor(() => - expect(screen.queryByText(/^The voice applies next time/)).toBeNull(), - ); + expect( + screen.getByText("Applies next session. Mute your mic to preview."), + ).toBeTruthy(); + expect( + screen.queryByRole("button", { name: "About voice selection" }), + ).toBeNull(); expect(screen.getByRole("combobox", { name: "Voice" })).not.toBeNull(); expect( screen.queryByText( @@ -1351,6 +1321,9 @@ describe("AiAssistantContents", () => { "The **queue** holds requests.", ), ); + expect( + screen.getByRole("button", { name: "Answer copied" }), + ).not.toBeNull(); fireEvent.click( screen.getAllByRole("button", { name: "Retry answer" })[0]!, ); @@ -2512,22 +2485,21 @@ describe("AiAssistantContents", () => { fireEvent.click( within(dock).getByRole("button", { name: "Show 1 Voice issue" }), ); - expect(await screen.findByText(warningMessage)).toBeTruthy(); + const warningTitle = warningMessage.split(". ")[0] ?? warningMessage; + expect(await screen.findByText(warningTitle)).toBeTruthy(); expect(within(dock).queryByText(warningMessage)).toBeNull(); expect( - screen.getByText(warningMessage).closest('[data-scope="toast"]'), + screen.getByText(warningTitle).closest('[data-scope="toast"]'), ).toBeNull(); act(() => { store.setState({ ...state, phase: "thinking", microphoneLevel: 0.5 }); }); - expect(screen.getAllByText(warningMessage)).toHaveLength(1); + expect(screen.getAllByText(warningTitle)).toHaveLength(1); act(() => { store.setState({ ...state, warningMessage: null }); }); - expect(screen.getByText(warningMessage)).toBeTruthy(); - fireEvent.click( - screen.getByRole("button", { name: "Dismiss Voice issues" }), - ); + expect(screen.getByText(warningTitle)).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: "Dismiss" })); expect(screen.queryByText(warningMessage)).toBeNull(); expect( screen.queryByRole("button", { name: /Show .*Voice issue/ }), @@ -2568,11 +2540,7 @@ describe("AiAssistantContents", () => { fireEvent.click( within(dock).getByRole("button", { name: "Show 1 Voice issue" }), ); - expect( - await screen.findByText( - "Microphone unavailable. Check your browser permissions.", - ), - ).toBeTruthy(); + expect(await screen.findByText("Microphone unavailable")).toBeTruthy(); expect( document.querySelector('[data-scope="toast"][data-part="root"]'), ).toBeNull(); @@ -2822,6 +2790,41 @@ describe("AiAssistantContents", () => { expect(screen.queryByRole("img", { name: "Sent using voice" })).toBeNull(); }); + test("animates new spoken words without replacing earlier words or losing spacing", () => { + const props = { + input: "", + onClose: noop, + onInputChange: noop, + onStop: noop, + onSubmit: noop, + status: "ready" as const, + }; + const message = (text: string): PetrinautAiMessage[] => [ + { + id: "reply", + role: "assistant", + parts: [ + { type: "data-voiceAgentReply", data: { state: "streaming", text } }, + ], + }, + ]; + const view = render( + , + ); + const first = view.container.querySelector("[data-streamed-word]"); + expect(first).not.toBeNull(); + view.rerender( + , + ); + expect(view.container.querySelector("[data-streamed-word]")).toBe(first); + expect( + view.container.querySelector('[data-answer="voice-reply"]')?.textContent, + ).toBe("I’ll check that.\nThen compare."); + }); + test("retains spoken tool answers without per-message voice markers", () => { const hostTool = definePetrinautAiInteractiveTool({ toolName: "answerQuestion", @@ -2959,10 +2962,127 @@ describe("AiAssistantContents", () => { const control = screen.getByRole("button", { name: "Alternate input" }); const sendButton = screen.getByRole("button", { name: "Send message" }); - expect(textarea.nextElementSibling).toBe(control); + expect(textarea.nextElementSibling?.contains(control)).toBe(true); expect(control.nextElementSibling?.contains(sendButton)).toBe(true); }); + test("puts the Brunch microphone left of the textarea and keeps Send on the right", () => { + const onInputModeChange = vi.fn(); + render( + , + ); + + const microphone = screen.getByRole("button", { name: "Start voice mode" }); + const textarea = screen.getByRole("textbox", { + name: "Message AI assistant", + }); + const send = screen.getByRole("button", { name: "Send message" }); + expect(microphone.compareDocumentPosition(send)).toBe( + Node.DOCUMENT_POSITION_FOLLOWING, + ); + expect(textarea.nextElementSibling?.contains(send)).toBe(true); + fireEvent.click(microphone); + expect(onInputModeChange).toHaveBeenCalledWith("voice"); + }); + + test("shows Brunch work hints and hides prompt chips while work is active", () => { + const props = { + input: "", + messages: [] as PetrinautAiMessage[], + onClose: noop, + onInputChange: noop, + onSendPrompt: noop, + onStop: noop, + onSubmit: noop, + primaryLabel: "Chat", + interactiveTools: [ + definePetrinautAiInteractiveTool({ + toolName: "confirm", + inputSchema: { parse: (value: unknown) => value }, + outputSchema: { parse: (value: unknown) => value }, + component: () => Approval, + }), + ], + promptChips: [{ id: "review", label: "Review", prompt: "Review" }], + status: "streaming" as const, + voiceModeAvailable: true, + }; + const view = render( + , + ); + expect(screen.getByText("Experiment running")).not.toBeNull(); + expect(screen.queryByRole("button", { name: "Review" })).toBeNull(); + + view.rerender( + , + ); + expect(screen.getByText("Waiting for your decision")).not.toBeNull(); + view.rerender( + value }, + outputSchema: { parse: (value: unknown) => value }, + component: () => Preparing a draft, + }), + ]} + messages={[ + { + id: "assistant", + role: "assistant", + parts: [ + { + type: "dynamic-tool", + toolName: "confirm", + toolCallId: "draft", + state: "input-available", + input: {}, + }, + ], + }, + ]} + />, + ); + expect(screen.queryByText("Waiting for your decision")).toBeNull(); + }); + test("switches the trailing action from Voice mode to Send for trimmed input", () => { const onInputModeChange = vi.fn(); const onSubmit = vi.fn(); @@ -3529,7 +3649,61 @@ describe("AiAssistantContents", () => { window.cancelAnimationFrame = originalCancelAnimationFrame; }); - test("renders a streaming ellipsis for empty streaming reasoning", () => { + test("does not auto-follow new content after the reader scrolls more than 96px from the end", () => { + // eslint-disable-next-line @typescript-eslint/unbound-method -- Saved only for restoration. + const originalScrollTo = window.HTMLElement.prototype.scrollTo; + const originalRequestAnimationFrame = window.requestAnimationFrame; + const scrollTo = vi.fn(); + window.HTMLElement.prototype.scrollTo = scrollTo; + window.requestAnimationFrame = (callback) => { + callback(0); + return 0; + }; + const props = { + input: "", + messages: [ + { + id: "assistant-1", + role: "assistant" as const, + parts: [ + { type: "text" as const, state: "streaming" as const, text: "One" }, + ], + }, + ], + onClose: noop, + onInputChange: noop, + onStop: noop, + onSubmit: noop, + status: "streaming" as const, + }; + const view = render(); + const transcript = screen.getByTestId("ai-transcript"); + Object.defineProperties(transcript, { + clientHeight: { configurable: true, value: 400 }, + scrollHeight: { configurable: true, value: 1000 }, + scrollTop: { configurable: true, writable: true, value: 400 }, + }); + fireEvent.scroll(transcript); + scrollTo.mockClear(); + + view.rerender( + , + ); + + expect(scrollTo).not.toHaveBeenCalled(); + window.HTMLElement.prototype.scrollTo = originalScrollTo; + window.requestAnimationFrame = originalRequestAnimationFrame; + }); + + test("renders a streaming ellipsis for empty streaming reasoning", async () => { const messages: PetrinautAiMessage[] = [ { id: "assistant-1", @@ -3546,6 +3720,7 @@ describe("AiAssistantContents", () => { render( { ); expect(screen.getByTestId("reasoning-loading")).not.toBeNull(); + expect( + screen + .getByRole("button", { name: /^Thinking/u }) + .getAttribute("aria-expanded"), + ).toBe("true"); + fireEvent.click(screen.getByRole("button", { name: /^Thinking/u })); + await waitFor(() => + expect( + screen + .getByRole("button", { name: /^Thinking/u }) + .getAttribute("aria-expanded"), + ).toBe("false"), + ); expect(screen.queryByText("Thinking...")).toBeNull(); }); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx index 413ed2ce064..f04ca4ad140 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx @@ -55,6 +55,7 @@ import { import { VoiceAlerts } from "./ai-assistant-contents/voice-alerts"; import { LiveVoiceDock, VoiceDock } from "./ai-assistant-contents/voice-dock"; import { VoiceInputProvenance } from "./ai-assistant-contents/voice-input-provenance"; +import { getInteractiveTool } from "./interactive-tools/registry"; import type { PetrinautAiAssistant, @@ -90,6 +91,7 @@ export type AiAssistantContentsProps = { composerFocusRequest?: number; error?: Error; experimentStates?: Record; + hostExperimentRunning?: boolean; onCancelExperiment?: (toolCallId: string) => void; input: string; inputMode?: PetrinautAiInputMode; @@ -413,6 +415,12 @@ const answerStyle = css({ padding: "[10px]", color: "neutral.s100", overflowWrap: "anywhere", + '&[data-streaming="true"]': { + animation: "[petrinautComposerActionSwap 180ms ease-out]", + }, + "@media (prefers-reduced-motion: reduce)": { + animation: "[none]", + }, "[data-work-status] &": { alignSelf: "stretch", maxWidth: "full", @@ -430,6 +438,8 @@ const workingStatusStyle = css({ color: "neutral.s80", fontSize: "sm", fontWeight: "medium", + animation: "[petrinautComposerActionSwap 180ms ease-out]", + "@media (prefers-reduced-motion: reduce)": { animation: "[none]" }, }); const composerWrapStyle = css({ @@ -478,6 +488,7 @@ const composerActionButtonStyle = css({ const composerStyle = css({ display: "flex", alignItems: "flex-end", + '&[data-brunch="true"]': { flexDirection: "column", alignItems: "stretch" }, gap: "1", borderRadius: "lg", backgroundColor: "neutral.s10", @@ -530,6 +541,37 @@ const composerTextareaStyle = css({ }, }); +const composerHintStyle = css({ + flex: "1", + paddingX: "2", + paddingBottom: "1", + color: "neutral.s80", + fontSize: "xs", + textAlign: "left", +}); + +const StreamingWords = ({ + text, + streaming, +}: { + text: string; + streaming: boolean; +}) => + streaming + ? [...text.matchAll(/\S+\s*|\s+/gu)].map((word) => ( + + {word[0]} + + )) + : text; + // The scroll effect only needs to know when *anything* changed — it doesn't // need to capture every byte of every part. Constant-time: look at the last // message and its last part. This runs on every render during streaming, so @@ -585,6 +627,7 @@ const AiAssistantMessage = memo( onCancelExperiment, resolveToolPresentation, voice, + expandReasoning, active, stopped, canRetry, @@ -598,12 +641,19 @@ const AiAssistantMessage = memo( onCancelExperiment?: (toolCallId: string) => void; resolveToolPresentation?: PetrinautAiToolPresentationResolver; voice: boolean; + expandReasoning: boolean; active: boolean; stopped: boolean; canRetry: boolean; latestAnswer: boolean; }) => { const { addNotification } = use(NotificationsContext); + const [copied, setCopied] = useState(false); + useEffect(() => { + if (!copied) return; + const timer = window.setTimeout(() => setCopied(false), 1_200); + return () => window.clearTimeout(timer); + }, [copied]); const role = message.role === "user" ? "user" : "assistant"; const renderItems = getMessageRenderItems( message, @@ -635,7 +685,11 @@ const AiAssistantMessage = memo( : "settled"; const writtenAnswer = answers.length > 0 ? ( -
+
{answers.map((item) => (
{item.part.text} @@ -678,7 +732,15 @@ const AiAssistantMessage = memo( )}
{answers.map((item) => ( -
{item.part.text}
+
+ +
))}
@@ -690,7 +752,10 @@ const AiAssistantMessage = memo( data-answer="voice-reply" aria-busy={voiceAgentReply.state === "streaming"} > - {voiceAgentReply.text} +
)} {showWork && ( @@ -699,6 +764,7 @@ const AiAssistantMessage = memo( ))} @@ -770,7 +836,10 @@ const AiAssistantMessage = memo( data-answer="voice-wrap-up" aria-busy={voiceAgentWrapUp.state === "streaming"} > - {voiceAgentWrapUp.text} +
)} {wasStopped && ( @@ -794,19 +863,18 @@ const AiAssistantMessage = memo(