From 833424f88129185c3e1dfd1285b3c78e3e412bfa Mon Sep 17 00:00:00 2001 From: PMullz Date: Fri, 18 Sep 2026 22:47:05 -0400 Subject: [PATCH 1/2] fix(widget): harden voice after independent review An independent review of the voice feature reproduced several real bugs that 507 green tests had missed, largely because the speechSynthesis test double encoded my own wrong assumption. Each fix below is pinned by a regression test (507 -> 541). Read aloud - Pause state now follows the utterance's pause / resume events. Chrome and Safari set `speechSynthesis.paused` only once the engine confirms, so reading the flag right after pause() left the UI on "Pause" with no way to resume. Confirmed against Chrome 153. The fake now confirms a pause asynchronously by default, and no longer clears `paused` on cancel(), which the spec says it must not. - The reader only cancels speech it started. `speechSynthesis` is a page global and the hook mounts even with voice off, so closing the chat silenced the host page's own speech on every embed. With two widgets, the first one's late "interrupted" error cancelled the second's queue and nothing played. - Stopping while paused resumes the engine, which cancel() leaves paused; otherwise the next reply queued in silence. - The controls keep keyboard focus. Swapping a bare button for a wrapper dropped focus to , outside the dialog's focus trap, so Tab could leave the dialog. Dictation - Hold mode lasts as long as the hold. With single-utterance sessions the engine ends at the first pause, silently dropping the rest and, with auto-submit, sending half a message while the button was still down. A hold now spans sessions and auto-submit waits for the release. Toggle starts, including the assistive-technology click, are not bridged, since nothing guarantees their release. - Pressing again while the previous session winds down no longer duplicates the text: a refused start() must not move the dictation base. - Read-aloud now ends dictation, as dictation already ended read-aloud. Otherwise the mic transcribed the reply and auto-submit sent it as the visitor's message. - A request starting to load ends dictation, so auto-submit cannot clear text that useChat then refuses to send. - An error releases the session without waiting for `end`; an engine-side "aborted" no longer runs auto-submit; a voice error clears on send. Helpers - chunkSpeechText: 150 characters (about ten seconds, with room for numbers), CJK sentence marks, and a last-resort split for unbroken tokens. 560 characters of Japanese used to become one utterance. - stripAnnouncementFormatting keeps the punctuation after a URL, which now decides where speech pauses. - The `voice` option fails closed: the string "false" no longer switches the microphone on. - Visible focus ring on the mic while it is filled with the accent color. e2e: fakes behave like Chrome, plus hold-to-talk through the IIFE bundle with a real mouse press and release, confirming in Chromium that the click fired after the release carries a click count and does not restart dictation. Refs #54 Co-Authored-By: Claude Fable 5.1 --- widget/e2e/voice.spec.ts | 71 ++++++- widget/src/components/ChatInput.tsx | 85 +++++++- widget/src/components/ChatWindow.tsx | 4 +- .../src/components/MessageSpeechControls.tsx | 84 +++++--- widget/src/components/VoiceInputButton.tsx | 15 +- .../__tests__/ChatInput.voice.test.tsx | 191 +++++++++++++++++- .../__tests__/ChatWindow.voice.test.tsx | 18 +- .../__tests__/MessageSpeechControls.test.tsx | 71 +++++++ .../__tests__/VoiceInputButton.test.tsx | 10 +- .../__tests__/useSpeechRecognition.test.ts | 91 +++++++++ .../__tests__/useSpeechSynthesis.test.ts | 113 ++++++++++- widget/src/hooks/useSpeechRecognition.ts | 56 +++-- widget/src/hooks/useSpeechSynthesis.ts | 37 +++- widget/src/test-utils/fakeSpeechSynthesis.ts | 34 +++- .../stripAnnouncementFormatting.test.ts | 12 ++ widget/src/utils/__tests__/voice.test.ts | 41 +++- .../src/utils/stripAnnouncementFormatting.ts | 6 +- widget/src/utils/voice.ts | 32 ++- 18 files changed, 863 insertions(+), 108 deletions(-) diff --git a/widget/e2e/voice.spec.ts b/widget/e2e/voice.spec.ts index 5cb723f..ad19166 100644 --- a/widget/e2e/voice.spec.ts +++ b/widget/e2e/voice.spec.ts @@ -1,6 +1,12 @@ import { test, expect, type Page } from "@playwright/test"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; import { mockChatApi } from "./helpers"; +const __dirname = dirname(fileURLToPath(import.meta.url)); +const IIFE_PATH = resolve(__dirname, "..", "dist", "claudius.iife.js"); +const CSS_PATH = resolve(__dirname, "..", "dist", "claudius.css"); + /** * Voice features against fake speech engines, injected before the app loads. * Real engines cannot run here: CI has no microphone, and Chromium builds ship @@ -31,6 +37,10 @@ async function installFakeSpeech(page: Page) { this.onend?.({}); } abort() {} + /** The engine ends the session by itself, as it does at a pause. */ + end() { + this.onend?.({}); + } hear(transcript: string, isFinal: boolean) { this.onresult?.({ results: [{ isFinal, length: 1, 0: { transcript } }], @@ -39,6 +49,7 @@ async function installFakeSpeech(page: Page) { } const spoken: string[] = []; + let current: SpeechSynthesisUtterance | null = null; const synth = { speaking: false, pending: false, @@ -46,17 +57,25 @@ async function installFakeSpeech(page: Page) { spoken, speak(utterance: SpeechSynthesisUtterance) { spoken.push(`${utterance.lang}|${utterance.text}`); + current = utterance; this.speaking = true; }, + // Per the spec, cancel() does not clear the paused state. cancel() { this.speaking = false; - this.paused = false; + current = null; }, + // Like Chrome and Safari: `paused` flips only when the engine confirms, + // which is also when the utterance gets its `pause` event. pause() { - this.paused = true; + setTimeout(() => { + this.paused = true; + current?.onpause?.call(current, new Event("pause") as never); + }, 0); }, resume() { this.paused = false; + current?.onresume?.call(current, new Event("resume") as never); }, getVoices: () => [], }; @@ -135,6 +154,54 @@ test.describe("voice input and read-aloud", () => { .toBeGreaterThan(0); }); + test("hold-to-talk through the embed bundle: a real press and release", async ({ + page, + }) => { + await installFakeSpeech(page); + const api = await mockChatApi(page); + api.enqueueReply("Yes, 20 percent."); + // Same approach as embed.spec.ts: the dev origin, minus the dev widget. + await page.goto("/"); + await page.evaluate(() => { + document.getElementById("root")?.remove(); + (window as unknown as { ClaudiusConfig: unknown }).ClaudiusConfig = { + apiUrl: "https://test.example", + voice: { mode: "hold", autoSubmit: true }, + }; + }); + await page.addStyleTag({ path: CSS_PATH }); + await page.addScriptTag({ path: IIFE_PATH }); + await page.getByRole("button", { name: /open chat/i }).click(); + + const mic = page.getByRole("button", { name: "Hold to talk" }); + const box = (await mic.boundingBox())!; + await page.mouse.move(box.x + box.width / 2, box.y + box.height / 2); + await page.mouse.down(); + await expect(mic).toHaveAttribute("aria-pressed", "true"); + await hear(page, "do you offer", true); + + // The engine ends the session at a pause, with the button still down. + // A new session takes over, and nothing is sent yet. + await page.evaluate(() => + ( + window as unknown as { __recognition: { end(): void } } + ).__recognition.end(), + ); + await expect(mic).toHaveAttribute("aria-pressed", "true"); + expect(api.callCount()).toBe(0); + await hear(page, "a discount", true); + + await page.mouse.up(); + + const log = page.getByRole("log"); + await expect(log.getByText("do you offer a discount")).toBeVisible(); + await expect(log.getByText("Yes, 20 percent.")).toBeVisible(); + // Exactly one message: the click Chromium fires after the release carries + // a click count, so it must not have started another dictation. + expect(api.callCount()).toBe(1); + await expect(mic).toHaveAttribute("aria-pressed", "false"); + }); + test("reads a reply aloud with pause and stop controls", async ({ page }) => { await installFakeSpeech(page); const api = await mockChatApi(page); diff --git a/widget/src/components/ChatInput.tsx b/widget/src/components/ChatInput.tsx index 38f2976..7415788 100644 --- a/widget/src/components/ChatInput.tsx +++ b/widget/src/components/ChatInput.tsx @@ -51,6 +51,11 @@ interface ChatInputProps { voice?: ResolvedVoiceConfig | null; /** Called when dictation starts, so the parent can silence read-aloud. */ onVoiceStart?: () => void; + /** + * True while a reply is being read aloud. Dictation ends when it turns + * true, or the microphone would transcribe the widget's own voice. + */ + isReadingAloud?: boolean; } function dragHasFiles(e: DragEvent): boolean { @@ -68,6 +73,7 @@ export function ChatInput({ attachments = null, voice = null, onVoiceStart, + isReadingAloud = false, }: ChatInputProps) { const t = translations ?? defaultTranslations; const [value, setValue] = useState(""); @@ -81,8 +87,15 @@ export function ChatInput({ const dragDepthRef = useRef(0); // Mirrors `value` for the speech callbacks, which fire outside a render. const valueRef = useRef(""); - // What the field held when dictation began; speech is appended to it. + // What the field held when the current session began; speech is appended. const dictationBaseRef = useRef(""); + // Hold mode: true from press to release. One dictation can span several + // engine sessions, because the engine ends a session at the first pause. + const heldRef = useRef(false); + // Whether this dictation has heard anything, and whether it really failed. + // Together they decide if auto-submit may fire when it ends. + const heardRef = useRef(false); + const failedRef = useRef(false); const updateValue = (next: string) => { valueRef.current = next; @@ -167,6 +180,7 @@ export function ChatInput({ pendingRef.current = []; setPending([]); setAttachmentError(null); + setVoiceError(null); }; const describeVoiceError = (kind: SpeechRecognitionErrorKind): string => { @@ -189,10 +203,24 @@ export function ChatInput({ joinDictation(dictationBaseRef.current, transcript, MAX_MESSAGE_LENGTH), ), onEnd: (transcript) => { - // Only a session that ended normally and actually heard something. - if (voice?.autoSubmit && transcript.trim()) submit(); + const heard = transcript.trim() !== ""; + if (heard) heardRef.current = true; + if (!heldRef.current) { + finishDictation(); + return; + } + // Still holding: the engine merely ended at a pause, so open another + // session and leave any auto-submit for the release. A session that + // heard nothing is not replaced, so a broken engine cannot spin. + if (heard) beginSession(); + }, + onError: (kind) => { + // A later session of a hold timing out in silence is not a failure: + // the visitor stopped talking and is simply still holding the button. + if (kind === "no-speech" && heardRef.current) return; + failedRef.current = true; + setVoiceError(describeVoiceError(kind)); }, - onError: (kind) => setVoiceError(describeVoiceError(kind)), }); const showMic = !!voice?.input && recognition.isSupported; const isListening = recognition.isListening; @@ -204,17 +232,54 @@ export function ChatInput({ if (isListening && input) input.scrollLeft = input.scrollWidth; }, [isListening, value]); - const startDictation = () => { + // Auto-submit fires only for a dictation that heard something and did not + // fail. It goes through the send button's own path. + const finishDictation = () => { + if (voice?.autoSubmit && heardRef.current && !failedRef.current) submit(); + }; + + const beginSession = () => { + const base = valueRef.current; + // Refused while the previous session is still winding down. Its results + // are still arriving against the old base, so that must not move. + if (recognition.start()) dictationBaseRef.current = base; + }; + + const startDictation = ({ held }: { held: boolean }) => { setVoiceError(null); - dictationBaseRef.current = valueRef.current; + heldRef.current = held; + heardRef.current = false; + failedRef.current = false; onVoiceStart?.(); - recognition.start(); + beginSession(); }; + const stopDictation = () => { + const wasHeld = heldRef.current; + heldRef.current = false; + // Nothing left to stop means the last session already ended mid-hold; the + // auto-submit it deferred is due now. + if (!recognition.stop() && wasHeld) finishDictation(); + }; + + const abortDictation = () => { + heldRef.current = false; + recognition.abort(); + }; + + // A request going out disables the field, and auto-submit would then clear + // text it could not send. A reply being read aloud would be transcribed. + const { abort } = recognition; + useEffect(() => { + if (!isLoading && !isReadingAloud) return; + heldRef.current = false; + abort(); + }, [isLoading, isReadingAloud, abort]); + const handleSubmit = (e: FormEvent) => { e.preventDefault(); // Otherwise the next result would put the sent text back in the field. - recognition.abort(); + abortDictation(); submit(); }; @@ -222,7 +287,7 @@ export function ChatInput({ const newValue = e.target.value; if (newValue.length <= MAX_MESSAGE_LENGTH) { // Typing takes over from dictation: keep the edit, drop the session. - recognition.abort(); + abortDictation(); setVoiceError(null); updateValue(newValue); } @@ -368,7 +433,7 @@ export function ChatInput({ isListening={isListening} disabled={isLoading} onStart={startDictation} - onStop={recognition.stop} + onStop={stopDictation} label={voice.mode === "hold" ? t.voiceInputHold : t.voiceInput} /> )} diff --git a/widget/src/components/ChatWindow.tsx b/widget/src/components/ChatWindow.tsx index 5c0deba..1b9f5c0 100644 --- a/widget/src/components/ChatWindow.tsx +++ b/widget/src/components/ChatWindow.tsx @@ -288,8 +288,10 @@ export function ChatWindow({ translations={translations} attachments={attachments} voice={voice} - // The mic would otherwise transcribe the widget's own voice. + // The mic would otherwise transcribe the widget's own voice, so each + // of the two ends the other. onVoiceStart={reader.cancel} + isReadingAloud={reader.activeId !== null} /> ); diff --git a/widget/src/components/MessageSpeechControls.tsx b/widget/src/components/MessageSpeechControls.tsx index d283e27..3f83c97 100644 --- a/widget/src/components/MessageSpeechControls.tsx +++ b/widget/src/components/MessageSpeechControls.tsx @@ -1,4 +1,4 @@ -import type { ReactNode } from "react"; +import { forwardRef, useEffect, useRef, type ReactNode } from "react"; /** Read-aloud state of one message. */ export type MessageSpeechState = "idle" | "speaking" | "paused"; @@ -12,17 +12,13 @@ interface MessageSpeechControlsProps { onStop: () => void; } -function IconButton({ - label, - onClick, - children, -}: { - label: string; - onClick: () => void; - children: ReactNode; -}) { +const IconButton = forwardRef< + HTMLButtonElement, + { label: string; onClick: () => void; children: ReactNode } +>(function IconButton({ label, onClick, children }, ref) { return ( ); -} +}); /** * Read-aloud controls for an assistant message: a speaker button while idle, * pause and stop while speaking, resume and stop while paused. + * + * The first button is one element that changes role (read aloud, pause, + * resume), so keyboard focus survives the change. Swapping in a different + * element would drop focus to ``, outside the dialog's focus trap. */ export function MessageSpeechControls({ state, @@ -58,31 +58,53 @@ export function MessageSpeechControls({ onResume, onStop, }: MessageSpeechControlsProps) { - if (state === "idle") { - return ( - - - - - - ); - } + const groupRef = useRef(null); + const primaryRef = useRef(null); + + // Stop disappears when playback ends or is stopped. Read focus now, during + // render: it is the last moment a focused Stop button is still in the DOM. + // If it then vanishes, hand focus to the button that remains. + const hadFocusWithin = !!groupRef.current?.contains(document.activeElement); + useEffect(() => { + if ( + state === "idle" && + hadFocusWithin && + !groupRef.current?.contains(document.activeElement) + ) { + primaryRef.current?.focus(); + } + }, [state, hadFocusWithin]); + + const primary = + state === "idle" + ? { label: labels.play, onClick: onPlay } + : state === "paused" + ? { label: labels.resume, onClick: onResume } + : { label: labels.pause, onClick: onPause }; return ( -
- {state === "paused" ? ( - +
+ + {state === "idle" ? ( + <> + + + + + ) : state === "paused" ? ( - - ) : ( - - - + ) : ( + <> + + + + )} + + {state !== "idle" && ( + + )} - - -
); } diff --git a/widget/src/components/VoiceInputButton.tsx b/widget/src/components/VoiceInputButton.tsx index 15eb660..e42171d 100644 --- a/widget/src/components/VoiceInputButton.tsx +++ b/widget/src/components/VoiceInputButton.tsx @@ -10,7 +10,12 @@ interface VoiceInputButtonProps { mode: VoiceInputMode; isListening: boolean; disabled?: boolean; - onStart: () => void; + /** + * `held` is true only for a real press-and-hold, which is guaranteed to end + * with a release. A toggle start (including the assistive-technology click + * in hold mode) has no such guarantee. + */ + onStart: (gesture: { held: boolean }) => void; onStop: () => void; /** * Accessible name. It stays the same in both states; `aria-pressed` carries @@ -42,7 +47,7 @@ export function VoiceInputButton({ const beginHold = () => { if (disabled || holdingRef.current) return; holdingRef.current = true; - onStart(); + onStart({ held: true }); }; // Idempotent: a release is often reported twice (pointerup, then @@ -59,7 +64,7 @@ export function VoiceInputButton({ // the button, which cannot hold it down. Let those toggle. if (isHold && e.detail !== 0) return; if (isListening) onStop(); - else onStart(); + else onStart({ held: false }); }; const handlePointerDown = (e: PointerEvent) => { @@ -102,7 +107,9 @@ export function VoiceInputButton({ style={isHold ? { WebkitTouchCallout: "none" } : undefined} className={`flex h-11 w-11 shrink-0 items-center justify-center rounded-claudius-sm border transition-colors focus:outline-none focus:ring-1 focus:ring-claudius-accent disabled:opacity-50 ${ isListening - ? "border-claudius-accent bg-claudius-accent text-claudius-accent-text" + ? // An accent ring on an accent fill is invisible; offset it so + // keyboard users can still see where focus is. + "border-claudius-accent bg-claudius-accent text-claudius-accent-text focus:ring-2 focus:ring-offset-2 focus:ring-offset-claudius-surface" : "border-claudius-border bg-claudius-field text-claudius-text-muted hover:text-claudius-text focus:border-claudius-accent" } ${isHold ? "touch-none select-none" : ""}`} > diff --git a/widget/src/components/__tests__/ChatInput.voice.test.tsx b/widget/src/components/__tests__/ChatInput.voice.test.tsx index 573133f..0597f6a 100644 --- a/widget/src/components/__tests__/ChatInput.voice.test.tsx +++ b/widget/src/components/__tests__/ChatInput.voice.test.tsx @@ -33,20 +33,27 @@ const translations = createTranslations({ function setup( voice: ResolvedVoiceConfig | null = VOICE, - props: { isLoading?: boolean } = {}, + props: { isLoading?: boolean; isReadingAloud?: boolean } = {}, ) { const onSend = vi.fn(); const onVoiceStart = vi.fn(); - const view = render( + const ui = (p: typeof props) => ( , + /> ); - return { ...view, onSend, onVoiceStart }; + const view = render(ui(props)); + return { + ...view, + onSend, + onVoiceStart, + update: (next: typeof props) => view.rerender(ui(next)), + }; } const mic = () => screen.getByRole("button", { name: "MIC" }); @@ -342,4 +349,178 @@ describe("ChatInput voice input", () => { expect(screen.queryByRole("alert")).not.toBeInTheDocument(); }); }); + + describe("hold mode lasts as long as the hold", () => { + const HOLD = { ...VOICE, mode: "hold" as const }; + const holdButton = () => screen.getByRole("button", { name: "MIC-HOLD" }); + + function press() { + fireEvent.pointerDown(holdButton(), { button: 0 }); + } + function release() { + fireEvent.pointerUp(holdButton()); + } + function sessions() { + return FakeSpeechRecognition.instances; + } + + it("keeps listening when the engine ends a session at a pause in speech", () => { + setup(HOLD); + press(); + act(() => latestRecognition().emitStart()); + hear("I would like to know", true); + + // The engine finalizes and ends at the pause; the button is still down. + act(() => latestRecognition().emitEnd()); + + expect(sessions()).toHaveLength(2); + expect(holdButton()).toHaveAttribute("aria-pressed", "true"); + hear("your prices", true); + expect(field()).toHaveValue("I would like to know your prices"); + }); + + it("does not restart a session that heard nothing, so a dead engine cannot spin", () => { + setup(HOLD); + press(); + + act(() => latestRecognition().emitEnd()); + + expect(sessions()).toHaveLength(1); + expect(holdButton()).toHaveAttribute("aria-pressed", "false"); + }); + + it("does not keep a toggle start alive, since nothing guarantees it will be released", () => { + setup(HOLD); + // A click with no click count: a screen reader activating the button. + fireEvent.click(holdButton()); + hear("hello", true); + + act(() => latestRecognition().emitEnd()); + + expect(sessions()).toHaveLength(1); + }); + + it("does not duplicate the text when pressed again while the last session winds down", () => { + setup(HOLD); + press(); + const first = latestRecognition(); + act(() => first.emitStart()); + hear("hello world", false); + release(); + + // Pressed again before the engine has delivered its final result. + press(); + act(() => + first.emitResult([{ transcript: "hello world", isFinal: true }]), + ); + expect(field()).toHaveValue("hello world"); + + // The old session ends with the button down: the new hold takes over. + act(() => first.emitEnd()); + expect(sessions()).toHaveLength(2); + hear("again", true); + expect(field()).toHaveValue("hello world again"); + }); + + describe("with auto-submit", () => { + const HOLD_AUTO = { ...HOLD, autoSubmit: true }; + + it("waits for the release instead of sending at the first pause", () => { + const { onSend } = setup(HOLD_AUTO); + press(); + act(() => latestRecognition().emitStart()); + hear("I would like to know", true); + act(() => latestRecognition().emitEnd()); + expect(onSend).not.toHaveBeenCalled(); + + act(() => latestRecognition().emitStart()); + hear("your prices", true); + release(); + act(() => latestRecognition().emitEnd()); + + expect(onSend).toHaveBeenCalledExactlyOnceWith( + "I would like to know your prices", + ); + }); + + it("sends on release even if the last session timed out in silence, without an error", () => { + const { onSend } = setup(HOLD_AUTO); + press(); + act(() => latestRecognition().emitStart()); + hear("hello", true); + act(() => latestRecognition().emitEnd()); + + // The visitor goes quiet, still holding; the restarted session times out. + const second = latestRecognition(); + act(() => second.emitError("no-speech")); + act(() => second.emitEnd()); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + expect(onSend).not.toHaveBeenCalled(); + + release(); + + expect(onSend).toHaveBeenCalledExactlyOnceWith("hello"); + }); + + it("does not send on release after a real failure mid-hold", () => { + const { onSend } = setup(HOLD_AUTO); + press(); + act(() => latestRecognition().emitStart()); + hear("hello", true); + act(() => latestRecognition().emitEnd()); + + act(() => latestRecognition().emitError("network")); + expect(screen.getByRole("alert")).toHaveTextContent("ERR-UNAVAILABLE"); + + release(); + + expect(onSend).not.toHaveBeenCalled(); + expect(field()).toHaveValue("hello"); + }); + }); + }); + + describe("things that must end a dictation", () => { + const AUTO = { ...VOICE, autoSubmit: true }; + + it("a reply starting to be read aloud, or the mic would transcribe it", () => { + const { onSend, update } = setup(AUTO); + fireEvent.click(mic()); + const session = latestRecognition(); + hear("hello", false); + + update({ isReadingAloud: true }); + + expect(session.abortCalls).toBe(1); + expect(mic()).toHaveAttribute("aria-pressed", "false"); + expect(field()).toHaveValue("hello"); + expect(onSend).not.toHaveBeenCalled(); + }); + + it("a request starting to load, or auto-submit would discard the text", () => { + const { onSend, update } = setup(AUTO); + fireEvent.click(mic()); + const session = latestRecognition(); + hear("hello", true); + + update({ isLoading: true }); + + expect(session.abortCalls).toBe(1); + expect(field()).toHaveValue("hello"); + expect(onSend).not.toHaveBeenCalled(); + }); + }); + + it("clears a voice error once a message has been sent", async () => { + const user = userEvent.setup(); + setup(); + fireEvent.click(mic()); + hear("hello", true); + act(() => latestRecognition().emitError("network")); + expect(screen.getByRole("alert")).toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: /send message/i })); + + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + }); }); diff --git a/widget/src/components/__tests__/ChatWindow.voice.test.tsx b/widget/src/components/__tests__/ChatWindow.voice.test.tsx index 4cf57b8..6ca0571 100644 --- a/widget/src/components/__tests__/ChatWindow.voice.test.tsx +++ b/widget/src/components/__tests__/ChatWindow.voice.test.tsx @@ -11,6 +11,7 @@ import { import { installFakeSpeechRecognition, uninstallFakeSpeechRecognition, + latestRecognition, } from "../../test-utils/fakeSpeechRecognition"; const VOICE: ResolvedVoiceConfig = { @@ -151,7 +152,8 @@ describe("ChatWindow read-aloud", () => { fireEvent.click(screen.getByRole("button", { name: "Read aloud" })); fireEvent.click(screen.getByRole("button", { name: "Pause reading" })); - expect(synth.paused).toBe(true); + // Chrome and Safari confirm a pause asynchronously. + act(() => synth.confirmPause()); fireEvent.click(screen.getByRole("button", { name: "Resume reading" })); expect(synth.paused).toBe(false); @@ -186,6 +188,20 @@ describe("ChatWindow read-aloud", () => { ).toHaveLength(1); }); + it("ends dictation when a reply starts being read, so the mic does not transcribe it", () => { + renderWindow([QUESTION, REPLY]); + const mic = screen.getByRole("button", { name: "Voice input" }); + fireEvent.click(mic); + const session = latestRecognition(); + expect(mic).toHaveAttribute("aria-pressed", "true"); + + fireEvent.click(screen.getByRole("button", { name: "Read aloud" })); + + expect(session.abortCalls).toBe(1); + expect(mic).toHaveAttribute("aria-pressed", "false"); + expect(synth.queuedText()).toEqual(["Plans start at $10."]); + }); + it("falls silent when the visitor starts dictating, so the mic does not hear it", () => { renderWindow([QUESTION, REPLY]); fireEvent.click(screen.getByRole("button", { name: "Read aloud" })); diff --git a/widget/src/components/__tests__/MessageSpeechControls.test.tsx b/widget/src/components/__tests__/MessageSpeechControls.test.tsx index 4ad7188..42b1219 100644 --- a/widget/src/components/__tests__/MessageSpeechControls.test.tsx +++ b/widget/src/components/__tests__/MessageSpeechControls.test.tsx @@ -1,4 +1,6 @@ +import { useState } from "react"; import { render, screen, fireEvent } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import { describe, it, expect, vi } from "vitest"; import { MessageSpeechControls, @@ -58,4 +60,73 @@ describe("MessageSpeechControls", () => { fireEvent.click(screen.getByRole("button", { name: "Stop reading" })); expect(onStop).toHaveBeenCalledTimes(1); }); + + // A removed element drops focus to , which is outside the chat + // dialog's focus trap, so Tab could then leave the dialog. + describe("keyboard focus", () => { + function Harness({ initial = "idle" as MessageSpeechState }) { + const [state, setState] = useState(initial); + return ( + <> + setState("speaking")} + onPause={() => setState("paused")} + onResume={() => setState("speaking")} + onStop={() => setState("idle")} + /> + + + ); + } + + it("stays on the control as Read aloud becomes Pause, then Resume", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByRole("button", { name: "Read aloud" })); + expect( + screen.getByRole("button", { name: "Pause reading" }), + ).toHaveFocus(); + + await user.click(screen.getByRole("button", { name: "Pause reading" })); + expect( + screen.getByRole("button", { name: "Resume reading" }), + ).toHaveFocus(); + }); + + it("moves to Read aloud when Stop is pressed and disappears", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByRole("button", { name: "Stop reading" })); + + expect(screen.getByRole("button", { name: "Read aloud" })).toHaveFocus(); + }); + + it("moves to Read aloud when playback ends by itself with focus on Stop", () => { + render(); + screen.getByRole("button", { name: "Stop reading" }).focus(); + + // fireEvent does not move focus, unlike a real click on this button. + fireEvent.click(screen.getByRole("button", { name: "playback ends" })); + + expect(screen.getByRole("button", { name: "Read aloud" })).toHaveFocus(); + }); + + it("does not steal focus when playback ends while the visitor is elsewhere", async () => { + const user = userEvent.setup(); + render(); + await user.click(screen.getByRole("button", { name: "Stop reading" })); + await user.click(screen.getByRole("button", { name: "Read aloud" })); + + // Focus leaves the controls for another element, then playback ends. + await user.click(screen.getByRole("button", { name: "playback ends" })); + + expect( + screen.getByRole("button", { name: "playback ends" }), + ).toHaveFocus(); + }); + }); }); diff --git a/widget/src/components/__tests__/VoiceInputButton.test.tsx b/widget/src/components/__tests__/VoiceInputButton.test.tsx index ae6683d..0acf0a2 100644 --- a/widget/src/components/__tests__/VoiceInputButton.test.tsx +++ b/widget/src/components/__tests__/VoiceInputButton.test.tsx @@ -44,7 +44,7 @@ describe("VoiceInputButton", () => { it("starts on a click when idle", () => { const { button, onStart, onStop } = setup(); fireEvent.click(button); - expect(onStart).toHaveBeenCalledTimes(1); + expect(onStart).toHaveBeenCalledExactlyOnceWith({ held: false }); expect(onStop).not.toHaveBeenCalled(); }); @@ -67,7 +67,7 @@ describe("VoiceInputButton", () => { const { button, onStart, onStop } = setup({ mode: "hold" }); fireEvent.pointerDown(button, { button: 0 }); - expect(onStart).toHaveBeenCalledTimes(1); + expect(onStart).toHaveBeenCalledExactlyOnceWith({ held: true }); expect(onStop).not.toHaveBeenCalled(); fireEvent.pointerUp(button); @@ -105,7 +105,7 @@ describe("VoiceInputButton", () => { fireEvent.keyDown(button, { key }); fireEvent.keyDown(button, { key, repeat: true }); - expect(onStart).toHaveBeenCalledTimes(1); + expect(onStart).toHaveBeenCalledExactlyOnceWith({ held: true }); fireEvent.keyUp(button, { key }); expect(onStop).toHaveBeenCalledTimes(1); @@ -128,7 +128,9 @@ describe("VoiceInputButton", () => { it("toggles on a bare click, which is how screen readers and switches activate it", () => { const { button, onStart } = setup({ mode: "hold" }); fireEvent.click(button); - expect(onStart).toHaveBeenCalledTimes(1); + // Not a hold: nothing guarantees a release, so the parent must not keep + // the microphone open through pauses the way it does for a real hold. + expect(onStart).toHaveBeenCalledExactlyOnceWith({ held: false }); }); it("stops on a bare click while listening", () => { diff --git a/widget/src/hooks/__tests__/useSpeechRecognition.test.ts b/widget/src/hooks/__tests__/useSpeechRecognition.test.ts index 584bbb3..20ad235 100644 --- a/widget/src/hooks/__tests__/useSpeechRecognition.test.ts +++ b/widget/src/hooks/__tests__/useSpeechRecognition.test.ts @@ -322,4 +322,95 @@ describe("useSpeechRecognition", () => { expect(onEnd).not.toHaveBeenCalled(); }); }); + + describe("telling the caller what happened", () => { + it("start() reports whether a session actually began", () => { + const { result } = setup(); + let first = false; + let second = true; + + act(() => { + first = result.current.start(); + }); + act(() => { + second = result.current.start(); + }); + + expect(first).toBe(true); + // Refused: the first session is still open. + expect(second).toBe(false); + }); + + it("start() reports false when the engine refuses", () => { + FakeSpeechRecognition.startError = new Error("InvalidStateError"); + const { result } = setup(); + let started = true; + + act(() => { + started = result.current.start(); + }); + + expect(started).toBe(false); + }); + + it("stop() reports whether there was a session to stop", () => { + const { result } = setup(); + let idle = true; + let open = false; + + act(() => { + idle = result.current.stop(); + }); + act(() => result.current.start()); + act(() => latestRecognition().emitStart()); + act(() => { + open = result.current.stop(); + }); + + expect(idle).toBe(false); + expect(open).toBe(true); + }); + }); + + describe("engines that misbehave", () => { + it("treats an abort by the engine (another tab took the mic) as a failed session, silently", () => { + const { result, onEnd, onError } = setup(); + act(() => result.current.start()); + const session = latestRecognition(); + act(() => + session.emitResult([{ transcript: "half a thought", isFinal: true }]), + ); + + act(() => session.emitError("aborted")); + act(() => session.emitEnd()); + + expect(onError).not.toHaveBeenCalled(); + // Not a normal end, so nothing may be auto-submitted from it. + expect(onEnd).not.toHaveBeenCalled(); + expect(result.current.isListening).toBe(false); + }); + + it("stops listening on an error even if the engine never sends end", () => { + const { result } = setup(); + act(() => result.current.start()); + const session = latestRecognition(); + + act(() => session.emitError("network")); + + expect(result.current.isListening).toBe(false); + expect(session.abortCalls).toBe(1); + }); + + it("still stops, rather than aborts, an engine that delivers results without ever firing start", () => { + const { result } = setup(); + act(() => result.current.start()); + const session = latestRecognition(); + act(() => session.emitResult([{ transcript: "hi", isFinal: false }])); + + act(() => result.current.stop()); + + expect(session.stopCalls).toBe(1); + expect(session.abortCalls).toBe(0); + }); + }); }); diff --git a/widget/src/hooks/__tests__/useSpeechSynthesis.test.ts b/widget/src/hooks/__tests__/useSpeechSynthesis.test.ts index e17e0be..bbe7a9d 100644 --- a/widget/src/hooks/__tests__/useSpeechSynthesis.test.ts +++ b/widget/src/hooks/__tests__/useSpeechSynthesis.test.ts @@ -3,16 +3,17 @@ import { describe, it, expect, beforeEach, afterEach } from "vitest"; import { useSpeechSynthesis } from "../useSpeechSynthesis"; import { FakeSpeechSynthesis, + FakeSpeechSynthesisUtterance, installFakeSpeechSynthesis, uninstallFakeSpeechSynthesis, } from "../../test-utils/fakeSpeechSynthesis"; -// Forty characters, so the default 200-character chunking is easy to reason -// about by hand: four sentences (163 chars) fit a chunk, five (204) do not. +// Forty characters, so the default 150-character chunking is easy to reason +// about by hand: three sentences (122 chars) fit a chunk, four (163) do not. const SENTENCE = "Alpha beta gamma delta epsilon zeta eta."; const LONG_REPLY = Array(6).fill(SENTENCE).join(" "); -const FIRST_CHUNK = Array(4).fill(SENTENCE).join(" "); -const SECOND_CHUNK = Array(2).fill(SENTENCE).join(" "); +const FIRST_CHUNK = Array(3).fill(SENTENCE).join(" "); +const SECOND_CHUNK = Array(3).fill(SENTENCE).join(" "); describe("useSpeechSynthesis", () => { let synth: FakeSpeechSynthesis; @@ -72,19 +73,50 @@ describe("useSpeechSynthesis", () => { expect(result.current.activeId).toBeNull(); }); - it("pauses and resumes without losing its place", () => { + it("shows paused only once the engine confirms it, as Chrome and Safari do", () => { const { result } = renderHook(() => useSpeechSynthesis("en-US")); act(() => result.current.speak("m1", LONG_REPLY)); act(() => result.current.pause()); - expect(synth.paused).toBe(true); + expect(result.current.isPaused).toBe(false); + + act(() => synth.confirmPause()); expect(result.current.isPaused).toBe(true); expect(result.current.activeId).toBe("m1"); + }); + + it("shows paused at once where the engine pauses synchronously, as Firefox does", () => { + synth.pauseBehavior = "sync"; + const { result } = renderHook(() => useSpeechSynthesis("en-US")); + act(() => result.current.speak("m1", LONG_REPLY)); + + act(() => result.current.pause()); + + expect(result.current.isPaused).toBe(true); + }); + + it("resumes the same reply", () => { + const { result } = renderHook(() => useSpeechSynthesis("en-US")); + act(() => result.current.speak("m1", LONG_REPLY)); + act(() => result.current.pause()); + act(() => synth.confirmPause()); act(() => result.current.resume()); + expect(synth.paused).toBe(false); expect(result.current.isPaused).toBe(false); - expect(synth.queuedText()).toEqual([FIRST_CHUNK, SECOND_CHUNK]); + expect(result.current.activeId).toBe("m1"); + }); + + it("ignores a pause event from a reply it has already replaced", () => { + const { result } = renderHook(() => useSpeechSynthesis("en-US")); + act(() => result.current.speak("m1", "First reply.")); + const stale = synth.queue[0]; + act(() => result.current.speak("m2", "Second reply.")); + + act(() => stale.onpause?.({})); + + expect(result.current.isPaused).toBe(false); }); it("stops outright where pause() ends the utterance (Android), instead of reading on", () => { @@ -100,10 +132,23 @@ describe("useSpeechSynthesis", () => { expect(result.current.isPaused).toBe(false); }); - it("cancel() empties the queue and returns to idle, even while paused", () => { + it("stops outright where pause() fails the utterance instead (Android), rather than reading on", () => { + synth.pauseBehavior = "error-utterance"; const { result } = renderHook(() => useSpeechSynthesis("en-US")); act(() => result.current.speak("m1", LONG_REPLY)); + expect(synth.queue).toHaveLength(2); + act(() => result.current.pause()); + + expect(synth.queue).toHaveLength(0); + expect(result.current.activeId).toBeNull(); + }); + + it("stops while paused without leaving the engine paused, or the next reply would queue in silence", () => { + const { result } = renderHook(() => useSpeechSynthesis("en-US")); + act(() => result.current.speak("m1", LONG_REPLY)); + act(() => result.current.pause()); + act(() => synth.confirmPause()); expect(result.current.isPaused).toBe(true); act(() => result.current.cancel()); @@ -111,6 +156,18 @@ describe("useSpeechSynthesis", () => { expect(synth.queue).toHaveLength(0); expect(result.current.activeId).toBeNull(); expect(result.current.isPaused).toBe(false); + // cancel() alone does not clear the engine's paused state (per the spec). + expect(synth.paused).toBe(false); + }); + + it("un-pauses an engine that was left paused before it reads", () => { + synth.paused = true; + const { result } = renderHook(() => useSpeechSynthesis("en-US")); + + act(() => result.current.speak("m1", "Hello.")); + + expect(synth.paused).toBe(false); + expect(synth.queuedText()).toEqual(["Hello."]); }); it("replaces the reply being read when another one is started", () => { @@ -154,4 +211,44 @@ describe("useSpeechSynthesis", () => { expect(synth.queue).toHaveLength(0); }); + + describe("speech it does not own", () => { + function hostPageSpeaks() { + synth.speak(new FakeSpeechSynthesisUtterance("Host page narration.")); + } + + it("is left alone on unmount by a reader that never spoke, as when voice is switched off", () => { + hostPageSpeaks(); + const { unmount } = renderHook(() => useSpeechSynthesis("en-US")); + + unmount(); + + expect(synth.cancelCalls).toBe(0); + expect(synth.queuedText()).toEqual(["Host page narration."]); + }); + + it("is left alone by cancel() when this reader has nothing playing", () => { + hostPageSpeaks(); + const { result } = renderHook(() => useSpeechSynthesis("en-US")); + + act(() => result.current.cancel()); + + expect(synth.cancelCalls).toBe(0); + expect(synth.queuedText()).toEqual(["Host page narration."]); + }); + + it("survives a second widget taking over: the first resets without cancelling the engine again", () => { + const first = renderHook(() => useSpeechSynthesis("en-US")); + const second = renderHook(() => useSpeechSynthesis("en-US")); + act(() => first.result.current.speak("m1", LONG_REPLY)); + act(() => second.result.current.speak("m2", "Second reply.")); + + // The first widget's utterances now get their late "interrupted" errors. + act(() => synth.flushCancelled()); + + expect(first.result.current.activeId).toBeNull(); + expect(second.result.current.activeId).toBe("m2"); + expect(synth.queuedText()).toEqual(["Second reply."]); + }); + }); }); diff --git a/widget/src/hooks/useSpeechRecognition.ts b/widget/src/hooks/useSpeechRecognition.ts index f1a6ded..c59a46e 100644 --- a/widget/src/hooks/useSpeechRecognition.ts +++ b/widget/src/hooks/useSpeechRecognition.ts @@ -24,9 +24,17 @@ export interface UseSpeechRecognitionResult { isListening: boolean; /** True while the engine reports hearing speech. Drives the level indicator. */ isSpeechDetected: boolean; - start: () => void; - /** Finish the session and keep what was heard. */ - stop: () => void; + /** + * Open a session. Returns whether one actually began: `false` when the + * browser is unsupported, the engine refused, or a previous session is still + * winding down. + */ + start: () => boolean; + /** + * Finish the session and keep what was heard. Returns whether there was a + * session to stop. + */ + stop: () => boolean; /** Drop the session; nothing more is reported for it. */ abort: () => void; } @@ -126,19 +134,17 @@ export function useSpeechRecognition( const stop = useCallback(() => { const session = sessionRef.current; - if (!session) return; + if (!session) return false; // stop() on an engine still waiting for microphone permission would let // a later "Allow" open the mic with nobody at the button. Abort instead. - if (!engineStartedRef.current) { - abort(); - return; - } - session.stop(); + if (!engineStartedRef.current) abort(); + else session.stop(); + return true; }, [abort]); const start = useCallback(() => { const Recognition = getSpeechRecognition(); - if (!Recognition || sessionRef.current) return; + if (!Recognition || sessionRef.current) return false; const session = new Recognition(); session.lang = optionsRef.current.lang; @@ -146,12 +152,16 @@ export function useSpeechRecognition( session.continuous = false; let finalTranscript = ""; - let failed = false; - session.onstart = () => { + // Some engines skip `start`; hearing anything proves the mic is open too. + const markStarted = () => { engineStartedRef.current = true; }; - session.onspeechstart = () => setIsSpeechDetected(true); + session.onstart = markStarted; + session.onspeechstart = () => { + markStarted(); + setIsSpeechDetected(true); + }; session.onspeechend = () => setIsSpeechDetected(false); session.onresult = (event) => { let finals = ""; @@ -162,18 +172,26 @@ export function useSpeechRecognition( else interim += result[0].transcript; } finalTranscript = finals; + markStarted(); setIsSpeechDetected(true); optionsRef.current.onTranscript?.(finals + interim); }; session.onerror = (event) => { - // Chrome reports our own abort() this way; it is not a failure. - if (event.error === "aborted") return; - failed = true; - optionsRef.current.onError?.(toErrorKind(event.error)); + // Our own abort() detaches these handlers first, so an "aborted" that + // arrives here came from the engine: another tab or widget took the + // microphone. Nothing to tell the visitor, but it was not a normal end, + // so onEnd (and any auto-submit hanging off it) must not run. + if (event.error !== "aborted") { + optionsRef.current.onError?.(toErrorKind(event.error)); + } + // The spec promises an `end` after every error. Do not depend on it: + // an engine that forgets would leave the UI listening forever. + release(session); + session.abort(); }; session.onend = () => { release(session); - if (!failed) optionsRef.current.onEnd?.(finalTranscript); + optionsRef.current.onEnd?.(finalTranscript); }; sessionRef.current = session; @@ -185,7 +203,9 @@ export function useSpeechRecognition( } catch { release(session); optionsRef.current.onError?.("unavailable"); + return false; } + return true; }, [release]); // Leaving the page or closing the chat must never leave the mic open. diff --git a/widget/src/hooks/useSpeechSynthesis.ts b/widget/src/hooks/useSpeechSynthesis.ts index 8adb84a..e680a89 100644 --- a/widget/src/hooks/useSpeechSynthesis.ts +++ b/widget/src/hooks/useSpeechSynthesis.ts @@ -49,9 +49,19 @@ export function useSpeechSynthesis(lang: string): UseSpeechSynthesisResult { setIsPaused(false); }, []); + // `speechSynthesis` is one global, shared with the host page and with any + // other widget on it. Only ever cancel speech this reader started: a reader + // that never spoke (voice switched off, say) must not silence the page when + // the chat closes. const cancel = useCallback(() => { + const ownsPlayback = utterancesRef.current.length > 0; reset(); - getSpeechSynthesis()?.cancel(); + const synth = getSpeechSynthesis(); + if (!ownsPlayback || !synth) return; + synth.cancel(); + // Per the spec cancel() leaves a paused engine paused, and the next reply + // would then queue in silence. + if (synth.paused) synth.resume(); }, [reset]); const speak = useCallback( @@ -63,6 +73,7 @@ export function useSpeechSynthesis(lang: string): UseSpeechSynthesisResult { // an earlier cutoff. An idle engine is left alone: in some browsers a // cancel() immediately before speak() swallows the new utterance. if (synth.speaking || synth.pending) synth.cancel(); + if (synth.paused) synth.resume(); const chunks = chunkSpeechText(text); if (chunks.length === 0) return; @@ -73,8 +84,23 @@ export function useSpeechSynthesis(lang: string): UseSpeechSynthesisResult { utterancesRef.current = chunks.map((chunk, index) => { const utterance = new SpeechSynthesisUtterance(chunk); utterance.lang = lang; - utterance.onerror = () => { - if (isCurrent()) cancel(); + utterance.onerror = (event) => { + if (!isCurrent()) return; + // "interrupted" / "canceled" with no pause of ours pending means + // someone else cancelled the engine: another widget, or the host + // page. The queue is theirs now, so forget ours and leave the engine + // alone. Cancelling again here would kill their speech. + const takenOver = + (event.error === "interrupted" || event.error === "canceled") && + !pauseRequestedRef.current; + if (takenOver) reset(); + else cancel(); + }; + utterance.onpause = () => { + if (isCurrent()) setIsPaused(true); + }; + utterance.onresume = () => { + if (isCurrent()) setIsPaused(false); }; utterance.onend = () => { if (!isCurrent()) return; @@ -97,7 +123,10 @@ export function useSpeechSynthesis(lang: string): UseSpeechSynthesisResult { if (!synth) return; pauseRequestedRef.current = true; synth.pause(); - setIsPaused(synth.paused); + // Firefox flips `paused` synchronously. Chrome and Safari flip it only + // once the engine confirms, which arrives as the utterance's `pause` + // event, handled above. + if (synth.paused) setIsPaused(true); }, []); const resume = useCallback(() => { diff --git a/widget/src/test-utils/fakeSpeechSynthesis.ts b/widget/src/test-utils/fakeSpeechSynthesis.ts index 6e85a39..530134a 100644 --- a/widget/src/test-utils/fakeSpeechSynthesis.ts +++ b/widget/src/test-utils/fakeSpeechSynthesis.ts @@ -37,10 +37,17 @@ export class FakeSpeechSynthesis { queue: FakeSpeechSynthesisUtterance[] = []; cancelCalls = 0; /** - * `"end-utterance"` reproduces Android, where `pause()` ends the current - * utterance instead of pausing it and never sets `paused`. + * How `pause()` behaves, which differs by browser: + * - `"async"` (the default; Chrome and Safari): nothing changes until the + * engine confirms. The test plays the engine with `confirmPause()`, which + * sets `paused` and fires the utterance's `pause` event. + * - `"sync"` (Firefox): `paused` is set immediately. + * - `"end-utterance"` / `"error-utterance"` (Android): `pause()` ends the + * current utterance instead, by `end` or by `error`, and never sets + * `paused`. */ - pauseBehavior: "pause" | "end-utterance" = "pause"; + pauseBehavior: "async" | "sync" | "end-utterance" | "error-utterance" = + "async"; private cancelled: FakeSpeechSynthesisUtterance[] = []; @@ -49,19 +56,32 @@ export class FakeSpeechSynthesis { this.sync(); } + /** Per the spec, cancelling does not change the paused state. */ cancel(): void { this.cancelCalls += 1; this.cancelled.push(...this.queue); this.queue = []; - this.paused = false; this.sync(); } pause(): void { - if (this.pauseBehavior === "end-utterance") { - this.finishCurrent(); - return; + switch (this.pauseBehavior) { + case "end-utterance": + this.finishCurrent(); + return; + case "error-utterance": + this.failCurrent("interrupted"); + return; + case "sync": + this.confirmPause(); + return; + case "async": + return; } + } + + /** The engine reports that the current utterance is now paused. */ + confirmPause(): void { this.paused = true; this.queue[0]?.onpause?.({}); } diff --git a/widget/src/utils/__tests__/stripAnnouncementFormatting.test.ts b/widget/src/utils/__tests__/stripAnnouncementFormatting.test.ts index 6058a5d..6c7f587 100644 --- a/widget/src/utils/__tests__/stripAnnouncementFormatting.test.ts +++ b/widget/src/utils/__tests__/stripAnnouncementFormatting.test.ts @@ -24,6 +24,18 @@ describe("stripAnnouncementFormatting", () => { ), ).toBe("Read pmds.info for more."); }); + it("keeps the sentence end that follows a URL", () => { + expect( + stripAnnouncementFormatting( + "See https://example.com/pricing. Then call us.", + ), + ).toBe("See example.com. Then call us."); + }); + it("keeps other punctuation that trails a URL", () => { + expect( + stripAnnouncementFormatting("Is it https://example.com/faq?! Yes."), + ).toBe("Is it example.com?! Yes."); + }); it("leaves plain text unchanged", () => { expect(stripAnnouncementFormatting("Hello there!")).toBe("Hello there!"); }); diff --git a/widget/src/utils/__tests__/voice.test.ts b/widget/src/utils/__tests__/voice.test.ts index 3e437ca..6c74c26 100644 --- a/widget/src/utils/__tests__/voice.test.ts +++ b/widget/src/utils/__tests__/voice.test.ts @@ -108,6 +108,15 @@ describe("resolveVoiceConfig", () => { ).toBeNull(); }); + it("stays off for anything that is not true or an options object", () => { + // A microphone feature must fail closed. `voice: "false"` from a CMS + // template is a truthy string, and must not switch the mic on. + expect(resolveVoiceConfig("false" as never, "en")).toBeNull(); + expect(resolveVoiceConfig("true" as never, "en")).toBeNull(); + expect(resolveVoiceConfig(1 as never, "en")).toBeNull(); + expect(resolveVoiceConfig([] as never, "en")).toBeNull(); + }); + it("falls back to toggle mode for an unknown mode", () => { expect(resolveVoiceConfig({ mode: "shout" as never }, "en")).toMatchObject({ mode: "toggle", @@ -177,8 +186,26 @@ describe("chunkSpeechText", () => { ]); }); - it("never splits inside a word, even one longer than the limit", () => { - expect(chunkSpeechText("abcdefghij", 5)).toEqual(["abcdefghij"]); + it("splits a token longer than the limit as a last resort", () => { + // Unbroken runs this long are not words in spaced scripts, but they are + // ordinary sentences in Chinese or Japanese written without punctuation. + expect(chunkSpeechText("abcdefghijkl", 5)).toEqual([ + "abcde", + "fghij", + "kl", + ]); + }); + + it("lets text after an over-long token share its last piece", () => { + expect(chunkSpeechText("abcdefg hi", 5)).toEqual(["abcde", "fg hi"]); + }); + + it("breaks Chinese and Japanese text at its own sentence marks, which have no space after them", () => { + expect(chunkSpeechText("こんにちは。元気ですか？はい！", 8)).toEqual([ + "こんにちは。", + "元気ですか？", + "はい！", + ]); }); it("does not treat a dot inside a word as a sentence end", () => { @@ -199,15 +226,17 @@ describe("chunkSpeechText", () => { ]); }); - it("keeps every chunk short enough to dodge Chrome's long-utterance cutoff", () => { + it("keeps every utterance to roughly ten seconds of speech, well inside Chrome's cutoff", () => { const sentence = "Alpha beta gamma delta epsilon zeta eta."; const text = Array(6).fill(sentence).join(" "); const chunks = chunkSpeechText(text); - expect(chunks.length).toBeGreaterThan(1); - for (const chunk of chunks) expect(chunk.length).toBeLessThanOrEqual(200); - expect(chunks.join(" ")).toBe(text); + // Forty-character sentences: three fit in 150 characters, four do not. + expect(chunks).toEqual([ + Array(3).fill(sentence).join(" "), + Array(3).fill(sentence).join(" "), + ]); }); }); diff --git a/widget/src/utils/stripAnnouncementFormatting.ts b/widget/src/utils/stripAnnouncementFormatting.ts index a24c09b..cec98df 100644 --- a/widget/src/utils/stripAnnouncementFormatting.ts +++ b/widget/src/utils/stripAnnouncementFormatting.ts @@ -12,8 +12,12 @@ export function stripAnnouncementFormatting(content: string): string { .replace(BOLD, "$1") .replace(ITALIC, "$1") .replace(URL_PATTERN, (match) => { + // Punctuation that ends the sentence is not part of the link. Keep it, + // or "see example.com/pricing. Then..." loses its sentence break. + const trailing = match.match(/[.,;:!?'"]+$/)?.[0] ?? ""; + const url = trailing ? match.slice(0, -trailing.length) : match; try { - return new URL(match).hostname; + return new URL(url).hostname + trailing; } catch { return match; } diff --git a/widget/src/utils/voice.ts b/widget/src/utils/voice.ts index 1b77c2a..0aaebb3 100644 --- a/widget/src/utils/voice.ts +++ b/widget/src/utils/voice.ts @@ -97,7 +97,12 @@ export function resolveVoiceConfig( input: boolean | VoiceOptions | undefined | null, locale: LocaleCode, ): ResolvedVoiceConfig | null { - if (!input) return null; + // Fail closed: this switches a microphone on. Only `true` or an options + // object enables it, so a stray truthy value such as the string "false" + // from a template cannot. + const isOptions = + typeof input === "object" && input !== null && !Array.isArray(input); + if (input !== true && !isOptions) return null; const options = input === true ? {} : input; const config: ResolvedVoiceConfig = { input: options.input ?? true, @@ -162,7 +167,11 @@ export function joinDictation( return (typed ? `${typed} ${heard}` : heard).slice(0, maxLength); } -/** Split after sentence punctuation that is followed by whitespace, and at newlines. */ +/** + * Split at newlines, after `.` `!` `?` when whitespace follows (so the dot in + * "example.com" or "$9.99" is not a break), and after the full-width marks + * Chinese and Japanese use, which have no space after them. + */ function splitSentences(text: string): string[] { const pieces: string[] = []; let start = 0; @@ -170,6 +179,9 @@ function splitSentences(text: string): string[] { const ch = text[i]; const endsSentence = ch === "\n" || + ch === "。" || + ch === "！" || + ch === "？" || ((ch === "." || ch === "!" || ch === "?") && (i + 1 === text.length || /\s/.test(text[i + 1]))); if (endsSentence) { @@ -185,10 +197,12 @@ function splitSentences(text: string): string[] { * Break text into utterance-sized chunks for `speechSynthesis`. Chrome stops * a long utterance after roughly 15 seconds when a network voice is in use, * so text is queued as several short utterances instead: whole sentences - * packed up to `maxLength`, an over-long sentence split between words. Words - * are never split. + * packed up to `maxLength`, an over-long sentence split between words, and a + * single over-long token cut as a last resort. The default is about ten + * seconds of speech, leaving room for numbers, which take far longer to say + * than to write. */ -export function chunkSpeechText(text: string, maxLength = 200): string[] { +export function chunkSpeechText(text: string, maxLength = 150): string[] { const chunks: string[] = []; let current = ""; @@ -208,8 +222,14 @@ export function chunkSpeechText(text: string, maxLength = 200): string[] { current = piece; continue; } - for (const word of piece.trim().split(/\s+/)) { + for (let word of piece.trim().split(/\s+/)) { if (current && `${current} ${word}`.length > maxLength) flush(); + // Not a word in a spaced script, but an ordinary unpunctuated sentence + // in Chinese or Japanese. Left whole it would hit the cutoff. + while (word.length > maxLength) { + chunks.push(word.slice(0, maxLength)); + word = word.slice(maxLength); + } current = current ? `${current} ${word}` : word; } } From da7765a9d483c0579dc0fbb9c8f2a8c0eba29802 Mon Sep 17 00:00:00 2001 From: PMullz Date: Fri, 18 Sep 2026 22:47:10 -0400 Subject: [PATCH 2/2] docs: voice guide and design follow the review fixes Hold mode now really does run from press to release, with auto-submit on release; say so, and explain why. Note that dictation and read-aloud end each other, that the widget only stops speech it started (relevant to sites that use speechSynthesis themselves), that the option fails closed, and that CJK sentence marks are recognized. The design doc records the reasoning behind each change, including the pause-state bug the first test double had hidden. Refs #54 Co-Authored-By: Claude Fable 5.1 --- .../2026-09-18-voice-input-tts-design.md | 100 ++++++++++++++---- docs/src/content/docs/configuration/voice.md | 42 +++++--- 2 files changed, 107 insertions(+), 35 deletions(-) diff --git a/docs/plans/2026-09-18-voice-input-tts-design.md b/docs/plans/2026-09-18-voice-input-tts-design.md index 3f5b320..57197fc 100644 --- a/docs/plans/2026-09-18-voice-input-tts-design.md +++ b/docs/plans/2026-09-18-voice-input-tts-design.md @@ -92,6 +92,9 @@ voice?: boolean | VoiceOptions; // Default false Both `false` resolves to disabled. - An unknown `mode` falls back to `"toggle"` rather than throwing, since it can arrive from an HTML attribute. +- The option **fails closed**: only `true` or an options object enables it. + The string `"false"` from a CMS template is truthy, and must not switch a + microphone on. - `` gains `voice`, `voice-mode`, `voice-auto-submit`, `voice-input`, `voice-output`, and `voice-lang`. Unlike attachment limits these are plain enums, booleans, and strings, and the web component cannot @@ -146,8 +149,9 @@ message may speak at a time and it already holds `streamingMessageId`. - `interimResults = true` so words appear in the field as they are spoken. `continuous = false` in both modes. Continuous mode duplicates results on Chrome for Android and misreports `isFinal` in Safari; a single utterance - per session is what a chat message needs. If the engine ends the session - during a pause, pressing the mic again appends. + per session is what a chat message needs. In toggle mode, if the engine + ends the session during a pause, pressing the mic again appends. Hold mode + bridges the pause itself (below). - Dictation **appends** to whatever is already typed. The field value at `start()` is the base; each result renders `joinDictation(base, transcript)` clipped to the existing 2000 character limit. @@ -167,16 +171,41 @@ message may speak at a time and it already holds `streamingMessageId`. stops; Space and Enter work the same way from the keyboard. Releasing before the engine has actually started calls `abort()` rather than `stop()`, so a permission prompt answered after release can never leave the - microphone open. A `click` with no preceding pointer or key press, which is - how screen readers and switch devices activate a button, toggles instead, - so hold mode stays usable without a sustained press. -- **Auto-submit** fires only when the session ended normally and produced - final text. Never after an error, an abort, or silence. It goes through the - same submit path as the send button, so pending attachments, the length - limit, and plugins all apply. -- Starting dictation **cancels any read-aloud** so the microphone does not - transcribe the widget's own voice. + microphone open. A `click` with no click count, which is how screen readers + and switch devices activate a button, toggles instead, so hold mode stays + usable without a sustained press. +- **A hold is one dictation spanning several engine sessions.** With + `continuous = false` the engine ends a session at the first pause in + speech, which would silently drop whatever the visitor says next with the + button still down. So when a session ends mid-hold after hearing something, + `ChatInput` opens another and keeps appending. A session that heard nothing + is not replaced, so an engine that ends immediately cannot spin. Only a + real hold is bridged this way: a toggle start has no guaranteed release, so + bridging it could leave the microphone open indefinitely. The button tells + its parent which it was (`onStart({ held })`). +- `start()` and `stop()` report whether they did anything. A press while the + previous session is still winding down is refused, and must not move the + dictation base: that session's final result is still to arrive against the + old base, and re-basing would duplicate the text. When it then ends with + the button down again, the bridge above picks the new hold up. +- **Auto-submit** fires once per dictation, when it ends: never after a real + error or an abort, and only if something was heard. In hold mode that means + on release, not at the first pause. A later session of a hold timing out in + silence (`no-speech`) is not an error, just a visitor who stopped talking + while still holding; the text heard earlier is still sent on release. It + goes through the same submit path as the send button, so pending + attachments, the length limit, and plugins all apply. +- **Dictation and read-aloud end each other**, in both directions, so the + microphone never transcribes the widget's own voice (which auto-submit + would then send as the visitor's message). +- A request starting to load also ends dictation. The field is disabled then, + and an auto-submit would clear text that `useChat` refuses to send. - Unmounting (closing the chat) aborts the session. +- Defensive against engines that misbehave: an error releases the session + without waiting for the `end` the spec promises; hearing anything counts as + "started" for engines that skip the `start` event; and an `aborted` raised + by the engine itself (another tab took the microphone) ends the dictation + quietly without running auto-submit. ### Listening indicator @@ -203,10 +232,15 @@ in opacity. have text. Nothing plays automatically. - Text passes through the existing `stripAnnouncementFormatting`, so markdown markers are not read out and URLs collapse to hostnames. -- The text is split at sentence boundaries into chunks of at most 200 - characters and queued as separate utterances. That avoids Chrome's - 15-second cutoff and gives clean pause points. Utterances are held in a ref - because Safari garbage-collects them mid-speech otherwise. +- The text is split at sentence boundaries into chunks of at most 150 + characters, about ten seconds of speech, and queued as separate utterances. + That avoids Chrome's 15-second cutoff, with room for numbers, which take + far longer to say than to write. Boundaries are newlines, `.` `!` `?` + followed by whitespace (so "example.com" and "$9.99" do not split), and the + full-width `。` `！` `？`, which have no space after them. An over-long + sentence splits between words and an over-long unbroken token is cut, since + that is what unpunctuated Chinese or Japanese looks like. Utterances are + held in a ref because Safari garbage-collects them mid-speech otherwise. - **The voice is left to the browser**: only `utterance.lang` is set. An earlier draft picked a local voice explicitly, for privacy and to avoid the cutoff. That was dropped. Voice lists are alphabetical, so "first local @@ -222,8 +256,26 @@ in opacity. `speak()` fires `error: synthesis-failed`, which returns the control to idle rather than leaving it stuck. - Controls: idle shows Read aloud; speaking shows Pause and Stop; paused - shows Resume and Stop. Starting another message stops the current one. -- A session counter makes late events from cancelled utterances harmless. + shows Resume and Stop. Starting another message stops the current one. The + first button is a single element that changes role, so keyboard focus + survives the swap; when Stop disappears with focus on it, focus moves to + the remaining button. Otherwise focus would fall to ``, outside the + dialog's focus trap. +- **Pause state comes from the utterance's `pause` / `resume` events**, plus + the `paused` flag for Firefox, which sets it synchronously. Chrome and + Safari set the flag only once the engine confirms, so reading it right + after `pause()` (the first implementation) left the UI on "Pause" with no + way to resume. Verified against Chrome 153. +- **The engine is a shared global, so the reader only cancels speech it + started.** A reader that never spoke must not silence the host page when + the chat closes, which matters even with voice switched off, because the + hook always mounts. An `interrupted` / `canceled` error with no pause of + ours pending means another widget or the host page took over: the reader + forgets its own playback and leaves the engine alone, since cancelling + again would kill the newcomer's speech. +- Per the spec `cancel()` leaves a paused engine paused, and the next reply + would queue in silence, so stopping while paused also resumes the engine. +- A run counter makes late events from cancelled utterances harmless. - **Android pause:** if a chunk ends while a pause was requested and the engine does not report `paused`, the platform treated pause as cancel. The rest of the queue is cancelled and the control returns to idle, instead of @@ -240,9 +292,10 @@ existing `role="alert"` slot: | `no-speech` | No speech was detected | | `audio-capture` | No microphone was found | | `network`, `language-not-supported`, anything else, `start()` throwing | Voice input is unavailable | -| `aborted` | ignored | +| `aborted` | no message | +| `no-speech` on a later session of the same hold | no message | -The message clears on the next start or edit. Synthesis errors return the +The message clears on the next start, edit, or send. Synthesis errors return the control to idle silently; there is nothing useful to tell the user. ## Accessibility @@ -276,9 +329,12 @@ implementations that tests drive event by event. - `useSpeechRecognition`: unsupported no-op, session setup, interim and final transcripts, stop versus abort, each error mapping, `start()` throwing, double start, unmount. -- `useSpeechSynthesis`: chunk queue with language and voice, state from - events, pause and resume, the Android pause path, switching messages, stale - events after cancel, unmount, unsupported no-op. +- `useSpeechSynthesis`: chunk queue with language, pause state as each + browser family reports it, the Android pause paths, switching messages, + stale events after cancel, speech it does not own, unmount, unsupported + no-op. The fake confirms a pause asynchronously by default, as Chrome and + Safari do; an earlier fake that set `paused` inside `pause()` hid a real + bug. - Components: visibility rules, both modes by pointer, keyboard, and bare click, appending, typing takes over, auto-submit and the cases where it must not fire, error display, read-aloud only on settled assistant diff --git a/docs/src/content/docs/configuration/voice.md b/docs/src/content/docs/configuration/voice.md index 1dadf8c..364f2df 100644 --- a/docs/src/content/docs/configuration/voice.md +++ b/docs/src/content/docs/configuration/voice.md @@ -40,7 +40,11 @@ recognition, and your privacy policy may need to say so. ``` -`true` turns on both the mic and read-aloud. Pass an object to tune them: +`true` turns on both the mic and read-aloud. Pass an object to tune them. In +the `voice` prop and in `ClaudiusConfig`, anything else leaves voice off, +including a string such as `"true"`: an option that switches on a microphone +fails closed. (HTML attributes are always strings, so `` parses +them for you, as shown below.) | Option | Default | Description | |--------|---------|-------------| @@ -87,8 +91,9 @@ an error or a silent session. It uses the same path as the send button, so pending [attachments](/configuration/attachments/), the length limit, and [plugins](/plugins/) all apply. -Starting dictation stops any reply that is being read aloud, so the -microphone does not transcribe the widget's own voice. +Dictation and read-aloud end each other: starting one stops the other, so +the microphone never transcribes the widget's own voice. A message being sent +ends dictation too. ### Toggle or hold @@ -96,11 +101,17 @@ microphone does not transcribe the widget's own voice. sustained press. While listening, the button shows a stop icon. `"hold"` is push-to-talk: recording runs from press to release, with a mouse, -a finger, or Space / Enter on the keyboard. If the visitor lets go before the -browser's permission prompt has been answered, nothing starts, so granting -permission afterwards can never open the microphone with nobody at the -button. Screen readers and switch devices activate buttons with a click they -cannot hold, so in hold mode a click of that kind toggles instead. +a finger, or Space / Enter on the keyboard. Browsers end a recognition session +at the first pause in speech, so while the button stays down Claudius opens +the next session itself and keeps appending; a visitor can stop to think +mid-sentence without losing the rest. With `autoSubmit`, the message is sent +**on release**, not at the first pause. + +If the visitor lets go before the browser's permission prompt has been +answered, nothing starts, so granting permission afterwards can never open +the microphone with nobody at the button. Screen readers and switch devices +activate buttons with a click they cannot hold, so in hold mode a click of +that kind toggles instead. ### Listening indicator @@ -123,11 +134,16 @@ in opacity only. The button appears once a reply has finished streaming. Formatting marks are not spoken and links are shortened to their hostname, the same cleanup used for screen-reader announcements. One reply plays at a time; starting another -stops the first, and closing the chat stops playback. - -Long replies are queued sentence by sentence. Chrome cuts a single long -utterance off after about 15 seconds when it is using a network voice, and -short utterances avoid that. +stops the first, and closing the chat stops playback. `speechSynthesis` is +shared with the rest of the page, so Claudius only ever stops speech it +started itself: if your site uses speech synthesis too, closing the chat or +starting dictation will not cut it off. Pressing Read aloud does take over, +since only one thing can speak at a time. + +Long replies are queued a few sentences at a time, about ten seconds of +speech each. Chrome cuts a single long utterance off after about 15 seconds +when it is using a network voice, and short utterances avoid that. Chinese +and Japanese sentence marks are recognized as well. On **Android**, browsers implement "pause" as "stop". Claudius detects this and returns the control to its idle state, rather than carrying on with the