diff --git a/docs/plans/2026-09-18-voice-input-tts-design.md b/docs/plans/2026-09-18-voice-input-tts-design.md index 3f5b320..57197fc 100644 --- a/docs/plans/2026-09-18-voice-input-tts-design.md +++ b/docs/plans/2026-09-18-voice-input-tts-design.md @@ -92,6 +92,9 @@ voice?: boolean | VoiceOptions; // Default false Both `false` resolves to disabled. - An unknown `mode` falls back to `"toggle"` rather than throwing, since it can arrive from an HTML attribute. +- The option **fails closed**: only `true` or an options object enables it. + The string `"false"` from a CMS template is truthy, and must not switch a + microphone on. - `` gains `voice`, `voice-mode`, `voice-auto-submit`, `voice-input`, `voice-output`, and `voice-lang`. Unlike attachment limits these are plain enums, booleans, and strings, and the web component cannot @@ -146,8 +149,9 @@ message may speak at a time and it already holds `streamingMessageId`. - `interimResults = true` so words appear in the field as they are spoken. `continuous = false` in both modes. Continuous mode duplicates results on Chrome for Android and misreports `isFinal` in Safari; a single utterance - per session is what a chat message needs. If the engine ends the session - during a pause, pressing the mic again appends. + per session is what a chat message needs. In toggle mode, if the engine + ends the session during a pause, pressing the mic again appends. Hold mode + bridges the pause itself (below). - Dictation **appends** to whatever is already typed. The field value at `start()` is the base; each result renders `joinDictation(base, transcript)` clipped to the existing 2000 character limit. @@ -167,16 +171,41 @@ message may speak at a time and it already holds `streamingMessageId`. stops; Space and Enter work the same way from the keyboard. Releasing before the engine has actually started calls `abort()` rather than `stop()`, so a permission prompt answered after release can never leave the - microphone open. A `click` with no preceding pointer or key press, which is - how screen readers and switch devices activate a button, toggles instead, - so hold mode stays usable without a sustained press. -- **Auto-submit** fires only when the session ended normally and produced - final text. Never after an error, an abort, or silence. It goes through the - same submit path as the send button, so pending attachments, the length - limit, and plugins all apply. -- Starting dictation **cancels any read-aloud** so the microphone does not - transcribe the widget's own voice. + microphone open. A `click` with no click count, which is how screen readers + and switch devices activate a button, toggles instead, so hold mode stays + usable without a sustained press. +- **A hold is one dictation spanning several engine sessions.** With + `continuous = false` the engine ends a session at the first pause in + speech, which would silently drop whatever the visitor says next with the + button still down. So when a session ends mid-hold after hearing something, + `ChatInput` opens another and keeps appending. A session that heard nothing + is not replaced, so an engine that ends immediately cannot spin. Only a + real hold is bridged this way: a toggle start has no guaranteed release, so + bridging it could leave the microphone open indefinitely. The button tells + its parent which it was (`onStart({ held })`). +- `start()` and `stop()` report whether they did anything. A press while the + previous session is still winding down is refused, and must not move the + dictation base: that session's final result is still to arrive against the + old base, and re-basing would duplicate the text. When it then ends with + the button down again, the bridge above picks the new hold up. +- **Auto-submit** fires once per dictation, when it ends: never after a real + error or an abort, and only if something was heard. In hold mode that means + on release, not at the first pause. A later session of a hold timing out in + silence (`no-speech`) is not an error, just a visitor who stopped talking + while still holding; the text heard earlier is still sent on release. It + goes through the same submit path as the send button, so pending + attachments, the length limit, and plugins all apply. +- **Dictation and read-aloud end each other**, in both directions, so the + microphone never transcribes the widget's own voice (which auto-submit + would then send as the visitor's message). +- A request starting to load also ends dictation. The field is disabled then, + and an auto-submit would clear text that `useChat` refuses to send. - Unmounting (closing the chat) aborts the session. +- Defensive against engines that misbehave: an error releases the session + without waiting for the `end` the spec promises; hearing anything counts as + "started" for engines that skip the `start` event; and an `aborted` raised + by the engine itself (another tab took the microphone) ends the dictation + quietly without running auto-submit. ### Listening indicator @@ -203,10 +232,15 @@ in opacity. have text. Nothing plays automatically. - Text passes through the existing `stripAnnouncementFormatting`, so markdown markers are not read out and URLs collapse to hostnames. -- The text is split at sentence boundaries into chunks of at most 200 - characters and queued as separate utterances. That avoids Chrome's - 15-second cutoff and gives clean pause points. Utterances are held in a ref - because Safari garbage-collects them mid-speech otherwise. +- The text is split at sentence boundaries into chunks of at most 150 + characters, about ten seconds of speech, and queued as separate utterances. + That avoids Chrome's 15-second cutoff, with room for numbers, which take + far longer to say than to write. Boundaries are newlines, `.` `!` `?` + followed by whitespace (so "example.com" and "$9.99" do not split), and the + full-width `。` `！` `？`, which have no space after them. An over-long + sentence splits between words and an over-long unbroken token is cut, since + that is what unpunctuated Chinese or Japanese looks like. Utterances are + held in a ref because Safari garbage-collects them mid-speech otherwise. - **The voice is left to the browser**: only `utterance.lang` is set. An earlier draft picked a local voice explicitly, for privacy and to avoid the cutoff. That was dropped. Voice lists are alphabetical, so "first local @@ -222,8 +256,26 @@ in opacity. `speak()` fires `error: synthesis-failed`, which returns the control to idle rather than leaving it stuck. - Controls: idle shows Read aloud; speaking shows Pause and Stop; paused - shows Resume and Stop. Starting another message stops the current one. -- A session counter makes late events from cancelled utterances harmless. + shows Resume and Stop. Starting another message stops the current one. The + first button is a single element that changes role, so keyboard focus + survives the swap; when Stop disappears with focus on it, focus moves to + the remaining button. Otherwise focus would fall to ``, outside the + dialog's focus trap. +- **Pause state comes from the utterance's `pause` / `resume` events**, plus + the `paused` flag for Firefox, which sets it synchronously. Chrome and + Safari set the flag only once the engine confirms, so reading it right + after `pause()` (the first implementation) left the UI on "Pause" with no + way to resume. Verified against Chrome 153. +- **The engine is a shared global, so the reader only cancels speech it + started.** A reader that never spoke must not silence the host page when + the chat closes, which matters even with voice switched off, because the + hook always mounts. An `interrupted` / `canceled` error with no pause of + ours pending means another widget or the host page took over: the reader + forgets its own playback and leaves the engine alone, since cancelling + again would kill the newcomer's speech. +- Per the spec `cancel()` leaves a paused engine paused, and the next reply + would queue in silence, so stopping while paused also resumes the engine. +- A run counter makes late events from cancelled utterances harmless. - **Android pause:** if a chunk ends while a pause was requested and the engine does not report `paused`, the platform treated pause as cancel. The rest of the queue is cancelled and the control returns to idle, instead of @@ -240,9 +292,10 @@ existing `role="alert"` slot: | `no-speech` | No speech was detected | | `audio-capture` | No microphone was found | | `network`, `language-not-supported`, anything else, `start()` throwing | Voice input is unavailable | -| `aborted` | ignored | +| `aborted` | no message | +| `no-speech` on a later session of the same hold | no message | -The message clears on the next start or edit. Synthesis errors return the +The message clears on the next start, edit, or send. Synthesis errors return the control to idle silently; there is nothing useful to tell the user. ## Accessibility @@ -276,9 +329,12 @@ implementations that tests drive event by event. - `useSpeechRecognition`: unsupported no-op, session setup, interim and final transcripts, stop versus abort, each error mapping, `start()` throwing, double start, unmount. -- `useSpeechSynthesis`: chunk queue with language and voice, state from - events, pause and resume, the Android pause path, switching messages, stale - events after cancel, unmount, unsupported no-op. +- `useSpeechSynthesis`: chunk queue with language, pause state as each + browser family reports it, the Android pause paths, switching messages, + stale events after cancel, speech it does not own, unmount, unsupported + no-op. The fake confirms a pause asynchronously by default, as Chrome and + Safari do; an earlier fake that set `paused` inside `pause()` hid a real + bug. - Components: visibility rules, both modes by pointer, keyboard, and bare click, appending, typing takes over, auto-submit and the cases where it must not fire, error display, read-aloud only on settled assistant diff --git a/docs/src/content/docs/configuration/voice.md b/docs/src/content/docs/configuration/voice.md index 1dadf8c..364f2df 100644 --- a/docs/src/content/docs/configuration/voice.md +++ b/docs/src/content/docs/configuration/voice.md @@ -40,7 +40,11 @@ recognition, and your privacy policy may need to say so. ``` -`true` turns on both the mic and read-aloud. Pass an object to tune them: +`true` turns on both the mic and read-aloud. Pass an object to tune them. In +the `voice` prop and in `ClaudiusConfig`, anything else leaves voice off, +including a string such as `"true"`: an option that switches on a microphone +fails closed. (HTML attributes are always strings, so `` parses +them for you, as shown below.) | Option | Default | Description | |--------|---------|-------------| @@ -87,8 +91,9 @@ an error or a silent session. It uses the same path as the send button, so pending [attachments](/configuration/attachments/), the length limit, and [plugins](/plugins/) all apply. -Starting dictation stops any reply that is being read aloud, so the -microphone does not transcribe the widget's own voice. +Dictation and read-aloud end each other: starting one stops the other, so +the microphone never transcribes the widget's own voice. A message being sent +ends dictation too. ### Toggle or hold @@ -96,11 +101,17 @@ microphone does not transcribe the widget's own voice. sustained press. While listening, the button shows a stop icon. `"hold"` is push-to-talk: recording runs from press to release, with a mouse, -a finger, or Space / Enter on the keyboard. If the visitor lets go before the -browser's permission prompt has been answered, nothing starts, so granting -permission afterwards can never open the microphone with nobody at the -button. Screen readers and switch devices activate buttons with a click they -cannot hold, so in hold mode a click of that kind toggles instead. +a finger, or Space / Enter on the keyboard. Browsers end a recognition session +at the first pause in speech, so while the button stays down Claudius opens +the next session itself and keeps appending; a visitor can stop to think +mid-sentence without losing the rest. With `autoSubmit`, the message is sent +**on release**, not at the first pause. + +If the visitor lets go before the browser's permission prompt has been +answered, nothing starts, so granting permission afterwards can never open +the microphone with nobody at the button. Screen readers and switch devices +activate buttons with a click they cannot hold, so in hold mode a click of +that kind toggles instead. ### Listening indicator @@ -123,11 +134,16 @@ in opacity only. The button appears once a reply has finished streaming. Formatting marks are not spoken and links are shortened to their hostname, the same cleanup used for screen-reader announcements. One reply plays at a time; starting another -stops the first, and closing the chat stops playback. - -Long replies are queued sentence by sentence. Chrome cuts a single long -utterance off after about 15 seconds when it is using a network voice, and -short utterances avoid that. +stops the first, and closing the chat stops playback. `speechSynthesis` is +shared with the rest of the page, so Claudius only ever stops speech it +started itself: if your site uses speech synthesis too, closing the chat or +starting dictation will not cut it off. Pressing Read aloud does take over, +since only one thing can speak at a time. + +Long replies are queued a few sentences at a time, about ten seconds of +speech each. Chrome cuts a single long utterance off after about 15 seconds +when it is using a network voice, and short utterances avoid that. Chinese +and Japanese sentence marks are recognized as well. On **Android**, browsers implement "pause" as "stop". Claudius detects this and returns the control to its idle state, rather than carrying on with the diff --git a/widget/e2e/voice.spec.ts b/widget/e2e/voice.spec.ts index 5cb723f..ad19166 100644 --- a/widget/e2e/voice.spec.ts +++ b/widget/e2e/voice.spec.ts @@ -1,6 +1,12 @@ import { test, expect, type Page } from "@playwright/test"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; import { mockChatApi } from "./helpers"; +const __dirname = dirname(fileURLToPath(import.meta.url)); +const IIFE_PATH = resolve(__dirname, "..", "dist", "claudius.iife.js"); +const CSS_PATH = resolve(__dirname, "..", "dist", "claudius.css"); + /** * Voice features against fake speech engines, injected before the app loads. * Real engines cannot run here: CI has no microphone, and Chromium builds ship @@ -31,6 +37,10 @@ async function installFakeSpeech(page: Page) { this.onend?.({}); } abort() {} + /** The engine ends the session by itself, as it does at a pause. */ + end() { + this.onend?.({}); + } hear(transcript: string, isFinal: boolean) { this.onresult?.({ results: [{ isFinal, length: 1, 0: { transcript } }], @@ -39,6 +49,7 @@ async function installFakeSpeech(page: Page) { } const spoken: string[] = []; + let current: SpeechSynthesisUtterance | null = null; const synth = { speaking: false, pending: false, @@ -46,17 +57,25 @@ async function installFakeSpeech(page: Page) { spoken, speak(utterance: SpeechSynthesisUtterance) { spoken.push(`${utterance.lang}|${utterance.text}`); + current = utterance; this.speaking = true; }, + // Per the spec, cancel() does not clear the paused state. cancel() { this.speaking = false; - this.paused = false; + current = null; }, + // Like Chrome and Safari: `paused` flips only when the engine confirms, + // which is also when the utterance gets its `pause` event. pause() { - this.paused = true; + setTimeout(() => { + this.paused = true; + current?.onpause?.call(current, new Event("pause") as never); + }, 0); }, resume() { this.paused = false; + current?.onresume?.call(current, new Event("resume") as never); }, getVoices: () => [], }; @@ -135,6 +154,54 @@ test.describe("voice input and read-aloud", () => { .toBeGreaterThan(0); }); + test("hold-to-talk through the embed bundle: a real press and release", async ({ + page, + }) => { + await installFakeSpeech(page); + const api = await mockChatApi(page); + api.enqueueReply("Yes, 20 percent."); + // Same approach as embed.spec.ts: the dev origin, minus the dev widget. + await page.goto("/"); + await page.evaluate(() => { + document.getElementById("root")?.remove(); + (window as unknown as { ClaudiusConfig: unknown }).ClaudiusConfig = { + apiUrl: "https://test.example", + voice: { mode: "hold", autoSubmit: true }, + }; + }); + await page.addStyleTag({ path: CSS_PATH }); + await page.addScriptTag({ path: IIFE_PATH }); + await page.getByRole("button", { name: /open chat/i }).click(); + + const mic = page.getByRole("button", { name: "Hold to talk" }); + const box = (await mic.boundingBox())!; + await page.mouse.move(box.x + box.width / 2, box.y + box.height / 2); + await page.mouse.down(); + await expect(mic).toHaveAttribute("aria-pressed", "true"); + await hear(page, "do you offer", true); + + // The engine ends the session at a pause, with the button still down. + // A new session takes over, and nothing is sent yet. + await page.evaluate(() => + ( + window as unknown as { __recognition: { end(): void } } + ).__recognition.end(), + ); + await expect(mic).toHaveAttribute("aria-pressed", "true"); + expect(api.callCount()).toBe(0); + await hear(page, "a discount", true); + + await page.mouse.up(); + + const log = page.getByRole("log"); + await expect(log.getByText("do you offer a discount")).toBeVisible(); + await expect(log.getByText("Yes, 20 percent.")).toBeVisible(); + // Exactly one message: the click Chromium fires after the release carries + // a click count, so it must not have started another dictation. + expect(api.callCount()).toBe(1); + await expect(mic).toHaveAttribute("aria-pressed", "false"); + }); + test("reads a reply aloud with pause and stop controls", async ({ page }) => { await installFakeSpeech(page); const api = await mockChatApi(page); diff --git a/widget/src/components/ChatInput.tsx b/widget/src/components/ChatInput.tsx index 38f2976..7415788 100644 --- a/widget/src/components/ChatInput.tsx +++ b/widget/src/components/ChatInput.tsx @@ -51,6 +51,11 @@ interface ChatInputProps { voice?: ResolvedVoiceConfig | null; /** Called when dictation starts, so the parent can silence read-aloud. */ onVoiceStart?: () => void; + /** + * True while a reply is being read aloud. Dictation ends when it turns + * true, or the microphone would transcribe the widget's own voice. + */ + isReadingAloud?: boolean; } function dragHasFiles(e: DragEvent): boolean { @@ -68,6 +73,7 @@ export function ChatInput({ attachments = null, voice = null, onVoiceStart, + isReadingAloud = false, }: ChatInputProps) { const t = translations ?? defaultTranslations; const [value, setValue] = useState(""); @@ -81,8 +87,15 @@ export function ChatInput({ const dragDepthRef = useRef(0); // Mirrors `value` for the speech callbacks, which fire outside a render. const valueRef = useRef(""); - // What the field held when dictation began; speech is appended to it. + // What the field held when the current session began; speech is appended. const dictationBaseRef = useRef(""); + // Hold mode: true from press to release. One dictation can span several + // engine sessions, because the engine ends a session at the first pause. + const heldRef = useRef(false); + // Whether this dictation has heard anything, and whether it really failed. + // Together they decide if auto-submit may fire when it ends. + const heardRef = useRef(false); + const failedRef = useRef(false); const updateValue = (next: string) => { valueRef.current = next; @@ -167,6 +180,7 @@ export function ChatInput({ pendingRef.current = []; setPending([]); setAttachmentError(null); + setVoiceError(null); }; const describeVoiceError = (kind: SpeechRecognitionErrorKind): string => { @@ -189,10 +203,24 @@ export function ChatInput({ joinDictation(dictationBaseRef.current, transcript, MAX_MESSAGE_LENGTH), ), onEnd: (transcript) => { - // Only a session that ended normally and actually heard something. - if (voice?.autoSubmit && transcript.trim()) submit(); + const heard = transcript.trim() !== ""; + if (heard) heardRef.current = true; + if (!heldRef.current) { + finishDictation(); + return; + } + // Still holding: the engine merely ended at a pause, so open another + // session and leave any auto-submit for the release. A session that + // heard nothing is not replaced, so a broken engine cannot spin. + if (heard) beginSession(); + }, + onError: (kind) => { + // A later session of a hold timing out in silence is not a failure: + // the visitor stopped talking and is simply still holding the button. + if (kind === "no-speech" && heardRef.current) return; + failedRef.current = true; + setVoiceError(describeVoiceError(kind)); }, - onError: (kind) => setVoiceError(describeVoiceError(kind)), }); const showMic = !!voice?.input && recognition.isSupported; const isListening = recognition.isListening; @@ -204,17 +232,54 @@ export function ChatInput({ if (isListening && input) input.scrollLeft = input.scrollWidth; }, [isListening, value]); - const startDictation = () => { + // Auto-submit fires only for a dictation that heard something and did not + // fail. It goes through the send button's own path. + const finishDictation = () => { + if (voice?.autoSubmit && heardRef.current && !failedRef.current) submit(); + }; + + const beginSession = () => { + const base = valueRef.current; + // Refused while the previous session is still winding down. Its results + // are still arriving against the old base, so that must not move. + if (recognition.start()) dictationBaseRef.current = base; + }; + + const startDictation = ({ held }: { held: boolean }) => { setVoiceError(null); - dictationBaseRef.current = valueRef.current; + heldRef.current = held; + heardRef.current = false; + failedRef.current = false; onVoiceStart?.(); - recognition.start(); + beginSession(); }; + const stopDictation = () => { + const wasHeld = heldRef.current; + heldRef.current = false; + // Nothing left to stop means the last session already ended mid-hold; the + // auto-submit it deferred is due now. + if (!recognition.stop() && wasHeld) finishDictation(); + }; + + const abortDictation = () => { + heldRef.current = false; + recognition.abort(); + }; + + // A request going out disables the field, and auto-submit would then clear + // text it could not send. A reply being read aloud would be transcribed. + const { abort } = recognition; + useEffect(() => { + if (!isLoading && !isReadingAloud) return; + heldRef.current = false; + abort(); + }, [isLoading, isReadingAloud, abort]); + const handleSubmit = (e: FormEvent) => { e.preventDefault(); // Otherwise the next result would put the sent text back in the field. - recognition.abort(); + abortDictation(); submit(); }; @@ -222,7 +287,7 @@ export function ChatInput({ const newValue = e.target.value; if (newValue.length <= MAX_MESSAGE_LENGTH) { // Typing takes over from dictation: keep the edit, drop the session. - recognition.abort(); + abortDictation(); setVoiceError(null); updateValue(newValue); } @@ -368,7 +433,7 @@ export function ChatInput({ isListening={isListening} disabled={isLoading} onStart={startDictation} - onStop={recognition.stop} + onStop={stopDictation} label={voice.mode === "hold" ? t.voiceInputHold : t.voiceInput} /> )} diff --git a/widget/src/components/ChatWindow.tsx b/widget/src/components/ChatWindow.tsx index 5c0deba..1b9f5c0 100644 --- a/widget/src/components/ChatWindow.tsx +++ b/widget/src/components/ChatWindow.tsx @@ -288,8 +288,10 @@ export function ChatWindow({ translations={translations} attachments={attachments} voice={voice} - // The mic would otherwise transcribe the widget's own voice. + // The mic would otherwise transcribe the widget's own voice, so each + // of the two ends the other. onVoiceStart={reader.cancel} + isReadingAloud={reader.activeId !== null} /> ); diff --git a/widget/src/components/MessageSpeechControls.tsx b/widget/src/components/MessageSpeechControls.tsx index d283e27..3f83c97 100644 --- a/widget/src/components/MessageSpeechControls.tsx +++ b/widget/src/components/MessageSpeechControls.tsx @@ -1,4 +1,4 @@ -import type { ReactNode } from "react"; +import { forwardRef, useEffect, useRef, type ReactNode } from "react"; /** Read-aloud state of one message. */ export type MessageSpeechState = "idle" | "speaking" | "paused"; @@ -12,17 +12,13 @@ interface MessageSpeechControlsProps { onStop: () => void; } -function IconButton({ - label, - onClick, - children, -}: { - label: string; - onClick: () => void; - children: ReactNode; -}) { +const IconButton = forwardRef< + HTMLButtonElement, + { label: string; onClick: () => void; children: ReactNode } +>(function IconButton({ label, onClick, children }, ref) { return ( ); -} +}); /** * Read-aloud controls for an assistant message: a speaker button while idle, * pause and stop while speaking, resume and stop while paused. + * + * The first button is one element that changes role (read aloud, pause, + * resume), so keyboard focus survives the change. Swapping in a different + * element would drop focus to ``, outside the dialog's focus trap. */ export function MessageSpeechControls({ state, @@ -58,31 +58,53 @@ export function MessageSpeechControls({ onResume, onStop, }: MessageSpeechControlsProps) { - if (state === "idle") { - return ( - - - - - - ); - } + const groupRef = useRef(null); + const primaryRef = useRef(null); + + // Stop disappears when playback ends or is stopped. Read focus now, during + // render: it is the last moment a focused Stop button is still in the DOM. + // If it then vanishes, hand focus to the button that remains. + const hadFocusWithin = !!groupRef.current?.contains(document.activeElement); + useEffect(() => { + if ( + state === "idle" && + hadFocusWithin && + !groupRef.current?.contains(document.activeElement) + ) { + primaryRef.current?.focus(); + } + }, [state, hadFocusWithin]); + + const primary = + state === "idle" + ? { label: labels.play, onClick: onPlay } + : state === "paused" + ? { label: labels.resume, onClick: onResume } + : { label: labels.pause, onClick: onPause }; return ( -
- {state === "paused" ? ( - +
+ + {state === "idle" ? ( + <> + + + + + ) : state === "paused" ? ( - - ) : ( - - - + ) : ( + <> + + + + )} + + {state !== "idle" && ( + + )} - - -
); } diff --git a/widget/src/components/VoiceInputButton.tsx b/widget/src/components/VoiceInputButton.tsx index 15eb660..e42171d 100644 --- a/widget/src/components/VoiceInputButton.tsx +++ b/widget/src/components/VoiceInputButton.tsx @@ -10,7 +10,12 @@ interface VoiceInputButtonProps { mode: VoiceInputMode; isListening: boolean; disabled?: boolean; - onStart: () => void; + /** + * `held` is true only for a real press-and-hold, which is guaranteed to end + * with a release. A toggle start (including the assistive-technology click + * in hold mode) has no such guarantee. + */ + onStart: (gesture: { held: boolean }) => void; onStop: () => void; /** * Accessible name. It stays the same in both states; `aria-pressed` carries @@ -42,7 +47,7 @@ export function VoiceInputButton({ const beginHold = () => { if (disabled || holdingRef.current) return; holdingRef.current = true; - onStart(); + onStart({ held: true }); }; // Idempotent: a release is often reported twice (pointerup, then @@ -59,7 +64,7 @@ export function VoiceInputButton({ // the button, which cannot hold it down. Let those toggle. if (isHold && e.detail !== 0) return; if (isListening) onStop(); - else onStart(); + else onStart({ held: false }); }; const handlePointerDown = (e: PointerEvent) => { @@ -102,7 +107,9 @@ export function VoiceInputButton({ style={isHold ? { WebkitTouchCallout: "none" } : undefined} className={`flex h-11 w-11 shrink-0 items-center justify-center rounded-claudius-sm border transition-colors focus:outline-none focus:ring-1 focus:ring-claudius-accent disabled:opacity-50 ${ isListening - ? "border-claudius-accent bg-claudius-accent text-claudius-accent-text" + ? // An accent ring on an accent fill is invisible; offset it so + // keyboard users can still see where focus is. + "border-claudius-accent bg-claudius-accent text-claudius-accent-text focus:ring-2 focus:ring-offset-2 focus:ring-offset-claudius-surface" : "border-claudius-border bg-claudius-field text-claudius-text-muted hover:text-claudius-text focus:border-claudius-accent" } ${isHold ? "touch-none select-none" : ""}`} > diff --git a/widget/src/components/__tests__/ChatInput.voice.test.tsx b/widget/src/components/__tests__/ChatInput.voice.test.tsx index 573133f..0597f6a 100644 --- a/widget/src/components/__tests__/ChatInput.voice.test.tsx +++ b/widget/src/components/__tests__/ChatInput.voice.test.tsx @@ -33,20 +33,27 @@ const translations = createTranslations({ function setup( voice: ResolvedVoiceConfig | null = VOICE, - props: { isLoading?: boolean } = {}, + props: { isLoading?: boolean; isReadingAloud?: boolean } = {}, ) { const onSend = vi.fn(); const onVoiceStart = vi.fn(); - const view = render( + const ui = (p: typeof props) => ( , + /> ); - return { ...view, onSend, onVoiceStart }; + const view = render(ui(props)); + return { + ...view, + onSend, + onVoiceStart, + update: (next: typeof props) => view.rerender(ui(next)), + }; } const mic = () => screen.getByRole("button", { name: "MIC" }); @@ -342,4 +349,178 @@ describe("ChatInput voice input", () => { expect(screen.queryByRole("alert")).not.toBeInTheDocument(); }); }); + + describe("hold mode lasts as long as the hold", () => { + const HOLD = { ...VOICE, mode: "hold" as const }; + const holdButton = () => screen.getByRole("button", { name: "MIC-HOLD" }); + + function press() { + fireEvent.pointerDown(holdButton(), { button: 0 }); + } + function release() { + fireEvent.pointerUp(holdButton()); + } + function sessions() { + return FakeSpeechRecognition.instances; + } + + it("keeps listening when the engine ends a session at a pause in speech", () => { + setup(HOLD); + press(); + act(() => latestRecognition().emitStart()); + hear("I would like to know", true); + + // The engine finalizes and ends at the pause; the button is still down. + act(() => latestRecognition().emitEnd()); + + expect(sessions()).toHaveLength(2); + expect(holdButton()).toHaveAttribute("aria-pressed", "true"); + hear("your prices", true); + expect(field()).toHaveValue("I would like to know your prices"); + }); + + it("does not restart a session that heard nothing, so a dead engine cannot spin", () => { + setup(HOLD); + press(); + + act(() => latestRecognition().emitEnd()); + + expect(sessions()).toHaveLength(1); + expect(holdButton()).toHaveAttribute("aria-pressed", "false"); + }); + + it("does not keep a toggle start alive, since nothing guarantees it will be released", () => { + setup(HOLD); + // A click with no click count: a screen reader activating the button. + fireEvent.click(holdButton()); + hear("hello", true); + + act(() => latestRecognition().emitEnd()); + + expect(sessions()).toHaveLength(1); + }); + + it("does not duplicate the text when pressed again while the last session winds down", () => { + setup(HOLD); + press(); + const first = latestRecognition(); + act(() => first.emitStart()); + hear("hello world", false); + release(); + + // Pressed again before the engine has delivered its final result. + press(); + act(() => + first.emitResult([{ transcript: "hello world", isFinal: true }]), + ); + expect(field()).toHaveValue("hello world"); + + // The old session ends with the button down: the new hold takes over. + act(() => first.emitEnd()); + expect(sessions()).toHaveLength(2); + hear("again", true); + expect(field()).toHaveValue("hello world again"); + }); + + describe("with auto-submit", () => { + const HOLD_AUTO = { ...HOLD, autoSubmit: true }; + + it("waits for the release instead of sending at the first pause", () => { + const { onSend } = setup(HOLD_AUTO); + press(); + act(() => latestRecognition().emitStart()); + hear("I would like to know", true); + act(() => latestRecognition().emitEnd()); + expect(onSend).not.toHaveBeenCalled(); + + act(() => latestRecognition().emitStart()); + hear("your prices", true); + release(); + act(() => latestRecognition().emitEnd()); + + expect(onSend).toHaveBeenCalledExactlyOnceWith( + "I would like to know your prices", + ); + }); + + it("sends on release even if the last session timed out in silence, without an error", () => { + const { onSend } = setup(HOLD_AUTO); + press(); + act(() => latestRecognition().emitStart()); + hear("hello", true); + act(() => latestRecognition().emitEnd()); + + // The visitor goes quiet, still holding; the restarted session times out. + const second = latestRecognition(); + act(() => second.emitError("no-speech")); + act(() => second.emitEnd()); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + expect(onSend).not.toHaveBeenCalled(); + + release(); + + expect(onSend).toHaveBeenCalledExactlyOnceWith("hello"); + }); + + it("does not send on release after a real failure mid-hold", () => { + const { onSend } = setup(HOLD_AUTO); + press(); + act(() => latestRecognition().emitStart()); + hear("hello", true); + act(() => latestRecognition().emitEnd()); + + act(() => latestRecognition().emitError("network")); + expect(screen.getByRole("alert")).toHaveTextContent("ERR-UNAVAILABLE"); + + release(); + + expect(onSend).not.toHaveBeenCalled(); + expect(field()).toHaveValue("hello"); + }); + }); + }); + + describe("things that must end a dictation", () => { + const AUTO = { ...VOICE, autoSubmit: true }; + + it("a reply starting to be read aloud, or the mic would transcribe it", () => { + const { onSend, update } = setup(AUTO); + fireEvent.click(mic()); + const session = latestRecognition(); + hear("hello", false); + + update({ isReadingAloud: true }); + + expect(session.abortCalls).toBe(1); + expect(mic()).toHaveAttribute("aria-pressed", "false"); + expect(field()).toHaveValue("hello"); + expect(onSend).not.toHaveBeenCalled(); + }); + + it("a request starting to load, or auto-submit would discard the text", () => { + const { onSend, update } = setup(AUTO); + fireEvent.click(mic()); + const session = latestRecognition(); + hear("hello", true); + + update({ isLoading: true }); + + expect(session.abortCalls).toBe(1); + expect(field()).toHaveValue("hello"); + expect(onSend).not.toHaveBeenCalled(); + }); + }); + + it("clears a voice error once a message has been sent", async () => { + const user = userEvent.setup(); + setup(); + fireEvent.click(mic()); + hear("hello", true); + act(() => latestRecognition().emitError("network")); + expect(screen.getByRole("alert")).toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: /send message/i })); + + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + }); }); diff --git a/widget/src/components/__tests__/ChatWindow.voice.test.tsx b/widget/src/components/__tests__/ChatWindow.voice.test.tsx index 4cf57b8..6ca0571 100644 --- a/widget/src/components/__tests__/ChatWindow.voice.test.tsx +++ b/widget/src/components/__tests__/ChatWindow.voice.test.tsx @@ -11,6 +11,7 @@ import { import { installFakeSpeechRecognition, uninstallFakeSpeechRecognition, + latestRecognition, } from "../../test-utils/fakeSpeechRecognition"; const VOICE: ResolvedVoiceConfig = { @@ -151,7 +152,8 @@ describe("ChatWindow read-aloud", () => { fireEvent.click(screen.getByRole("button", { name: "Read aloud" })); fireEvent.click(screen.getByRole("button", { name: "Pause reading" })); - expect(synth.paused).toBe(true); + // Chrome and Safari confirm a pause asynchronously. + act(() => synth.confirmPause()); fireEvent.click(screen.getByRole("button", { name: "Resume reading" })); expect(synth.paused).toBe(false); @@ -186,6 +188,20 @@ describe("ChatWindow read-aloud", () => { ).toHaveLength(1); }); + it("ends dictation when a reply starts being read, so the mic does not transcribe it", () => { + renderWindow([QUESTION, REPLY]); + const mic = screen.getByRole("button", { name: "Voice input" }); + fireEvent.click(mic); + const session = latestRecognition(); + expect(mic).toHaveAttribute("aria-pressed", "true"); + + fireEvent.click(screen.getByRole("button", { name: "Read aloud" })); + + expect(session.abortCalls).toBe(1); + expect(mic).toHaveAttribute("aria-pressed", "false"); + expect(synth.queuedText()).toEqual(["Plans start at $10."]); + }); + it("falls silent when the visitor starts dictating, so the mic does not hear it", () => { renderWindow([QUESTION, REPLY]); fireEvent.click(screen.getByRole("button", { name: "Read aloud" })); diff --git a/widget/src/components/__tests__/MessageSpeechControls.test.tsx b/widget/src/components/__tests__/MessageSpeechControls.test.tsx index 4ad7188..42b1219 100644 --- a/widget/src/components/__tests__/MessageSpeechControls.test.tsx +++ b/widget/src/components/__tests__/MessageSpeechControls.test.tsx @@ -1,4 +1,6 @@ +import { useState } from "react"; import { render, screen, fireEvent } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import { describe, it, expect, vi } from "vitest"; import { MessageSpeechControls, @@ -58,4 +60,73 @@ describe("MessageSpeechControls", () => { fireEvent.click(screen.getByRole("button", { name: "Stop reading" })); expect(onStop).toHaveBeenCalledTimes(1); }); + + // A removed element drops focus to , which is outside the chat + // dialog's focus trap, so Tab could then leave the dialog. + describe("keyboard focus", () => { + function Harness({ initial = "idle" as MessageSpeechState }) { + const [state, setState] = useState(initial); + return ( + <> + setState("speaking")} + onPause={() => setState("paused")} + onResume={() => setState("speaking")} + onStop={() => setState("idle")} + /> + + + ); + } + + it("stays on the control as Read aloud becomes Pause, then Resume", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByRole("button", { name: "Read aloud" })); + expect( + screen.getByRole("button", { name: "Pause reading" }), + ).toHaveFocus(); + + await user.click(screen.getByRole("button", { name: "Pause reading" })); + expect( + screen.getByRole("button", { name: "Resume reading" }), + ).toHaveFocus(); + }); + + it("moves to Read aloud when Stop is pressed and disappears", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByRole("button", { name: "Stop reading" })); + + expect(screen.getByRole("button", { name: "Read aloud" })).toHaveFocus(); + }); + + it("moves to Read aloud when playback ends by itself with focus on Stop", () => { + render(); + screen.getByRole("button", { name: "Stop reading" }).focus(); + + // fireEvent does not move focus, unlike a real click on this button. + fireEvent.click(screen.getByRole("button", { name: "playback ends" })); + + expect(screen.getByRole("button", { name: "Read aloud" })).toHaveFocus(); + }); + + it("does not steal focus when playback ends while the visitor is elsewhere", async () => { + const user = userEvent.setup(); + render(); + await user.click(screen.getByRole("button", { name: "Stop reading" })); + await user.click(screen.getByRole("button", { name: "Read aloud" })); + + // Focus leaves the controls for another element, then playback ends. + await user.click(screen.getByRole("button", { name: "playback ends" })); + + expect( + screen.getByRole("button", { name: "playback ends" }), + ).toHaveFocus(); + }); + }); }); diff --git a/widget/src/components/__tests__/VoiceInputButton.test.tsx b/widget/src/components/__tests__/VoiceInputButton.test.tsx index ae6683d..0acf0a2 100644 --- a/widget/src/components/__tests__/VoiceInputButton.test.tsx +++ b/widget/src/components/__tests__/VoiceInputButton.test.tsx @@ -44,7 +44,7 @@ describe("VoiceInputButton", () => { it("starts on a click when idle", () => { const { button, onStart, onStop } = setup(); fireEvent.click(button); - expect(onStart).toHaveBeenCalledTimes(1); + expect(onStart).toHaveBeenCalledExactlyOnceWith({ held: false }); expect(onStop).not.toHaveBeenCalled(); }); @@ -67,7 +67,7 @@ describe("VoiceInputButton", () => { const { button, onStart, onStop } = setup({ mode: "hold" }); fireEvent.pointerDown(button, { button: 0 }); - expect(onStart).toHaveBeenCalledTimes(1); + expect(onStart).toHaveBeenCalledExactlyOnceWith({ held: true }); expect(onStop).not.toHaveBeenCalled(); fireEvent.pointerUp(button); @@ -105,7 +105,7 @@ describe("VoiceInputButton", () => { fireEvent.keyDown(button, { key }); fireEvent.keyDown(button, { key, repeat: true }); - expect(onStart).toHaveBeenCalledTimes(1); + expect(onStart).toHaveBeenCalledExactlyOnceWith({ held: true }); fireEvent.keyUp(button, { key }); expect(onStop).toHaveBeenCalledTimes(1); @@ -128,7 +128,9 @@ describe("VoiceInputButton", () => { it("toggles on a bare click, which is how screen readers and switches activate it", () => { const { button, onStart } = setup({ mode: "hold" }); fireEvent.click(button); - expect(onStart).toHaveBeenCalledTimes(1); + // Not a hold: nothing guarantees a release, so the parent must not keep + // the microphone open through pauses the way it does for a real hold. + expect(onStart).toHaveBeenCalledExactlyOnceWith({ held: false }); }); it("stops on a bare click while listening", () => { diff --git a/widget/src/hooks/__tests__/useSpeechRecognition.test.ts b/widget/src/hooks/__tests__/useSpeechRecognition.test.ts index 584bbb3..20ad235 100644 --- a/widget/src/hooks/__tests__/useSpeechRecognition.test.ts +++ b/widget/src/hooks/__tests__/useSpeechRecognition.test.ts @@ -322,4 +322,95 @@ describe("useSpeechRecognition", () => { expect(onEnd).not.toHaveBeenCalled(); }); }); + + describe("telling the caller what happened", () => { + it("start() reports whether a session actually began", () => { + const { result } = setup(); + let first = false; + let second = true; + + act(() => { + first = result.current.start(); + }); + act(() => { + second = result.current.start(); + }); + + expect(first).toBe(true); + // Refused: the first session is still open. + expect(second).toBe(false); + }); + + it("start() reports false when the engine refuses", () => { + FakeSpeechRecognition.startError = new Error("InvalidStateError"); + const { result } = setup(); + let started = true; + + act(() => { + started = result.current.start(); + }); + + expect(started).toBe(false); + }); + + it("stop() reports whether there was a session to stop", () => { + const { result } = setup(); + let idle = true; + let open = false; + + act(() => { + idle = result.current.stop(); + }); + act(() => result.current.start()); + act(() => latestRecognition().emitStart()); + act(() => { + open = result.current.stop(); + }); + + expect(idle).toBe(false); + expect(open).toBe(true); + }); + }); + + describe("engines that misbehave", () => { + it("treats an abort by the engine (another tab took the mic) as a failed session, silently", () => { + const { result, onEnd, onError } = setup(); + act(() => result.current.start()); + const session = latestRecognition(); + act(() => + session.emitResult([{ transcript: "half a thought", isFinal: true }]), + ); + + act(() => session.emitError("aborted")); + act(() => session.emitEnd()); + + expect(onError).not.toHaveBeenCalled(); + // Not a normal end, so nothing may be auto-submitted from it. + expect(onEnd).not.toHaveBeenCalled(); + expect(result.current.isListening).toBe(false); + }); + + it("stops listening on an error even if the engine never sends end", () => { + const { result } = setup(); + act(() => result.current.start()); + const session = latestRecognition(); + + act(() => session.emitError("network")); + + expect(result.current.isListening).toBe(false); + expect(session.abortCalls).toBe(1); + }); + + it("still stops, rather than aborts, an engine that delivers results without ever firing start", () => { + const { result } = setup(); + act(() => result.current.start()); + const session = latestRecognition(); + act(() => session.emitResult([{ transcript: "hi", isFinal: false }])); + + act(() => result.current.stop()); + + expect(session.stopCalls).toBe(1); + expect(session.abortCalls).toBe(0); + }); + }); }); diff --git a/widget/src/hooks/__tests__/useSpeechSynthesis.test.ts b/widget/src/hooks/__tests__/useSpeechSynthesis.test.ts index e17e0be..bbe7a9d 100644 --- a/widget/src/hooks/__tests__/useSpeechSynthesis.test.ts +++ b/widget/src/hooks/__tests__/useSpeechSynthesis.test.ts @@ -3,16 +3,17 @@ import { describe, it, expect, beforeEach, afterEach } from "vitest"; import { useSpeechSynthesis } from "../useSpeechSynthesis"; import { FakeSpeechSynthesis, + FakeSpeechSynthesisUtterance, installFakeSpeechSynthesis, uninstallFakeSpeechSynthesis, } from "../../test-utils/fakeSpeechSynthesis"; -// Forty characters, so the default 200-character chunking is easy to reason -// about by hand: four sentences (163 chars) fit a chunk, five (204) do not. +// Forty characters, so the default 150-character chunking is easy to reason +// about by hand: three sentences (122 chars) fit a chunk, four (163) do not. const SENTENCE = "Alpha beta gamma delta epsilon zeta eta."; const LONG_REPLY = Array(6).fill(SENTENCE).join(" "); -const FIRST_CHUNK = Array(4).fill(SENTENCE).join(" "); -const SECOND_CHUNK = Array(2).fill(SENTENCE).join(" "); +const FIRST_CHUNK = Array(3).fill(SENTENCE).join(" "); +const SECOND_CHUNK = Array(3).fill(SENTENCE).join(" "); describe("useSpeechSynthesis", () => { let synth: FakeSpeechSynthesis; @@ -72,19 +73,50 @@ describe("useSpeechSynthesis", () => { expect(result.current.activeId).toBeNull(); }); - it("pauses and resumes without losing its place", () => { + it("shows paused only once the engine confirms it, as Chrome and Safari do", () => { const { result } = renderHook(() => useSpeechSynthesis("en-US")); act(() => result.current.speak("m1", LONG_REPLY)); act(() => result.current.pause()); - expect(synth.paused).toBe(true); + expect(result.current.isPaused).toBe(false); + + act(() => synth.confirmPause()); expect(result.current.isPaused).toBe(true); expect(result.current.activeId).toBe("m1"); + }); + + it("shows paused at once where the engine pauses synchronously, as Firefox does", () => { + synth.pauseBehavior = "sync"; + const { result } = renderHook(() => useSpeechSynthesis("en-US")); + act(() => result.current.speak("m1", LONG_REPLY)); + + act(() => result.current.pause()); + + expect(result.current.isPaused).toBe(true); + }); + + it("resumes the same reply", () => { + const { result } = renderHook(() => useSpeechSynthesis("en-US")); + act(() => result.current.speak("m1", LONG_REPLY)); + act(() => result.current.pause()); + act(() => synth.confirmPause()); act(() => result.current.resume()); + expect(synth.paused).toBe(false); expect(result.current.isPaused).toBe(false); - expect(synth.queuedText()).toEqual([FIRST_CHUNK, SECOND_CHUNK]); + expect(result.current.activeId).toBe("m1"); + }); + + it("ignores a pause event from a reply it has already replaced", () => { + const { result } = renderHook(() => useSpeechSynthesis("en-US")); + act(() => result.current.speak("m1", "First reply.")); + const stale = synth.queue[0]; + act(() => result.current.speak("m2", "Second reply.")); + + act(() => stale.onpause?.({})); + + expect(result.current.isPaused).toBe(false); }); it("stops outright where pause() ends the utterance (Android), instead of reading on", () => { @@ -100,10 +132,23 @@ describe("useSpeechSynthesis", () => { expect(result.current.isPaused).toBe(false); }); - it("cancel() empties the queue and returns to idle, even while paused", () => { + it("stops outright where pause() fails the utterance instead (Android), rather than reading on", () => { + synth.pauseBehavior = "error-utterance"; const { result } = renderHook(() => useSpeechSynthesis("en-US")); act(() => result.current.speak("m1", LONG_REPLY)); + expect(synth.queue).toHaveLength(2); + act(() => result.current.pause()); + + expect(synth.queue).toHaveLength(0); + expect(result.current.activeId).toBeNull(); + }); + + it("stops while paused without leaving the engine paused, or the next reply would queue in silence", () => { + const { result } = renderHook(() => useSpeechSynthesis("en-US")); + act(() => result.current.speak("m1", LONG_REPLY)); + act(() => result.current.pause()); + act(() => synth.confirmPause()); expect(result.current.isPaused).toBe(true); act(() => result.current.cancel()); @@ -111,6 +156,18 @@ describe("useSpeechSynthesis", () => { expect(synth.queue).toHaveLength(0); expect(result.current.activeId).toBeNull(); expect(result.current.isPaused).toBe(false); + // cancel() alone does not clear the engine's paused state (per the spec). + expect(synth.paused).toBe(false); + }); + + it("un-pauses an engine that was left paused before it reads", () => { + synth.paused = true; + const { result } = renderHook(() => useSpeechSynthesis("en-US")); + + act(() => result.current.speak("m1", "Hello.")); + + expect(synth.paused).toBe(false); + expect(synth.queuedText()).toEqual(["Hello."]); }); it("replaces the reply being read when another one is started", () => { @@ -154,4 +211,44 @@ describe("useSpeechSynthesis", () => { expect(synth.queue).toHaveLength(0); }); + + describe("speech it does not own", () => { + function hostPageSpeaks() { + synth.speak(new FakeSpeechSynthesisUtterance("Host page narration.")); + } + + it("is left alone on unmount by a reader that never spoke, as when voice is switched off", () => { + hostPageSpeaks(); + const { unmount } = renderHook(() => useSpeechSynthesis("en-US")); + + unmount(); + + expect(synth.cancelCalls).toBe(0); + expect(synth.queuedText()).toEqual(["Host page narration."]); + }); + + it("is left alone by cancel() when this reader has nothing playing", () => { + hostPageSpeaks(); + const { result } = renderHook(() => useSpeechSynthesis("en-US")); + + act(() => result.current.cancel()); + + expect(synth.cancelCalls).toBe(0); + expect(synth.queuedText()).toEqual(["Host page narration."]); + }); + + it("survives a second widget taking over: the first resets without cancelling the engine again", () => { + const first = renderHook(() => useSpeechSynthesis("en-US")); + const second = renderHook(() => useSpeechSynthesis("en-US")); + act(() => first.result.current.speak("m1", LONG_REPLY)); + act(() => second.result.current.speak("m2", "Second reply.")); + + // The first widget's utterances now get their late "interrupted" errors. + act(() => synth.flushCancelled()); + + expect(first.result.current.activeId).toBeNull(); + expect(second.result.current.activeId).toBe("m2"); + expect(synth.queuedText()).toEqual(["Second reply."]); + }); + }); }); diff --git a/widget/src/hooks/useSpeechRecognition.ts b/widget/src/hooks/useSpeechRecognition.ts index f1a6ded..c59a46e 100644 --- a/widget/src/hooks/useSpeechRecognition.ts +++ b/widget/src/hooks/useSpeechRecognition.ts @@ -24,9 +24,17 @@ export interface UseSpeechRecognitionResult { isListening: boolean; /** True while the engine reports hearing speech. Drives the level indicator. */ isSpeechDetected: boolean; - start: () => void; - /** Finish the session and keep what was heard. */ - stop: () => void; + /** + * Open a session. Returns whether one actually began: `false` when the + * browser is unsupported, the engine refused, or a previous session is still + * winding down. + */ + start: () => boolean; + /** + * Finish the session and keep what was heard. Returns whether there was a + * session to stop. + */ + stop: () => boolean; /** Drop the session; nothing more is reported for it. */ abort: () => void; } @@ -126,19 +134,17 @@ export function useSpeechRecognition( const stop = useCallback(() => { const session = sessionRef.current; - if (!session) return; + if (!session) return false; // stop() on an engine still waiting for microphone permission would let // a later "Allow" open the mic with nobody at the button. Abort instead. - if (!engineStartedRef.current) { - abort(); - return; - } - session.stop(); + if (!engineStartedRef.current) abort(); + else session.stop(); + return true; }, [abort]); const start = useCallback(() => { const Recognition = getSpeechRecognition(); - if (!Recognition || sessionRef.current) return; + if (!Recognition || sessionRef.current) return false; const session = new Recognition(); session.lang = optionsRef.current.lang; @@ -146,12 +152,16 @@ export function useSpeechRecognition( session.continuous = false; let finalTranscript = ""; - let failed = false; - session.onstart = () => { + // Some engines skip `start`; hearing anything proves the mic is open too. + const markStarted = () => { engineStartedRef.current = true; }; - session.onspeechstart = () => setIsSpeechDetected(true); + session.onstart = markStarted; + session.onspeechstart = () => { + markStarted(); + setIsSpeechDetected(true); + }; session.onspeechend = () => setIsSpeechDetected(false); session.onresult = (event) => { let finals = ""; @@ -162,18 +172,26 @@ export function useSpeechRecognition( else interim += result[0].transcript; } finalTranscript = finals; + markStarted(); setIsSpeechDetected(true); optionsRef.current.onTranscript?.(finals + interim); }; session.onerror = (event) => { - // Chrome reports our own abort() this way; it is not a failure. - if (event.error === "aborted") return; - failed = true; - optionsRef.current.onError?.(toErrorKind(event.error)); + // Our own abort() detaches these handlers first, so an "aborted" that + // arrives here came from the engine: another tab or widget took the + // microphone. Nothing to tell the visitor, but it was not a normal end, + // so onEnd (and any auto-submit hanging off it) must not run. + if (event.error !== "aborted") { + optionsRef.current.onError?.(toErrorKind(event.error)); + } + // The spec promises an `end` after every error. Do not depend on it: + // an engine that forgets would leave the UI listening forever. + release(session); + session.abort(); }; session.onend = () => { release(session); - if (!failed) optionsRef.current.onEnd?.(finalTranscript); + optionsRef.current.onEnd?.(finalTranscript); }; sessionRef.current = session; @@ -185,7 +203,9 @@ export function useSpeechRecognition( } catch { release(session); optionsRef.current.onError?.("unavailable"); + return false; } + return true; }, [release]); // Leaving the page or closing the chat must never leave the mic open. diff --git a/widget/src/hooks/useSpeechSynthesis.ts b/widget/src/hooks/useSpeechSynthesis.ts index 8adb84a..e680a89 100644 --- a/widget/src/hooks/useSpeechSynthesis.ts +++ b/widget/src/hooks/useSpeechSynthesis.ts @@ -49,9 +49,19 @@ export function useSpeechSynthesis(lang: string): UseSpeechSynthesisResult { setIsPaused(false); }, []); + // `speechSynthesis` is one global, shared with the host page and with any + // other widget on it. Only ever cancel speech this reader started: a reader + // that never spoke (voice switched off, say) must not silence the page when + // the chat closes. const cancel = useCallback(() => { + const ownsPlayback = utterancesRef.current.length > 0; reset(); - getSpeechSynthesis()?.cancel(); + const synth = getSpeechSynthesis(); + if (!ownsPlayback || !synth) return; + synth.cancel(); + // Per the spec cancel() leaves a paused engine paused, and the next reply + // would then queue in silence. + if (synth.paused) synth.resume(); }, [reset]); const speak = useCallback( @@ -63,6 +73,7 @@ export function useSpeechSynthesis(lang: string): UseSpeechSynthesisResult { // an earlier cutoff. An idle engine is left alone: in some browsers a // cancel() immediately before speak() swallows the new utterance. if (synth.speaking || synth.pending) synth.cancel(); + if (synth.paused) synth.resume(); const chunks = chunkSpeechText(text); if (chunks.length === 0) return; @@ -73,8 +84,23 @@ export function useSpeechSynthesis(lang: string): UseSpeechSynthesisResult { utterancesRef.current = chunks.map((chunk, index) => { const utterance = new SpeechSynthesisUtterance(chunk); utterance.lang = lang; - utterance.onerror = () => { - if (isCurrent()) cancel(); + utterance.onerror = (event) => { + if (!isCurrent()) return; + // "interrupted" / "canceled" with no pause of ours pending means + // someone else cancelled the engine: another widget, or the host + // page. The queue is theirs now, so forget ours and leave the engine + // alone. Cancelling again here would kill their speech. + const takenOver = + (event.error === "interrupted" || event.error === "canceled") && + !pauseRequestedRef.current; + if (takenOver) reset(); + else cancel(); + }; + utterance.onpause = () => { + if (isCurrent()) setIsPaused(true); + }; + utterance.onresume = () => { + if (isCurrent()) setIsPaused(false); }; utterance.onend = () => { if (!isCurrent()) return; @@ -97,7 +123,10 @@ export function useSpeechSynthesis(lang: string): UseSpeechSynthesisResult { if (!synth) return; pauseRequestedRef.current = true; synth.pause(); - setIsPaused(synth.paused); + // Firefox flips `paused` synchronously. Chrome and Safari flip it only + // once the engine confirms, which arrives as the utterance's `pause` + // event, handled above. + if (synth.paused) setIsPaused(true); }, []); const resume = useCallback(() => { diff --git a/widget/src/test-utils/fakeSpeechSynthesis.ts b/widget/src/test-utils/fakeSpeechSynthesis.ts index 6e85a39..530134a 100644 --- a/widget/src/test-utils/fakeSpeechSynthesis.ts +++ b/widget/src/test-utils/fakeSpeechSynthesis.ts @@ -37,10 +37,17 @@ export class FakeSpeechSynthesis { queue: FakeSpeechSynthesisUtterance[] = []; cancelCalls = 0; /** - * `"end-utterance"` reproduces Android, where `pause()` ends the current - * utterance instead of pausing it and never sets `paused`. + * How `pause()` behaves, which differs by browser: + * - `"async"` (the default; Chrome and Safari): nothing changes until the + * engine confirms. The test plays the engine with `confirmPause()`, which + * sets `paused` and fires the utterance's `pause` event. + * - `"sync"` (Firefox): `paused` is set immediately. + * - `"end-utterance"` / `"error-utterance"` (Android): `pause()` ends the + * current utterance instead, by `end` or by `error`, and never sets + * `paused`. */ - pauseBehavior: "pause" | "end-utterance" = "pause"; + pauseBehavior: "async" | "sync" | "end-utterance" | "error-utterance" = + "async"; private cancelled: FakeSpeechSynthesisUtterance[] = []; @@ -49,19 +56,32 @@ export class FakeSpeechSynthesis { this.sync(); } + /** Per the spec, cancelling does not change the paused state. */ cancel(): void { this.cancelCalls += 1; this.cancelled.push(...this.queue); this.queue = []; - this.paused = false; this.sync(); } pause(): void { - if (this.pauseBehavior === "end-utterance") { - this.finishCurrent(); - return; + switch (this.pauseBehavior) { + case "end-utterance": + this.finishCurrent(); + return; + case "error-utterance": + this.failCurrent("interrupted"); + return; + case "sync": + this.confirmPause(); + return; + case "async": + return; } + } + + /** The engine reports that the current utterance is now paused. */ + confirmPause(): void { this.paused = true; this.queue[0]?.onpause?.({}); } diff --git a/widget/src/utils/__tests__/stripAnnouncementFormatting.test.ts b/widget/src/utils/__tests__/stripAnnouncementFormatting.test.ts index 6058a5d..6c7f587 100644 --- a/widget/src/utils/__tests__/stripAnnouncementFormatting.test.ts +++ b/widget/src/utils/__tests__/stripAnnouncementFormatting.test.ts @@ -24,6 +24,18 @@ describe("stripAnnouncementFormatting", () => { ), ).toBe("Read pmds.info for more."); }); + it("keeps the sentence end that follows a URL", () => { + expect( + stripAnnouncementFormatting( + "See https://example.com/pricing. Then call us.", + ), + ).toBe("See example.com. Then call us."); + }); + it("keeps other punctuation that trails a URL", () => { + expect( + stripAnnouncementFormatting("Is it https://example.com/faq?! Yes."), + ).toBe("Is it example.com?! Yes."); + }); it("leaves plain text unchanged", () => { expect(stripAnnouncementFormatting("Hello there!")).toBe("Hello there!"); }); diff --git a/widget/src/utils/__tests__/voice.test.ts b/widget/src/utils/__tests__/voice.test.ts index 3e437ca..6c74c26 100644 --- a/widget/src/utils/__tests__/voice.test.ts +++ b/widget/src/utils/__tests__/voice.test.ts @@ -108,6 +108,15 @@ describe("resolveVoiceConfig", () => { ).toBeNull(); }); + it("stays off for anything that is not true or an options object", () => { + // A microphone feature must fail closed. `voice: "false"` from a CMS + // template is a truthy string, and must not switch the mic on. + expect(resolveVoiceConfig("false" as never, "en")).toBeNull(); + expect(resolveVoiceConfig("true" as never, "en")).toBeNull(); + expect(resolveVoiceConfig(1 as never, "en")).toBeNull(); + expect(resolveVoiceConfig([] as never, "en")).toBeNull(); + }); + it("falls back to toggle mode for an unknown mode", () => { expect(resolveVoiceConfig({ mode: "shout" as never }, "en")).toMatchObject({ mode: "toggle", @@ -177,8 +186,26 @@ describe("chunkSpeechText", () => { ]); }); - it("never splits inside a word, even one longer than the limit", () => { - expect(chunkSpeechText("abcdefghij", 5)).toEqual(["abcdefghij"]); + it("splits a token longer than the limit as a last resort", () => { + // Unbroken runs this long are not words in spaced scripts, but they are + // ordinary sentences in Chinese or Japanese written without punctuation. + expect(chunkSpeechText("abcdefghijkl", 5)).toEqual([ + "abcde", + "fghij", + "kl", + ]); + }); + + it("lets text after an over-long token share its last piece", () => { + expect(chunkSpeechText("abcdefg hi", 5)).toEqual(["abcde", "fg hi"]); + }); + + it("breaks Chinese and Japanese text at its own sentence marks, which have no space after them", () => { + expect(chunkSpeechText("こんにちは。元気ですか？はい！", 8)).toEqual([ + "こんにちは。", + "元気ですか？", + "はい！", + ]); }); it("does not treat a dot inside a word as a sentence end", () => { @@ -199,15 +226,17 @@ describe("chunkSpeechText", () => { ]); }); - it("keeps every chunk short enough to dodge Chrome's long-utterance cutoff", () => { + it("keeps every utterance to roughly ten seconds of speech, well inside Chrome's cutoff", () => { const sentence = "Alpha beta gamma delta epsilon zeta eta."; const text = Array(6).fill(sentence).join(" "); const chunks = chunkSpeechText(text); - expect(chunks.length).toBeGreaterThan(1); - for (const chunk of chunks) expect(chunk.length).toBeLessThanOrEqual(200); - expect(chunks.join(" ")).toBe(text); + // Forty-character sentences: three fit in 150 characters, four do not. + expect(chunks).toEqual([ + Array(3).fill(sentence).join(" "), + Array(3).fill(sentence).join(" "), + ]); }); }); diff --git a/widget/src/utils/stripAnnouncementFormatting.ts b/widget/src/utils/stripAnnouncementFormatting.ts index a24c09b..cec98df 100644 --- a/widget/src/utils/stripAnnouncementFormatting.ts +++ b/widget/src/utils/stripAnnouncementFormatting.ts @@ -12,8 +12,12 @@ export function stripAnnouncementFormatting(content: string): string { .replace(BOLD, "$1") .replace(ITALIC, "$1") .replace(URL_PATTERN, (match) => { + // Punctuation that ends the sentence is not part of the link. Keep it, + // or "see example.com/pricing. Then..." loses its sentence break. + const trailing = match.match(/[.,;:!?'"]+$/)?.[0] ?? ""; + const url = trailing ? match.slice(0, -trailing.length) : match; try { - return new URL(match).hostname; + return new URL(url).hostname + trailing; } catch { return match; } diff --git a/widget/src/utils/voice.ts b/widget/src/utils/voice.ts index 1b77c2a..0aaebb3 100644 --- a/widget/src/utils/voice.ts +++ b/widget/src/utils/voice.ts @@ -97,7 +97,12 @@ export function resolveVoiceConfig( input: boolean | VoiceOptions | undefined | null, locale: LocaleCode, ): ResolvedVoiceConfig | null { - if (!input) return null; + // Fail closed: this switches a microphone on. Only `true` or an options + // object enables it, so a stray truthy value such as the string "false" + // from a template cannot. + const isOptions = + typeof input === "object" && input !== null && !Array.isArray(input); + if (input !== true && !isOptions) return null; const options = input === true ? {} : input; const config: ResolvedVoiceConfig = { input: options.input ?? true, @@ -162,7 +167,11 @@ export function joinDictation( return (typed ? `${typed} ${heard}` : heard).slice(0, maxLength); } -/** Split after sentence punctuation that is followed by whitespace, and at newlines. */ +/** + * Split at newlines, after `.` `!` `?` when whitespace follows (so the dot in + * "example.com" or "$9.99" is not a break), and after the full-width marks + * Chinese and Japanese use, which have no space after them. + */ function splitSentences(text: string): string[] { const pieces: string[] = []; let start = 0; @@ -170,6 +179,9 @@ function splitSentences(text: string): string[] { const ch = text[i]; const endsSentence = ch === "\n" || + ch === "。" || + ch === "！" || + ch === "？" || ((ch === "." || ch === "!" || ch === "?") && (i + 1 === text.length || /\s/.test(text[i + 1]))); if (endsSentence) { @@ -185,10 +197,12 @@ function splitSentences(text: string): string[] { * Break text into utterance-sized chunks for `speechSynthesis`. Chrome stops * a long utterance after roughly 15 seconds when a network voice is in use, * so text is queued as several short utterances instead: whole sentences - * packed up to `maxLength`, an over-long sentence split between words. Words - * are never split. + * packed up to `maxLength`, an over-long sentence split between words, and a + * single over-long token cut as a last resort. The default is about ten + * seconds of speech, leaving room for numbers, which take far longer to say + * than to write. */ -export function chunkSpeechText(text: string, maxLength = 200): string[] { +export function chunkSpeechText(text: string, maxLength = 150): string[] { const chunks: string[] = []; let current = ""; @@ -208,8 +222,14 @@ export function chunkSpeechText(text: string, maxLength = 200): string[] { current = piece; continue; } - for (const word of piece.trim().split(/\s+/)) { + for (let word of piece.trim().split(/\s+/)) { if (current && `${current} ${word}`.length > maxLength) flush(); + // Not a word in a spaced script, but an ordinary unpunctuated sentence + // in Chinese or Japanese. Left whole it would hit the cutoff. + while (word.length > maxLength) { + chunks.push(word.slice(0, maxLength)); + word = word.slice(maxLength); + } current = current ? `${current} ${word}` : word; } }