diff --git a/.changeset/clear-voice-controls.md b/.changeset/clear-voice-controls.md
new file mode 100644
index 00000000000..fc0f333b2d9
--- /dev/null
+++ b/.changeset/clear-voice-controls.md
@@ -0,0 +1,5 @@
+---
+"@hashintel/petrinaut": patch
+---
+
+Improve Voice controls with provider-specific audio options, independent microphone and speaker state, distinct Stop and End actions, and a responsive compact dock.
diff --git a/apps/petrinaut-website/MISSION.md b/apps/petrinaut-website/MISSION.md
index 4478f01f342..ecc8929624c 100644
--- a/apps/petrinaut-website/MISSION.md
+++ b/apps/petrinaut-website/MISSION.md
@@ -1,29 +1,38 @@
-# GPT-Live capture mitigation
+# Improve Brunch Voice controls
The child branch's sole execution authority is the
-[Brunch mission](../../libs/@hashintel/brunch-agent/MISSION.md).
-This file is a pointer, not a second mission.
+[Brunch mission](../../libs/@hashintel/brunch-agent/MISSION.md). This file is a
+pointer, not a second mission.
-FE-1712 permits explicit browser capture preferences, semantic VAD with medium
-eagerness on the separate Live transcription session, and provider-free checks.
-It also permits a 500 ms Speaking-indicator hold and a patient-listening Live
-instruction. Submission timing, separate finalized transcription and Realtime stay
-unchanged. The indicator does not control playback or establish turn completion.
-Acoustic benefit remains Kostandin's matched speaker/headphone witness; no
-deterministic feedback prevention or migration-readiness claim is established.
-The sole mission specifies bounded headless probe allocations and their results;
-natural human turn boundaries still require the owner witness. No new publication authority.
+FE-1722 selects the reduced Voice-control cut: one compact dock, direct
+microphone mute, one secondary audio popover and the existing conversation
+panel for output. Both Live and Realtime expose canonical Stop only while
+Brunch is submitted or streaming; End remains separate Voice teardown.
+Show/Hide conversation changes visibility only.
-The publication base is restacked FE-1664 at
-[6d188da42f](https://github.com/hashintel/hash/commit/6d188da42f86b2d6ef3d211ec55058685185e8c7).
-Its integration contract and earlier standalone comparisons are retained in the
-[future spine](../../libs/@hashintel/brunch-agent/MISSION.next.md#voice-feedback-follow-up).
-The parent has removed its superseded `PR_DESCRIPTION.md` draft; its PR body on
-GitHub is its authority. This child's local Git branch description mirrors its mission.
+Live microphone mute gates the existing shared capture track without silencing
+playback. Realtime preserves its current microphone gating. Both providers gain
+session-local speaker mute and normalized volume, reset for every new session.
+Read-full-response, repeat-question and interruption-by-speaking remain
+Realtime-only. Speaker settings do not redefine Speaking, and a
+provider-finalized partial transcript after mid-utterance mute is allowed.
-Kostandin authorizes pushing this child and opening its draft PR against FE-1664.
-The authorized conflict fix preserves the parent's consent and Thinking dock behavior
-and refreshes this draft's proof record. No other issue/PR changes, agent
-microphone access, merge or deployment are authorized. The sole provider exception
-is the bounded synthetic transcription probe specified in the mission.
-Delegation-driven invocation and transcript filtering remain deferred.
+The branch is stacked on FE-1664 at
+[`023a26b96b`](https://github.com/hashintel/hash/commit/023a26b96b51169da0acdb188697e159d001bcc0),
+the squash base incorporating merged
+[FE-1712 PR #9704](https://github.com/hashintel/hash/pull/9704). FE-1712 remains
+the protected behavior and evidence source inherited through that base. Its
+capture preferences, semantic VAD, patient listening, 500 ms output hold and
+unfinished owner-held obligations remain unchanged. Device switching, voice
+and speed selection, helmet animation and persistence remain deferred in the
+[future spine](../../libs/@hashintel/brunch-agent/MISSION.next.md#voice-control-follow-up).
+
+FE-1722 implementation and deterministic verification are established on this
+branch through `49ffd6836a..bdc699c923`. Post-rebase evidence passed 160
+focused Petrinaut tests, 349 focused website unit tests and 5 Voice preview
+tests under network denial; the Brunch mission owns the exact proof and
+remaining limitations. Real microphone, speaker and headphone behavior remains
+unproven and owner-held. No agent microphone or provider session is authorized.
+Current owner direction authorizes the subsequent branch push and stacked draft
+PR after the committed final gate; this correction task performs neither.
+Merge, deployment, Linear writes and other tracker changes remain unauthorized.
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx
index d6b09aa88ac..5e9792f167b 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx
+++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx
@@ -23,6 +23,9 @@ import type { FlueClient, FlueConversationState } from "@flue/sdk";
import type { PetrinautAiVoiceModeContext } from "@hashintel/petrinaut/ui";
const liveConversationMocks = vi.hoisted(() => ({
+ setMicrophoneMuted: vi.fn(),
+ setSpeakerMuted: vi.fn(),
+ setSpeakerVolume: vi.fn(),
stop: vi.fn(async () => {}),
}));
@@ -33,6 +36,9 @@ vi.mock("./live-conversation", () => ({
stop: liveConversationMocks.stop,
appendCommentary: vi.fn(() => true),
appendInstructions: vi.fn(() => true),
+ setMicrophoneMuted: liveConversationMocks.setMicrophoneMuted,
+ setSpeakerMuted: liveConversationMocks.setSpeakerMuted,
+ setSpeakerVolume: liveConversationMocks.setSpeakerVolume,
})),
}));
afterEach(() => {
@@ -204,6 +210,8 @@ test("reuses setup and reports failure to the host dock and notification surface
phase: "listening",
microphoneLevel: 0,
microphoneMuted: false,
+ speakerMuted: false,
+ speakerVolume: 1,
errorMessage: null,
notice: null,
warningMessage: null,
@@ -265,6 +273,8 @@ test("reuses setup and reports failure to the host dock and notification surface
microphoneMuted: false,
errorMessage: null,
notice: playbackNotice,
+ speakerMuted: false,
+ speakerVolume: 1,
warningMessage: null,
});
if (!props.registerVoiceModeSessionControls)
@@ -276,6 +286,9 @@ test("reuses setup and reports failure to the host dock and notification surface
"end",
"pause",
"retryPlayback",
+ "setMicrophoneMuted",
+ "setSpeakerMuted",
+ "setSpeakerVolume",
]);
controls.retryPlayback?.();
const liveSession = vi.mocked(createLiveConversation).mock.results[0]!
@@ -295,6 +308,8 @@ test("reuses setup and reports failure to the host dock and notification surface
microphoneLevel: 0,
microphoneMuted: true,
notice: null,
+ speakerMuted: false,
+ speakerVolume: 1,
warningMessage: null,
});
expect(screen.getByText(connectionError)).toBeTruthy();
@@ -306,6 +321,246 @@ test("reuses setup and reports failure to the host dock and notification surface
expect(createLiveConversation).toHaveBeenCalledOnce();
});
+test("registers truthful microphone and speaker controls for the same Live session", async () => {
+ const props = context();
+ const { rerender } = render(
+ ,
+ );
+ await start();
+ const onState = vi.mocked(createLiveConversation).mock.lastCall![0];
+ act(() =>
+ onState({
+ phase: "connected",
+ message: null,
+ activity: { microphoneLevel: 0.42, outputActive: false },
+ }),
+ );
+ if (!props.registerVoiceModeSessionControls)
+ throw new Error("Session control registration was not provided");
+ const controls = vi.mocked(props.registerVoiceModeSessionControls).mock
+ .lastCall![0];
+
+ act(() => controls.setMicrophoneMuted?.(true));
+ expect(liveConversationMocks.setMicrophoneMuted).toHaveBeenCalledWith(true);
+ expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ microphoneLevel: 0,
+ microphoneMuted: true,
+ phase: "muted",
+ }),
+ );
+
+ act(() =>
+ onState({
+ phase: "connected",
+ message: null,
+ activity: { microphoneLevel: 0.8, outputActive: true },
+ }),
+ );
+ expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ microphoneLevel: 0,
+ microphoneMuted: true,
+ phase: "speaking",
+ }),
+ );
+
+ act(() => controls.setSpeakerMuted?.(true));
+ act(() => controls.setSpeakerVolume?.(0));
+ expect(liveConversationMocks.setSpeakerMuted).toHaveBeenCalledWith(true);
+ expect(liveConversationMocks.setSpeakerVolume).toHaveBeenCalledWith(0);
+ expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ phase: "speaking",
+ speakerMuted: true,
+ speakerVolume: 0,
+ }),
+ );
+
+ act(() =>
+ onState({
+ phase: "connected",
+ message: null,
+ activity: { microphoneLevel: 0.8, outputActive: false },
+ }),
+ );
+ rerender(
+ ,
+ );
+ expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ microphoneMuted: true,
+ phase: "thinking",
+ speakerMuted: true,
+ speakerVolume: 0,
+ }),
+ );
+
+ act(() => controls.setMicrophoneMuted?.(false));
+ expect(liveConversationMocks.setMicrophoneMuted).toHaveBeenLastCalledWith(
+ false,
+ );
+});
+
+test("ignores media controls without a usable Live session", async () => {
+ const props = context();
+ render();
+ if (!props.registerVoiceModeSessionControls)
+ throw new Error("Session control registration was not provided");
+ const controls = vi.mocked(props.registerVoiceModeSessionControls).mock
+ .lastCall![0];
+
+ act(() => {
+ controls.setMicrophoneMuted?.(true);
+ controls.setSpeakerMuted?.(true);
+ controls.setSpeakerVolume?.(0.4);
+ });
+
+ expect(liveConversationMocks.setMicrophoneMuted).not.toHaveBeenCalled();
+ expect(liveConversationMocks.setSpeakerMuted).not.toHaveBeenCalled();
+ expect(liveConversationMocks.setSpeakerVolume).not.toHaveBeenCalled();
+
+ await start();
+ const onState = vi.mocked(createLiveConversation).mock.lastCall![0];
+ act(() => onState({ phase: "connected", message: null }));
+ act(() =>
+ onState({
+ phase: "error",
+ message: "Live media connection ended.",
+ }),
+ );
+ liveConversationMocks.setMicrophoneMuted.mockClear();
+ liveConversationMocks.setSpeakerMuted.mockClear();
+ liveConversationMocks.setSpeakerVolume.mockClear();
+ vi.mocked(props.reportVoiceSessionState).mockClear();
+
+ act(() => {
+ controls.setMicrophoneMuted?.(true);
+ controls.setSpeakerMuted?.(true);
+ controls.setSpeakerVolume?.(0.4);
+ });
+
+ expect(liveConversationMocks.setMicrophoneMuted).not.toHaveBeenCalled();
+ expect(liveConversationMocks.setSpeakerMuted).not.toHaveBeenCalled();
+ expect(liveConversationMocks.setSpeakerVolume).not.toHaveBeenCalled();
+ expect(props.reportVoiceSessionState).not.toHaveBeenCalled();
+});
+
+test("caches speaker controls while a Live session is connecting", async () => {
+ const props = context();
+ render();
+ await start();
+ if (!props.registerVoiceModeSessionControls)
+ throw new Error("Session control registration was not provided");
+ const controls = vi.mocked(props.registerVoiceModeSessionControls).mock
+ .lastCall![0];
+ liveConversationMocks.setSpeakerMuted.mockClear();
+ liveConversationMocks.setSpeakerVolume.mockClear();
+
+ act(() => {
+ controls.setSpeakerMuted?.(true);
+ controls.setSpeakerVolume?.(0.4);
+ });
+
+ expect(liveConversationMocks.setSpeakerMuted).toHaveBeenCalledExactlyOnceWith(
+ true,
+ );
+ expect(
+ liveConversationMocks.setSpeakerVolume,
+ ).toHaveBeenCalledExactlyOnceWith(0.4);
+ expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ phase: "connecting",
+ speakerMuted: true,
+ speakerVolume: 0.4,
+ }),
+ );
+});
+
+test("ignores media controls as soon as a Live session ends", async () => {
+ const props = context();
+ render();
+ await start();
+ if (!props.registerVoiceModeSessionControls)
+ throw new Error("Session control registration was not provided");
+ const controls = vi.mocked(props.registerVoiceModeSessionControls).mock
+ .lastCall![0];
+ const onState = vi.mocked(createLiveConversation).mock.lastCall![0];
+ act(() => onState({ phase: "connected", message: null }));
+
+ await act(() => controls.end());
+ liveConversationMocks.setMicrophoneMuted.mockClear();
+ liveConversationMocks.setSpeakerMuted.mockClear();
+ liveConversationMocks.setSpeakerVolume.mockClear();
+ vi.mocked(props.reportVoiceSessionState).mockClear();
+ act(() => {
+ controls.setMicrophoneMuted?.(true);
+ controls.setSpeakerMuted?.(true);
+ controls.setSpeakerVolume?.(0.4);
+ });
+
+ expect(liveConversationMocks.setMicrophoneMuted).not.toHaveBeenCalled();
+ expect(liveConversationMocks.setSpeakerMuted).not.toHaveBeenCalled();
+ expect(liveConversationMocks.setSpeakerVolume).not.toHaveBeenCalled();
+ expect(props.reportVoiceSessionState).not.toHaveBeenCalled();
+});
+
+test("resets and applies audio defaults when a Live session restarts", async () => {
+ const props = context();
+ render();
+ await start();
+ const firstOnState = vi.mocked(createLiveConversation).mock.calls[0]![0];
+ act(() => firstOnState({ phase: "connected", message: null }));
+ if (!props.registerVoiceModeSessionControls)
+ throw new Error("Session control registration was not provided");
+ const controls = vi.mocked(props.registerVoiceModeSessionControls).mock
+ .lastCall![0];
+
+ act(() => controls.setMicrophoneMuted?.(true));
+ act(() => controls.setSpeakerMuted?.(true));
+ act(() => controls.setSpeakerVolume?.(0.25));
+ expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ microphoneMuted: true,
+ speakerMuted: true,
+ speakerVolume: 0.25,
+ }),
+ );
+ await act(() => controls.end());
+ act(() =>
+ firstOnState({
+ phase: "ended",
+ message: "Microphone and playback stopped.",
+ }),
+ );
+ liveConversationMocks.setMicrophoneMuted.mockClear();
+ liveConversationMocks.setSpeakerMuted.mockClear();
+ liveConversationMocks.setSpeakerVolume.mockClear();
+
+ await start();
+
+ expect(createLiveConversation).toHaveBeenCalledTimes(2);
+ expect(
+ liveConversationMocks.setMicrophoneMuted,
+ ).toHaveBeenCalledExactlyOnceWith(false);
+ expect(liveConversationMocks.setSpeakerMuted).toHaveBeenCalledExactlyOnceWith(
+ false,
+ );
+ expect(
+ liveConversationMocks.setSpeakerVolume,
+ ).toHaveBeenCalledExactlyOnceWith(1);
+ const secondOnState = vi.mocked(createLiveConversation).mock.calls[1]![0];
+ act(() => secondOnState({ phase: "connected", message: null }));
+ expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ microphoneMuted: false,
+ phase: "listening",
+ speakerMuted: false,
+ speakerVolume: 1,
+ }),
+ );
+});
+
test("does not show a successful prior session close on the next consent card", async () => {
render();
await start();
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx
index 0bb33d632af..e4f39ba8d4d 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx
+++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx
@@ -58,6 +58,9 @@ export const LiveConversationControl = ({
}: LiveControlsContext) => {
const [consented, setConsented] = useState(false);
const [warningMessage, setWarningMessage] = useState(null);
+ const [microphoneMuted, setMicrophoneMutedState] = useState(false);
+ const [speakerMuted, setSpeakerMutedState] = useState(false);
+ const [speakerVolume, setSpeakerVolumeState] = useState(1);
const [state, setState] = useState({
phase: "idle",
message: null,
@@ -125,11 +128,28 @@ export const LiveConversationControl = ({
);
const end = useCallback(async () => {
bridge.current?.stop();
+ sessionActive.current = false;
const closing = session.current?.stop();
setVoiceActive(false);
setConsented(false);
await closing;
}, [setVoiceActive]);
+ const setMicrophoneMuted = useCallback((muted: boolean) => {
+ if (!sessionActive.current || !session.current) return;
+ session.current.setMicrophoneMuted(muted);
+ setMicrophoneMutedState(muted);
+ }, []);
+ const setSpeakerMuted = useCallback((muted: boolean) => {
+ if (!sessionActive.current || !session.current) return;
+ session.current.setSpeakerMuted(muted);
+ setSpeakerMutedState(muted);
+ }, []);
+ const setSpeakerVolume = useCallback((volume: number) => {
+ if (!sessionActive.current || !session.current) return;
+ const clampedVolume = Math.min(1, Math.max(0, volume));
+ session.current.setSpeakerVolume(clampedVolume);
+ setSpeakerVolumeState(clampedVolume);
+ }, []);
useEffect(
() =>
subscribeToStopRequested?.(() => {
@@ -156,8 +176,17 @@ export const LiveConversationControl = ({
retryPlayback: () => {
void session.current?.retryPlayback();
},
+ setMicrophoneMuted,
+ setSpeakerMuted,
+ setSpeakerVolume,
}),
- [end, registerVoiceModeSessionControls],
+ [
+ end,
+ registerVoiceModeSessionControls,
+ setMicrophoneMuted,
+ setSpeakerMuted,
+ setSpeakerVolume,
+ ],
);
useEffect(() => {
@@ -176,12 +205,18 @@ export const LiveConversationControl = ({
: !stopped &&
(status === "submitted" || status === "streaming")
? "thinking"
- : "listening",
+ : microphoneMuted
+ ? "muted"
+ : "listening",
microphoneLevel:
- phase === "error" ? 0 : (activity?.microphoneLevel ?? 0),
- microphoneMuted: phase === "error",
+ phase === "error" || microphoneMuted
+ ? 0
+ : (activity?.microphoneLevel ?? 0),
+ microphoneMuted: phase === "error" || microphoneMuted,
errorMessage: phase === "error" ? message : null,
notice: playbackBlocked ? message : null,
+ speakerMuted,
+ speakerVolume,
warningMessage,
...(playbackBlocked ? { canRetryPlayback: true } : {}),
}
@@ -196,6 +231,9 @@ export const LiveConversationControl = ({
activity,
status,
stopped,
+ microphoneMuted,
+ speakerMuted,
+ speakerVolume,
warningMessage,
reportVoiceSessionState,
]);
@@ -231,12 +269,19 @@ export const LiveConversationControl = ({
if (!consented || phase === "stopping" || sessionActive.current) return;
sessionActive.current = true;
setConsented(false);
+ setMicrophoneMutedState(false);
+ setSpeakerMutedState(false);
+ setSpeakerVolumeState(1);
setWarningMessage(null);
setState({ phase: "connecting", message: null });
const next = createLiveConversation(
(nextState) => {
if (session.current !== next) return;
- if (nextState.phase === "ended" || nextState.phase === "error") {
+ if (
+ nextState.phase === "stopping" ||
+ nextState.phase === "ended" ||
+ nextState.phase === "error"
+ ) {
sessionActive.current = false;
}
if (
@@ -279,6 +324,9 @@ export const LiveConversationControl = ({
);
},
);
+ next.setMicrophoneMuted(false);
+ next.setSpeakerMuted(false);
+ next.setSpeakerVolume(1);
bridge.current = new LiveBrunchBridge({
submit: (input) => latest.current.submit(input),
appendCommentary: next.appendCommentary,
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts
index 2aa97f6e46b..cdb2efb88cf 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts
+++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts
@@ -7,7 +7,15 @@ beforeEach(() => {
vi.spyOn(console, "debug").mockImplementation(() => {});
});
-const setup = () => {
+const setup = ({
+ audioMuted = false,
+ audioVolume = 1,
+ inputEnabled = true,
+}: {
+ readonly audioMuted?: boolean;
+ readonly audioVolume?: number;
+ readonly inputEnabled?: boolean;
+} = {}) => {
const sent = [[], []] as [string[], string[]];
const createChannel = (events: string[]) =>
Object.assign(new EventTarget(), {
@@ -16,9 +24,15 @@ const setup = () => {
close: vi.fn(),
});
const channels = [createChannel(sent[0]), createChannel(sent[1])] as const;
- const input = Object.assign(new EventTarget(), { stop: vi.fn() });
+ const input = Object.assign(new EventTarget(), {
+ enabled: inputEnabled,
+ stop: vi.fn(),
+ });
const outputs = [{ stop: vi.fn() }, { stop: vi.fn() }];
- const stream = { getTracks: () => [input] };
+ const stream = {
+ getAudioTracks: () => [input],
+ getTracks: () => [input],
+ };
const peers = channels.map((channel, index) =>
Object.assign(new EventTarget(), {
connectionState: "new",
@@ -42,8 +56,9 @@ const setup = () => {
const audio = {
autoplay: false,
srcObject: null,
- muted: false,
+ muted: audioMuted,
paused: false,
+ volume: audioVolume,
play: vi.fn(async () => undefined),
pause: vi.fn(),
};
@@ -418,6 +433,105 @@ test("starts Live and transcription WebRTC from one consented capture and connec
});
});
+test("applies cached settings and fresh defaults to newly created Live media", async () => {
+ const cached = setup({
+ audioMuted: false,
+ audioVolume: 0.8,
+ inputEnabled: true,
+ });
+ cached.conversation.setMicrophoneMuted(true);
+ cached.conversation.setSpeakerMuted(true);
+ cached.conversation.setSpeakerVolume(0.3);
+
+ await connect(cached);
+
+ expect(cached.input.enabled).toBe(false);
+ expect(cached.audio).toMatchObject({ muted: true, volume: 0.3 });
+
+ const fresh = setup({
+ audioMuted: true,
+ audioVolume: 0.2,
+ inputEnabled: false,
+ });
+ await connect(fresh);
+
+ expect(fresh.input.enabled).toBe(true);
+ expect(fresh.audio).toMatchObject({ muted: false, volume: 1 });
+});
+
+test("mutes the one shared capture without disturbing output or finalized input", async () => {
+ const fixture = setup();
+ await connect(fixture);
+ const remoteStream = { getTracks: () => [fixture.outputs[0]!] };
+ fixture.peers[0]!.dispatchEvent(
+ Object.assign(new Event("track"), {
+ track: fixture.outputs[0]!,
+ streams: [remoteStream],
+ }),
+ );
+
+ fixture.conversation.setMicrophoneMuted(true);
+
+ expect(fixture.input.enabled).toBe(false);
+ expect(fixture.getUserMedia).toHaveBeenCalledOnce();
+ expect(fixture.peers[0]!.addTrack).toHaveBeenCalledWith(
+ fixture.input,
+ fixture.stream,
+ );
+ expect(fixture.peers[1]!.addTrack).toHaveBeenCalledWith(
+ fixture.input,
+ fixture.stream,
+ );
+ expect(fixture.input.stop).not.toHaveBeenCalled();
+ expect(fixture.audio).toMatchObject({
+ muted: false,
+ paused: false,
+ srcObject: remoteStream,
+ });
+ expect(fixture.audio.pause).not.toHaveBeenCalled();
+
+ fixture.emit(1, {
+ type: "input_audio_buffer.committed",
+ item_id: "started-before-mute",
+ previous_item_id: null,
+ });
+ const completion = {
+ type: "conversation.item.input_audio_transcription.completed",
+ item_id: "started-before-mute",
+ content_index: 0,
+ transcript: "Keep this finalized answer.",
+ };
+ fixture.emit(1, completion);
+ fixture.emit(1, completion);
+ expect(fixture.onFinalizedInput).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({ text: "Keep this finalized answer." }),
+ );
+
+ fixture.conversation.setMicrophoneMuted(false);
+ expect(fixture.input.enabled).toBe(true);
+});
+
+test("keeps speaker mute and clamped volume local and independent", async () => {
+ const fixture = setup();
+ await connect(fixture);
+ const sentBefore = fixture.sent.map((events) => [...events]);
+
+ fixture.conversation.setSpeakerVolume(2);
+ fixture.conversation.setSpeakerMuted(true);
+ fixture.conversation.setSpeakerVolume(-0.5);
+
+ expect(fixture.audio.volume).toBe(0);
+ expect(fixture.audio.muted).toBe(true);
+ expect(fixture.input.enabled).toBe(true);
+ expect(fixture.sent).toEqual(sentBefore);
+ expect(fixture.audio.pause).not.toHaveBeenCalled();
+ expect(fixture.input.stop).not.toHaveBeenCalled();
+
+ fixture.conversation.setSpeakerVolume(0.35);
+ fixture.conversation.setSpeakerMuted(false);
+ expect(fixture.audio).toMatchObject({ muted: false, volume: 0.35 });
+});
+
test("emits only completed transcripts in committed provider order and deduplicates identical events", async () => {
const fixture = setup();
await connect(fixture);
@@ -766,6 +880,25 @@ test("Stop synchronously silences playback and capture, closes both transports,
);
});
+test("late media settings cannot change a stopped session", async () => {
+ const fixture = setup();
+ await connect(fixture);
+
+ const stopped = fixture.conversation.stop();
+ expect(fixture.audio).toMatchObject({ muted: true, volume: 1 });
+ expect(fixture.input.enabled).toBe(true);
+
+ fixture.conversation.setMicrophoneMuted(true);
+ fixture.conversation.setSpeakerMuted(false);
+ fixture.conversation.setSpeakerVolume(0.2);
+
+ expect(fixture.input.enabled).toBe(true);
+ expect(fixture.audio).toMatchObject({ muted: true, volume: 1 });
+ expect(fixture.audio.pause).toHaveBeenCalledOnce();
+ fixture.emit(0, { type: "session.closed" });
+ await stopped;
+});
+
test("a failure on either media connection stops both and remote audio remains native and unbuffered", async () => {
const fixture = setup();
await connect(fixture);
@@ -1048,6 +1181,8 @@ test("telemetry shows activity but silence and late samples never settle or revi
streams: [fixture.stream],
}),
);
+ fixture.conversation.setSpeakerMuted(true);
+ fixture.conversation.setSpeakerVolume(0);
await vi.advanceTimersByTimeAsync(100);
expect(fixture.onState.mock.lastCall?.[0].activity).toEqual({
microphoneLevel: 0.24,
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts
index 454bcb1f0a1..dd06116ac72 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts
+++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts
@@ -65,6 +65,9 @@ export const createLiveConversation = (
const emitted = new Set();
let microphone: MediaStream | undefined;
let audio: HTMLAudioElement | undefined;
+ let microphoneMuted = false;
+ let speakerMuted = false;
+ let speakerVolume = 1;
let started = false;
let playbackBlocked = false;
let playbackAttempt = 0;
@@ -116,6 +119,13 @@ export const createLiveConversation = (
);
};
+ const applyMicrophoneMuted = () => {
+ if (!microphone) return;
+ for (const audioTrack of microphone.getAudioTracks()) {
+ audioTrack.enabled = !microphoneMuted;
+ }
+ };
+
const finish = (liveConfirmed: boolean) => {
if (finished) return;
logLiveDiagnostic("session.finished", {
@@ -252,10 +262,12 @@ export const createLiveConversation = (
if (abort.signal.aborted) return;
activityTimer = setTimeout(() => void sampleActivity(), 100);
if (recoveryTimers.size > 0) return;
- const playing = audio?.srcObject && !audio.paused && !audio.muted;
+ const playing = audio?.srcObject && !audio.paused;
if (playing && outputLevel > 0.01) lastOutputActivity = Date.now();
const activity = {
- microphoneLevel: Math.round(microphoneLevel * 100) / 100,
+ microphoneLevel: microphoneMuted
+ ? 0
+ : Math.round(microphoneLevel * 100) / 100,
outputActive: Boolean(playing) && Date.now() - lastOutputActivity < 500,
};
if (
@@ -701,6 +713,8 @@ export const createLiveConversation = (
try {
audio = new Audio();
audio.autoplay = true;
+ audio.muted = speakerMuted;
+ audio.volume = speakerVolume;
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
autoGainControl: true,
@@ -713,6 +727,7 @@ export const createLiveConversation = (
return;
}
microphone = stream;
+ applyMicrophoneMuted();
stream.getTracks().forEach((track) =>
track.addEventListener(
"ended",
@@ -773,10 +788,31 @@ export const createLiveConversation = (
return true;
};
+ const setMicrophoneMuted = (muted: boolean): void => {
+ if (stopping || finished) return;
+ microphoneMuted = muted;
+ applyMicrophoneMuted();
+ };
+
+ const setSpeakerMuted = (muted: boolean): void => {
+ if (stopping || finished) return;
+ speakerMuted = muted;
+ if (audio) audio.muted = muted;
+ };
+
+ const setSpeakerVolume = (volume: number): void => {
+ if (stopping || finished) return;
+ speakerVolume = Math.min(1, Math.max(0, volume));
+ if (audio) audio.volume = speakerVolume;
+ };
+
return {
retryPlayback: playAudio,
start,
stop,
+ setMicrophoneMuted,
+ setSpeakerMuted,
+ setSpeakerVolume,
openDelegations: openDelegations as ReadonlySet,
appendCommentary: (text: string, delegationId: string | null) =>
append("commentary", text, delegationId),
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts
index 339175de4b5..d4d9a3900be 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts
+++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts
@@ -56,9 +56,11 @@ const createHarness = ({
}> = [];
const remoteAudios: Array<{
autoplay: boolean;
+ muted: boolean;
pause: ReturnType;
play: ReturnType;
srcObject: MediaStream | null;
+ volume: number;
}> = [];
const peers: Array<{
addTrack: ReturnType;
@@ -125,9 +127,11 @@ const createHarness = ({
createRemoteAudio: () => {
const audio = {
autoplay: false,
+ muted: false,
pause: vi.fn(),
play: vi.fn(async () => undefined),
srcObject: null as MediaStream | null,
+ volume: 1,
};
remoteAudios.push(audio);
return audio;
@@ -623,6 +627,42 @@ describe("OpenAIRealtimeSession", () => {
expect(harness.peers[0]!.close).toHaveBeenCalledOnce();
});
+ test("applies cached and live speaker settings only to remote audio", async () => {
+ const harness = createHarness();
+ harness.session.setSpeakerMuted(true);
+ harness.session.setSpeakerVolume(1.5);
+
+ await harness.session.connect();
+
+ expect(harness.remoteAudios[0]).toMatchObject({
+ muted: true,
+ volume: 1,
+ });
+ expect(harness.localTracks[0]!.enabled).toBe(false);
+ expect(sentEvents(harness.channels[0]!)).toEqual([]);
+
+ harness.session.setSpeakerMuted(false);
+ harness.session.setSpeakerVolume(-0.25);
+ expect(harness.remoteAudios[0]).toMatchObject({
+ muted: false,
+ volume: 0,
+ });
+ expect(harness.localTracks[0]!.enabled).toBe(false);
+ expect(sentEvents(harness.channels[0]!)).toEqual([]);
+
+ harness.session.setSpeakerMuted(true);
+ harness.session.setSpeakerVolume(0.35);
+ expect(harness.remoteAudios[0]).toMatchObject({
+ muted: true,
+ volume: 0.35,
+ });
+ expect(harness.remoteAudios[0]!.pause).not.toHaveBeenCalled();
+
+ await harness.session.disconnect();
+ expect(harness.remoteAudios[0]!.pause).toHaveBeenCalledOnce();
+ expect(harness.localTracks[0]!.stop).toHaveBeenCalledOnce();
+ });
+
test("keeps the microphone closed and rejects audio detected during playback", async () => {
const harness = createHarness();
await harness.session.connect();
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts
index 82c8c4a97a8..0fba3d3c429 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts
+++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts
@@ -85,7 +85,9 @@ export type OpenAIRealtimeSessionEvent =
interface RemoteAudio {
autoplay: boolean;
+ muted: boolean;
srcObject: HTMLMediaElement["srcObject"];
+ volume: number;
pause(): void;
play(): Promise;
}
@@ -255,6 +257,8 @@ export class OpenAIRealtimeSession {
#microphoneTrack: MediaStreamTrack | null = null;
#peerConnection: RTCPeerConnection | null = null;
#remoteAudio: RemoteAudio | null = null;
+ #speakerMuted = false;
+ #speakerVolume = 1;
#responseCreateEventId: string | null = null;
#responseTerminalSequence = 0;
#speakingResponseId: string | null = null;
@@ -317,6 +321,8 @@ export class OpenAIRealtimeSession {
this.#peerConnection = peerConnection;
this.#remoteAudio = this.#dependencies.createRemoteAudio();
this.#remoteAudio.autoplay = true;
+ this.#remoteAudio.muted = this.#speakerMuted;
+ this.#remoteAudio.volume = this.#speakerVolume;
peerConnection.ontrack = (event) => {
if (
this.#activeEpoch !== connectionEpoch ||
@@ -427,6 +433,16 @@ export class OpenAIRealtimeSession {
this.#syncMicrophoneTrack();
}
+ public setSpeakerMuted(muted: boolean): void {
+ this.#speakerMuted = muted;
+ if (this.#remoteAudio) this.#remoteAudio.muted = muted;
+ }
+
+ public setSpeakerVolume(volume: number): void {
+ this.#speakerVolume = Math.min(1, Math.max(0, volume));
+ if (this.#remoteAudio) this.#remoteAudio.volume = this.#speakerVolume;
+ }
+
public setInterruptionBySpeaking(enabled: boolean): void {
this.#interruptionBySpeaking = enabled;
this.#syncMicrophoneTrack();
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx
index 0be154c6c07..3829e73bc4d 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx
+++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx
@@ -557,6 +557,26 @@ describe("voice interview control", () => {
expect(registeredVoiceModeControls?.repeatQuestion).toBeTypeOf("function");
});
+ test("keeps idle speaker controls inert and retires them on unmount", async () => {
+ const setSpeakerMuted = vi
+ .spyOn(OpenAIRealtimeSession.prototype, "setSpeakerMuted")
+ .mockImplementation(() => {});
+ const setSpeakerVolume = vi
+ .spyOn(OpenAIRealtimeSession.prototype, "setSpeakerVolume")
+ .mockImplementation(() => {});
+ const { unmount } = render();
+ await waitFor(() => expect(registeredVoiceModeControls).toBeDefined());
+
+ registeredVoiceModeControls?.setSpeakerMuted?.(true);
+ registeredVoiceModeControls?.setSpeakerVolume?.(0.35);
+
+ expect(setSpeakerMuted).not.toHaveBeenCalled();
+ expect(setSpeakerVolume).not.toHaveBeenCalled();
+
+ unmount();
+ expect(registeredVoiceModeControls).toBeUndefined();
+ });
+
test("restarts when Voice is reselected before teardown completes", async () => {
window.localStorage.setItem(
VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY,
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx
index e73fea68f52..4439256ce37 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx
+++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx
@@ -451,6 +451,8 @@ const AvailableVoiceInterviewControl = ({
},
setMicrophoneMuted: (muted) =>
store.controller.setMicrophoneMuted(muted),
+ setSpeakerMuted: (muted) => store.controller.setSpeakerMuted(muted),
+ setSpeakerVolume: (volume) => store.controller.setSpeakerVolume(volume),
takeTurn: () => store.controller.takeTurn(),
}),
[registerVoiceModeControls, store],
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts
index 93740ae72d7..40d6e156f99 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts
+++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts
@@ -253,9 +253,11 @@ describe("controlled voice preview", () => {
} as unknown as MediaStream;
const remoteAudio = {
autoplay: false,
+ muted: false,
pause: vi.fn(),
play: vi.fn(async () => undefined),
srcObject: null as MediaStream | null,
+ volume: 1,
};
const peer = {
addTrack: vi.fn(),
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts
index 4d5033a88e7..5fa2a72fe8b 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts
+++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts
@@ -23,6 +23,8 @@ const listeningSnapshot = {
microphoneLevel: 0.24,
output: "idle",
partialText: "The request goes to dispatch",
+ speakerMuted: false,
+ speakerVolume: 1,
} satisfies VoiceTurnSnapshot;
const mapSnapshot = (overrides: Partial = {}) =>
@@ -44,6 +46,8 @@ describe("toVoiceSessionState", () => {
microphoneMuted: false,
notice: null,
phase: "listening",
+ speakerMuted: false,
+ speakerVolume: 1,
});
});
@@ -101,6 +105,37 @@ describe("toVoiceSessionState", () => {
).toMatchObject({ microphoneMuted: true, phase: "thinking" });
});
+ test("reports speaker state without changing phase priority", () => {
+ expect(mapSnapshot({ speakerMuted: true, speakerVolume: 0 })).toMatchObject(
+ {
+ phase: "listening",
+ speakerMuted: true,
+ speakerVolume: 0,
+ },
+ );
+ expect(
+ mapSnapshot({
+ microphoneEnabled: false,
+ output: "speaking",
+ speakerMuted: true,
+ speakerVolume: 0,
+ }),
+ ).toMatchObject({
+ microphoneMuted: true,
+ phase: "speaking",
+ speakerMuted: true,
+ speakerVolume: 0,
+ });
+ expect(
+ mapSnapshot({
+ connection: "error",
+ input: "paused",
+ output: "speaking",
+ speakerMuted: true,
+ }),
+ ).toMatchObject({ phase: "error" });
+ });
+
test("prefers paused over the turn phases", () => {
expect(mapSnapshot({ input: "paused", output: "speaking" })).toMatchObject({
microphoneMuted: false,
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts
index 5a713fc2440..01eb070c11a 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts
+++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts
@@ -116,5 +116,7 @@ export const toVoiceSessionState = ({
? "That answer is too long. Please try a shorter response."
: null,
phase: phaseOf(snapshot),
+ speakerMuted: snapshot.speakerMuted,
+ speakerVolume: snapshot.speakerVolume,
};
};
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts
index 44f233ef891..fa37d7ea00b 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts
+++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts
@@ -36,6 +36,8 @@ const createHarness = () => {
disconnect: vi.fn(async () => undefined),
setInterruptionBySpeaking: vi.fn(),
setMicrophoneEnabled: vi.fn(),
+ setSpeakerMuted: vi.fn(),
+ setSpeakerVolume: vi.fn(),
speakCanonical: vi.fn(),
subscribe: vi.fn(
(listener: (event: OpenAIRealtimeSessionEvent) => void) => {
@@ -2137,6 +2139,103 @@ describe("VoiceTurnController", () => {
});
});
+ test("forwards speaker settings and resets them on each new connection", async () => {
+ const harness = createHarness();
+ await harness.controller.start();
+ harness.session.setMicrophoneEnabled.mockClear();
+ harness.session.cancelOutput.mockClear();
+ harness.bridge.cancelPendingSpeech.mockClear();
+
+ harness.controller.setSpeakerMuted(true);
+ harness.controller.setSpeakerVolume(1.5);
+ harness.controller.setSpeakerVolume(-0.2);
+
+ expect(harness.session.setSpeakerMuted).toHaveBeenLastCalledWith(true);
+ expect(harness.session.setSpeakerVolume.mock.calls.slice(-2)).toEqual([
+ [1],
+ [0],
+ ]);
+ expect(harness.controller.getSnapshot()).toMatchObject({
+ speakerMuted: true,
+ speakerVolume: 0,
+ });
+ expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalled();
+ expect(harness.session.cancelOutput).not.toHaveBeenCalled();
+ expect(harness.bridge.cancelPendingSpeech).not.toHaveBeenCalled();
+
+ await harness.controller.reconnect();
+ expect(harness.session.setSpeakerMuted).toHaveBeenLastCalledWith(false);
+ expect(harness.session.setSpeakerVolume).toHaveBeenLastCalledWith(1);
+ expect(harness.controller.getSnapshot()).toMatchObject({
+ speakerMuted: false,
+ speakerVolume: 1,
+ });
+
+ harness.controller.setSpeakerMuted(true);
+ harness.controller.setSpeakerVolume(0.4);
+ await harness.controller.end();
+ await harness.controller.start();
+ expect(harness.controller.getSnapshot()).toMatchObject({
+ speakerMuted: false,
+ speakerVolume: 1,
+ });
+ });
+
+ test("ignores speaker settings unless Realtime is connected", async () => {
+ const harness = createHarness();
+
+ harness.controller.setSpeakerMuted(true);
+ harness.controller.setSpeakerVolume(0.4);
+ expect(harness.session.setSpeakerMuted).not.toHaveBeenCalled();
+ expect(harness.session.setSpeakerVolume).not.toHaveBeenCalled();
+ expect(harness.controller.getSnapshot()).toMatchObject({
+ speakerMuted: false,
+ speakerVolume: 1,
+ });
+
+ let finishConnection: ((epoch: number) => void) | undefined;
+ harness.session.connect.mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ finishConnection = resolve;
+ }),
+ );
+ const start = harness.controller.start();
+ harness.session.setSpeakerMuted.mockClear();
+ harness.session.setSpeakerVolume.mockClear();
+
+ harness.controller.setSpeakerMuted(true);
+ harness.controller.setSpeakerVolume(0.4);
+ expect(harness.session.setSpeakerMuted).not.toHaveBeenCalled();
+ expect(harness.session.setSpeakerVolume).not.toHaveBeenCalled();
+ expect(harness.controller.getSnapshot()).toMatchObject({
+ connection: "connecting",
+ speakerMuted: false,
+ speakerVolume: 1,
+ });
+
+ finishConnection?.(1);
+ await start;
+ harness.emitSession({
+ code: "network",
+ message: "Voice connection unavailable.",
+ requestId: "request-speaker",
+ type: "error",
+ });
+ harness.session.setSpeakerMuted.mockClear();
+ harness.session.setSpeakerVolume.mockClear();
+
+ harness.controller.setSpeakerMuted(true);
+ harness.controller.setSpeakerVolume(0.4);
+ expect(harness.session.setSpeakerMuted).not.toHaveBeenCalled();
+ expect(harness.session.setSpeakerVolume).not.toHaveBeenCalled();
+ expect(harness.controller.getSnapshot()).toMatchObject({
+ connection: "error",
+ speakerMuted: false,
+ speakerVolume: 1,
+ });
+ });
+
test("ignores muting while the session is paused", async () => {
const harness = createHarness();
await harness.controller.start();
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts
index d23483947ba..9d0dde05495 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts
+++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts
@@ -52,6 +52,8 @@ export interface VoiceTurnSnapshot {
readonly microphoneLevel: number;
readonly output: VoiceOutputState;
readonly partialText: string;
+ readonly speakerMuted: boolean;
+ readonly speakerVolume: number;
}
export interface VoiceLatencyEvent {
@@ -75,6 +77,8 @@ interface RealtimeSession {
disconnect(): Promise;
setInterruptionBySpeaking(enabled: boolean): void;
setMicrophoneEnabled(enabled: boolean): void;
+ setSpeakerMuted(muted: boolean): void;
+ setSpeakerVolume(volume: number): void;
speakCanonical(segments: CanonicalSpeechSegment[]): void;
subscribe(listener: (event: OpenAIRealtimeSessionEvent) => void): () => void;
}
@@ -140,6 +144,8 @@ const initialSnapshot: VoiceTurnSnapshot = {
microphoneLevel: 0,
output: "idle",
partialText: "",
+ speakerMuted: false,
+ speakerVolume: 1,
};
export class VoiceTurnController {
@@ -254,6 +260,8 @@ export class VoiceTurnController {
this.#activeSpeechOutputEnded = false;
this.#activeSpeechResponseId = null;
this.#activeSpeechResponseTerminal = false;
+ this.#session.setSpeakerMuted(false);
+ this.#session.setSpeakerVolume(1);
this.#update({
connection: "connecting",
errorCode: null,
@@ -262,6 +270,8 @@ export class VoiceTurnController {
input: "paused",
output: "idle",
partialText: "",
+ speakerMuted: false,
+ speakerVolume: 1,
});
try {
const connectionEpoch = await this.#session.connect();
@@ -399,6 +409,19 @@ export class VoiceTurnController {
this.#update({ microphoneEnabled: !muted, microphoneLevel: 0 });
}
+ public setSpeakerMuted(muted: boolean): void {
+ if (this.#snapshot.connection !== "connected") return;
+ this.#session.setSpeakerMuted(muted);
+ this.#update({ speakerMuted: muted });
+ }
+
+ public setSpeakerVolume(volume: number): void {
+ if (this.#snapshot.connection !== "connected") return;
+ const clampedVolume = Math.min(1, Math.max(0, volume));
+ this.#session.setSpeakerVolume(clampedVolume);
+ this.#update({ speakerVolume: clampedVolume });
+ }
+
public async resume(): Promise {
if (
this.#snapshot.connection !== "connected" ||
diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md
index 04648879d0f..f33fcbaee5d 100644
--- a/libs/@hashintel/brunch-agent/MISSION.md
+++ b/libs/@hashintel/brunch-agent/MISSION.md
@@ -1,466 +1,226 @@
-# Stabilize GPT-Live full-duplex voice feedback
+# Improve Brunch Voice controls
## Status
-Live capture and transcription turn-boundary mission for
-[FE-1712](https://linear.app/hash/issue/FE-1712/stabilize-gpt-live-full-duplex-voice-feedback).
-Publication base: restacked FE-1664 at
-[6d188da42f](https://github.com/hashintel/hash/commit/6d188da42f86b2d6ef3d211ec55058685185e8c7),
-not `origin/main`. The original comparison revision is
-[3cf4ca6b1f](https://github.com/hashintel/hash/commit/3cf4ca6b1f75f78cb2e086463517c02affd6ce54);
-the previous publication base was
-[006cbced7f](https://github.com/hashintel/hash/commit/006cbced7f10263f8b5f3cc305ee1ca6b722b9ce).
-Only this child's commits were rebased; FE-1664 and its PR are not modified by
-this mission. The latest restack applied cleanly; earlier restacks only conflicted
-in this mission and its website pointer. The parent's consent, Thinking dock and
-separation of temporary status notices from durable Voice warnings are preserved
-without expanding the capture-only cut. The inherited `PR_DESCRIPTION.md` draft
-remains removed.
-The separate authority commit is
-[466034cfe1](https://github.com/hashintel/hash/commit/466034cfe138e4f1f7befdcde3d16cd9f6037905).
-The capture-only implementation is prepared: its assertion failed before the
-change, and 317 targeted tests, website typechecking, lint and build now pass.
-The semantic-VAD recut is implemented locally and provider-free checks pass.
-The corrected medium probe completed all three synthetic transcripts, with the
-correction retained in one item and continuous silence verified. Earlier correction
-verdicts remain invalid because those harnesses stopped sending after the last clip.
-The accepted 500 ms Speaking-indicator hold and patient Live listening prompt are
-implemented and provider-free checks pass. Next: Kostandin tries a fresh Live session
-to judge hesitation, self-corrections and the indicator's feel. Human conversational
-latency and the physical speaker/headphone witness remain owner-held; no additional
-automatic tuning or provider run. Commit and push of this preparation are authorized.
-Acoustic benefit, natural turn boundaries and mission acceptance remain unproved.
-All three provider allocations are consumed; no further provider run. Publication
-of the prepared work is authorized below. The child is restacked on the parent's
-current head; the inherited root `PR_DESCRIPTION.md` that failed CI Markdown lint
-and formatting is gone with the parent. Repository-wide format and Markdown lint
-pass locally; the GitHub Lint workflow had not run since the parent conflict began.
+Live execution authority for
+[FE-1722](https://linear.app/hash/issue/FE-1722/improve-brunch-voice-controls).
+This branch is based directly on `origin/main` after
+[foundation PR #9745](https://github.com/hashintel/hash/pull/9745) merged.
+That foundation incorporates
+[FE-1712 PR #9704](https://github.com/hashintel/hash/pull/9704). FE-1712's
+implementation and evidence remain protected behavior; its unfinished speech,
+acoustic and recovery obligations are not accepted or replaced here.
+
+FE-1722 implementation exists on this branch across the shared-control,
+provider-control, documentation and lifecycle work reviewed in
+[PR #9747](https://github.com/hashintel/hash/pull/9747). It is the bottom entry
+of GitHub stack #9750, with follow-up
+[PR #9748](https://github.com/hashintel/hash/pull/9748) above it. The
+deterministic product proof below is established. Real microphone, speaker and
+headphone behavior remains unproven and owner-held; Kostandin owns that browser
+witness, and no microphone or provider session is authorized for an agent.
+
+The owner has authorized branch and PR maintenance for FE-1722. Merge,
+deployment and tracker writes remain unauthorized unless separately requested.
## Imperative
-Reduce the risk that assistant playback becomes fresh user input while preserving
-genuine interruptions and existing Realtime support. First determine whether
-requesting the browser processing already used by Realtime improves Live's capture.
-This is a mitigation hypothesis, not deterministic feedback-loop prevention.
-Also reduce premature single-word submissions reported by Kostandin: use semantic
-turn detection on the separate transcription session rather than silence alone.
-Reduce rapid Speaking/Thinking/Listening flicker and ask Live to allow hesitation
-and self-correction without taking over the person's unfinished thought.
+Make an active Brunch Voice session compact and predictable without changing
+who owns capture, canonical work or playback. Keep microphone mute immediately
+available, move secondary audio controls into one popover, expose the canonical
+Stop action only while Brunch is working, and use the existing conversation
+panel for visible output.
## Throughline
-Consented Start → one microphone capture → Live and `gpt-4o-transcribe` WebRTC
-sessions → committed/completed input ordering → existing composer/Flue admission
-→ Brunch settlement → frozen commentary → native Live playback. Playback can still
-re-enter capture; filtering canonical input alone would not prevent Live reacting.
-
-Protected source: FE-1664 at the pinned base above. Its complete integration,
-canonical ownership, admission, delivery and recovery contracts remain inherited
-behavior, not accepted proof. Permitted deltas are Live's `getUserMedia` preferences
-and the separate transcription session's turn-detection configuration below,
-plus the accepted indicator hold and listening-prompt recut below.
-The prior mission and future obligations remain discoverable through
-[the future spine](MISSION.next.md#voice-feedback-follow-up).
-
-Preserve the parent's newer consent and dock contracts: concise OpenAI voice and
-transcription disclosure, permission checkbox, Start voice and Cancel; stationary
-dock/viewport controls with consent above them; Voice setup before Start. Separate
-transcription still incurs additional provider usage. During submitted/streaming
-Brunch work, show Thinking only while Live is connected, not stopped and not playing
-output. Connection/error states take precedence, and playback remains Speaking.
-These are inherited local UI semantics, not progress speech, `session.thinking.append`,
-new invocation or completion proof. Parent controller tests own the transitions;
-parent desktop/mobile witnesses own the visual layout. Only the output-activity
-hold changes; status precedence and layout remain unchanged.
-
-Cold-start paths in `apps/petrinaut-website/src/main/app/voice-interview/`:
-
-- `live-conversation.ts`: change `{ audio: true }` to explicit
- `autoGainControl: true`, `echoCancellation: true`, `noiseSuppression: true`.
- Retain one capture and all connection/cleanup behavior. These are preferences,
- not required capabilities; add no unsupported-device refusal or fallback retry.
-- `live-conversation.test.ts`: extend the existing “starts Live and transcription
- WebRTC from one consented capture and connects only when both are usable” test
- with the exact requested preferences. Its existing assertions own shared track
- identity and dual readiness. Watch the new assertion fail before implementation.
-- `openai-realtime-session.ts`: reference for the preferences; leave it unchanged.
- No shared helper is warranted for this small literal.
-
-Turn-boundary recut in `apps/petrinaut-website/src/server/voice/`:
-
-- `openai-transcription-session.ts`: replace `server_vad` with
- `{ type: "semantic_vad", eagerness: "medium" }` for `gpt-4o-transcribe` only.
- No silence timer, transcript aggregation, admission change or fallback retry.
-- `openai-transcription-session.test.ts`: update the existing exact outbound
- session-body assertion first; observe failure on server VAD, then pass on the
- selected semantic configuration. Preserve model, scoped credential and raw SDP.
-- `openai-voice-policy.ts` and Realtime routes remain unchanged. PR #9619 already
- used semantic VAD with medium eagerness; this comparison now matches that setting.
-
-Patient-listening recut, relative to the website's `src/`:
-
-- `main/app/voice-interview/live-conversation.ts`: increase the existing local
- output-activity hold from 300 to 500 ms. Keep the 100 ms sampler, immediate
- activity onset and teardown/recovery behavior. This is display telemetry only,
- never a playback-completion signal or a submission delay.
-- Extend its existing telemetry test first: 400 and 499 ms stay active; the next
- sample at 500 ms clears activity. A later audio burst restarts the hold; Stop
- during the hold mutes playback immediately and late samples cannot revive it.
-- `server/voice/openai-live-session.ts`: keep sparse backchannels and add a short
- instruction to listen through thinking pauses and self-corrections rather than
- take over unfinished thoughts. Preserve Brunch authority and interruption policy.
-- Run the Live transport, controller, session-creation, transcription, bridge and
- Realtime regression suites plus website build/typecheck/lint. Review prompt
- delivery in the existing request test; a phrase-inventory test is not a speech
- oracle. No new files, mechanism, queue, gate, dependency or provider allocation.
+After the existing consented Start path connects either Live or Realtime,
+Petrinaut renders one compact Voice dock while the existing conversation panel
+continues to show the transcript and canonical Brunch output:
+
+1. The dock keeps microphone mute directly available. In Live, mute toggles the
+ one shared capture track that already feeds Live and the separate
+ transcription session; it does not mute playback or create another capture.
+ In Realtime, it preserves the existing microphone-gating behavior.
+2. One audio popover contains session-local speaker mute and normalized volume
+ for both providers. The existing read-full-response, repeat-question and
+ interruption-by-speaking controls remain Realtime-only in that popover.
+3. While canonical status is exactly `submitted` or `streaming`, both providers
+ show Stop and invoke the existing `onStop` path. Live consequently retains
+ the established `recordStopRequested()` →
+ `LiveBrunchBridge.stopResponse()` behavior: stop the current Brunch response
+ while leaving Live and transcription media connected.
+4. End remains the separate Voice-session teardown. It does not stop canonical
+ work. The existing session-collapse control is relabelled Show conversation
+ or Hide conversation and changes only conversation visibility.
+5. Status keeps the precedence connection/error → Speaking → Thinking →
+ microphone-muted → Listening. Speaker mute and volume zero do not make
+ Speaking false.
+6. Speaker mute and volume start from their ordinary unmuted/full-volume
+ defaults for every new Voice session and are never persisted.
+
+The protected source is FE-1712 at the pinned parent above. Its browser capture
+preferences, semantic VAD, patient-listening instruction and 500 ms
+output-activity hold are unchanged. The production destinations and permitted
+deltas are:
+
+- `libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/`:
+ keep the dock and existing conversation panel as the visible surface; thread
+ the canonical busy state and `onStop` to the dock; relabel the visibility
+ action; and compose the common audio popover from existing design-system
+ primitives.
+- `libs/@hashintel/petrinaut/src/react/voice-session/` and
+ `libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts`:
+ extend the host/session contract only enough to report and change
+ session-local speaker mute and normalized volume.
+- `apps/petrinaut-website/src/main/app/voice-interview/`: adapt the existing
+ Live and Realtime sessions to that contract, preserving shared capture,
+ Realtime gating, output ownership, admission, canonical Stop and teardown.
+
+Stop on an unlisted semantic delta. A local helper is warranted only when both
+providers actually share the same contract; do not add a second control
+surface, media owner or settings store.
### Owner decisions
-- **2026-09-14:** Kostandin approves the separate constraints-only implementation
- mission following the FE-1712 planning handoff. This permits its local branch,
- separate authority commit, minimal implementation and provider-free checks.
- Delegation-driven invocation and filtering remain deferred.
-- **2026-09-14:** Kostandin authorizes pushing this branch and opening its draft PR
- against FE-1664. This supersedes only the local-only publication restriction;
- existing issues/PRs, parent branches, agent microphone/provider sessions, merge
- and deployment remain outside scope.
-- **2026-09-14:** Kostandin authorizes fixing this child's parent conflict by
- rebasing, reconciling the mission, rerunning checks and pushing with an explicit
- lease. Refresh this draft PR's proof record; leave other issues/PRs unchanged.
-- **2026-09-14:** Kostandin accepts switching Live's separate transcription session
- to semantic VAD after reporting single-word submissions. Use the discussed low
- eagerness, keep Realtime unchanged, commit this authority separately and verify
- locally without microphone/provider sessions. No new push or tracker write.
-- **2026-09-14:** Kostandin authorizes one real `gpt-4o-transcribe` session with
- at most three minutes of synthetic audio, no microphone access and no retries,
- to test hesitation, short replies and correction retention headlessly.
-- **2026-09-14:** After the low run, Kostandin accepts testing medium eagerness:
- change only that transcription setting and its exact request assertion, then
- repeat one session under the same 180-second/audio limit, without retries,
- microphone access, other providers, Brunch inference or publication.
-- **2026-09-14:** Kostandin authorizes one corrected medium session under the same
- three-minute cap, with continuous synthetic silence, no microphone and no retries.
-- **2026-09-14:** Kostandin authorizes pushing the prepared semantic-VAD change and
- removing Amp thread-ID trailers from this child's commit messages. Preserve
- authorship and parent commits; no merge, deployment or new provider allocation.
-- **2026-09-14:** Kostandin accepts the patient-listening recut above with a 500 ms
- indicator hold, not 800 ms, plus the prompt change. Test locally, commit without
- Amp thread IDs, push and refresh this draft's proof. No restack, changed submission
- timing, Realtime change, new provider run or other tracker write.
-- **2026-09-14:** After CI Markdown lint and formatting failed on the inherited
- `PR_DESCRIPTION.md`, Kostandin authorizes the cleanup, removal of Amp thread IDs
- from this child's commits, and restacking onto the parent's current head with an
- explicit lease, reconciling this mission and refreshing the draft PR. No Linear,
- parent-branch, merge, deployment or microphone/provider change.
+- **2026-09-15:** Kostandin accepts reduced Option B: compact dock, secondary
+ audio controls in one popover and the existing conversation panel for output.
+ The authority commit must remain separate from product code.
## Proof
-### Patient-listening recut — provider-free proof passed, human witness pending
-
-The existing `live-conversation.test.ts` telemetry case owns the 500 ms boundary,
-renewed activity and immediate Stop/late-sample behavior. Existing
-`live-conversation-control.test.tsx` cases own Speaking/Thinking/Listening and
-connection/error precedence. These tests do not establish conversational patience.
-Verified 2026-09-14: the new 400 ms assertion failed on the old 300 ms hold, then
-passed with 500 ms. The test also checks 499/500 ms, renewed activity, immediate
-Stop at 499 ms during a pending stats read, and no late-state revival. All 386 tests
-in 11 targeted Live/Realtime suites passed under OS network denial; after tightening
-the Stop timing, all 40 transport tests passed again. A temporary jsdom render of
-the real `VoiceDock` verified its accessible region and Speaking → Thinking →
-Listening text transitions; the temporary probe was removed. No layout changed.
-Website build, typecheck and lint passed all 16 Turbo tasks (10 cached); changed-file
-formatting and whitespace checks passed. The full website suite was not rerun.
-The Live session-creation test owns outgoing instruction carriage and unchanged
-provider configuration; the prompt was inspected against OpenAI's
-[pause-handling guidance](https://developers.openai.com/api/docs/guides/live-prompting).
-Provider-free checks may establish timing and configuration only. Kostandin's next
-fresh Live session remains the oracle for natural hesitation, short complete replies,
-corrections, sparse acknowledgments and stopping speech when interrupted. Stop and
-reorient if the indicator lingers misleadingly or the prompt worsens interruption
-handling. The previous three synthetic transcription probes do not evaluate this
-Live prompt, and their allocations remain consumed.
-
-### Corrected medium probe — synthetic retention verified, human latency pending
-
-One `gpt-4o-transcribe` session, semantic VAD / medium, at most 180 seconds of
-synthetic audio and closed within 180 seconds after connection. No retries,
-alternate model/setting, microphone, GPT-Live or Brunch inference. Use the same
-actual endpoint, fixture bytes (compare hashes to the medium run), pause schedule
-and 15-second final wait. Keep a zero-valued `ConstantSourceNode` connected and
-active until teardown; inspect increasing RTP packet count and sample duration
-through the final wait. No forced commits. Retain exact transcripts, timings,
-session identity, usage and verified cleanup in the local-only native record at
-`/tmp/fe1712-semantic-vad-medium-silence-T-01a09fe5/`. Remove temporary harness and
-audio after inspection. Stop after this allocation for owner review. This can
-adjudicate the three synthetic transcription cases, not human speech or echo.
-
-Observed 2026-09-14: session `sess_EO2X7U9GGGko3GvAOeQFU` confirmed semantic VAD
-with medium eagerness. Input WAV hashes matched the inspected medium fixtures.
-Exactly three committed items and three completed transcripts arrived in the
-same predecessor order, with no extra inputs or errors:
-
-- “The inventory should contain twelve items, not twenty.” — 6.04 seconds after
- speech ended; the one-second mid-sentence pause did not split the input.
-- “Yes” — 5.42 seconds after speech ended, before the next case.
-- “Set it to twenty. Actually, twelve.” — one item, 1.29 seconds after the
- correction ended, retaining both values in order.
-
-All eight final-silence samples showed increasing packet count and source duration:
-1698 → 2428 packets and 35.13 → 49.48 seconds. The corrected sender did not stall.
-An independent read of `events.jsonl` asserted exact transcripts, item order, one
-allocation, no provider errors, and closed peer/context/track with zero microphone
-calls. The session lasted about 48 seconds; 5.49 seconds were synthesized speech.
-Reported completed-item usage was 102 audio-input and 26 output tokens (128 total);
-invoice cost is not established. `events.jsonl` and `attempt.json` in the directory
-above are the local-only evidence. No product change, retry, push or deployment
-was made during this probe. Keep medium locally for owner review; the 5–6 second
-wait on the first two cases remains a usability concern, not an accepted latency.
-
-### Medium-eagerness comparison — completed, correction oracle invalid
-
-Reuse the actual panel endpoint and the low run's five locally generated clips,
-one-second internal pauses, 12-second inter-case gaps and 15-second final wait.
-Inspect fixture contents before dispatch. Record outbound RTP and audio-source
-stats through the final silence to distinguish unfinished provider output from
-a stopped synthetic sender. Do not force a commit or manufacture a final event.
-The only new paid allocation is one `gpt-4o-transcribe` session, closed within
-180 seconds after connection, with at most 180 seconds of synthetic input.
-No retry or alternate setting in that session. Store safe native records under
-`/tmp/fe1712-semantic-vad-medium-T-01a09fe5/`; remove temporary harness/audio after
-inspection. The same boundary, retention, latency and cleanup oracles below apply.
-Provider-free proof: the exact request-body assertion must fail on low and pass
-on medium; rerun the five targeted suites, website typecheck and lint.
-This single synthetic comparison cannot establish human speech or echo behavior.
-
-Observed 2026-09-14: session `sess_EO2SYiIYt07MDK84kntaf` confirmed semantic VAD
-with medium eagerness. The inventory sentence stayed together and completed
-5.52 seconds after its scheduled end (low: 9.50); “Yes.” completed in 4.44 seconds
-(low: 6.48). The correction sequence started one item but never finalized.
-RTP evidence explains why that last case cannot adjudicate VAD: after the final
-clip, outbound packets remained at 1645 and source duration at 32.49 seconds
-throughout the 15-second wait. The audio context still ran, but sent no silence.
-The earlier cases had increasing packet counts during their pauses, so their
-latencies remain observations, not controlled proof of improvement across runs.
-
-A local-only RTC pair reproduced the instrument defect and checked its repair:
-after a completed clip, the old sender emitted zero additional packets over
-three seconds; an active `ConstantSourceNode` with offset zero emitted 150 packets
-and 3.01 additional audio seconds. No provider was called for this contrast.
-Any future paid probe must retain that zero-valued source through the final wait
-and inspect increasing outbound sample duration before judging finalization.
-This repairs only the synthetic instrument, not product microphone behavior.
-
-Native `events.jsonl`, `attempt.json` and `local-silence-check.jsonl` in the medium
-directory above retain the inspected evidence. One session, roughly 48 seconds,
-5.49 seconds of synthetic speech; zero microphone calls and verified teardown.
-Completed items reported 62 audio-input plus 16 output tokens (78 total);
-unfinalized-item usage and invoice remain unknown. No retry or publication.
-The medium request assertion failed on low then passed; all 127 targeted tests,
-changed-file formatting and 15 website typecheck/lint tasks pass (10 cached).
-Product code remains medium, unaccepted for full conversational quality.
-
-### Bounded headless transcription probe — completed, acceptance not established
-
-Use the actual `createOpenAITranscriptionSessionHandler` with the website's Vite
-development environment loader (process values win). Drive its raw-SDP WebRTC
-boundary from installed headless Chromium with a Web Audio synthetic track, not
-`getUserMedia`. Generate only the three fixed witness phrases locally with macOS
-speech synthesis; no TTS provider, Live session, Brunch inference or private data.
-The only paid allocation is one `gpt-4o-transcribe` session, at most 180 seconds
-of synthetic input, closed within 180 seconds after connection. No retries,
-alternate models, second allocation or provider fallback on rejection/timeout.
-
-Inspect effective session configuration, provider item boundaries, exact completed
-transcripts and timing relative to the scheduled one-second intra-phrase pauses.
-Hesitation should remain one item; “Yes” must finalize without waiting for another
-utterance; correction must retain both twenty and twelve in order. Report latency
-rather than claiming a universal acceptable threshold. This is a synthetic
-transcription-boundary probe, not proof of Brunch admission or physical echo.
-Keep request/session IDs, returned usage and safe events in one local-only native
-record under `/tmp/fe1712-semantic-vad-T-01a09fe5/`; unknown billing is not zero.
-Stop after the single run or first rejection and return its evidence. Remove
-temporary harness/audio after inspection; retain the safe native result.
-
-Observed 2026-09-14 via the already-running panel at `localhost:4915`, whose process
-cwd is this checkout's website and whose API loader imports the current handler:
-OpenAI session `sess_EO2KwbqCN3NRqcsrfVQ80` confirmed `gpt-4o-transcribe` and
-`semantic_vad` / `low`. Safe native records are `events.jsonl` and `attempt.json`
-in the local-only directory above. One attempt, roughly 48 seconds connected,
-5.49 seconds of synthesized speech plus silence, zero microphone calls; the peer,
-audio context and track all closed. No retry, GPT-Live or Brunch inference.
-
-- Hesitation: the one-second pause after “The inventory” stayed in one exact
- completed sentence: “The inventory should contain twelve items, not twenty.”
- Completion arrived 9.50 seconds after the scheduled end of that sentence.
-- Short reply: “Yes.” finalized before the next input, but 6.48 seconds after
- its scheduled end. Promptness is not established.
-- Correction: “Set it to twenty.” finalized separately (1.34 seconds after its
- end); the provider began another item for “Actually, twelve” but never emitted
- its stop/commit/completion during the remaining 15 seconds. Both correction
- words were verified in the exact input fixture after its internal pause.
- **Correction oracle invalid:** the medium probe and local sender contrast above
- exposed a shared harness defect: after the last clip it stops sending silence.
- Retract the earlier retention/finalization-failure interpretation; this case
- cannot establish provider loss or behavior with a real microphone. Brunch was
- not invoked. No low-run RTP trace exists to adjudicate that session independently.
-
-Latencies use the browser's common monotonic clock for scheduled audio and event
-receipt; they include provider/network delay, not just the VAD classifier.
-Completed items reported 79 audio-input and 23 output tokens (102 total).
-Unfinalized-item usage and invoice cost remain unknown, not zero. This evidence
-motivates the separately accepted medium comparison above, not an acceptance claim.
-
-### Semantic turn-boundary recut — locally verified, owner witness pending
-
-Verified 2026-09-14: the exact outbound-body assertion failed on `server_vad`
-before the change. After switching to semantic VAD with low eagerness, 127 tests
-pass under OS network denial: `openai-transcription-session.test.ts`,
-`openai-realtime-call.test.ts` and `openai-voice-policy.test.ts` under
-`src/server/voice/`, plus `live-conversation.test.ts` and `live-brunch-bridge.test.ts`
-under `src/main/app/voice-interview/`. Use the network-denied unit command below
-with those five paths. These prove request configuration, unchanged Realtime policy
-and existing failure/no-retry behavior, not provider acceptance or speech quality.
-`turbo run lint:tsc lint:eslint --filter @apps/petrinaut-website
---output-logs=errors-only` passes all 15 tasks (10 cached). Changed TypeScript
-formatting and `git diff --check` pass. Full website tests/build were not rerun
-for this configuration-only recut. No provider session or UI change was made.
-
-Owner-held witness: in a fresh Live session, compare a hesitant phrase such as
-“The inventory ... um ... purchase quantity is twelve, not twenty” against a
-deliberately complete “Yes.” Check exact retained words, submission count/order,
-and whether waiting feels excessive. Repeat with speakers and headphones. Do not
-discard short legitimate answers to make the witness pass. Compatibility and
-improved boundaries remain unproved until this actual product observation.
-
-### Provider-free configuration and regressions
-
-Baseline: `live-conversation.test.ts` passes 40 tests on the original comparison base under
-OS network denial. The new capture assertion failed specifically because the old
-call supplied `{ audio: true }`, then passed with the selected preferences.
-
-Run from the repository root with the pinned Node/Yarn toolchain:
-
-```sh
-sandbox-exec -p '(version 1)(allow default)(deny network*)' yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/live-conversation.test.ts
-```
-
-Verified 2026-09-14 after conflict resolution: these seven files pass 317 tests:
-`live-conversation.test.ts`, `live-brunch-bridge.test.ts`,
-`live-conversation-control.test.tsx`, `openai-realtime-session.test.ts`,
-`realtime-brunch-bridge.test.ts`, `voice-turn-controller.test.ts`, and
-`voice-interview-control.test.tsx`, all under the cold-start directory above.
-Existing late-permission, partial-failure and Stop tests retain media release and
-stale-callback invalidation. These suites guard input admission, settlement,
-interruption, consent, handoff and teardown; they do not establish acoustic correctness.
-The full website suite was not rerun for this localized change.
-
-`yarn workspace @apps/petrinaut-website lint:tsc`, `lint:eslint` and `build` pass.
-Lint reports zero warnings/errors. Build reports unchanged React Compiler
-`try`/`finally` optimization and chunk-size warnings. Changed-file `oxfmt --check`
-and `git diff --check` pass; Brunch Markdown is excluded by repository formatter
-configuration and reviewed directly. No UI appearance or interaction controls change.
-
-Prior capture-only restack verification: `turbo run build lint:tsc lint:eslint --filter
-@apps/petrinaut-website --output-logs=errors-only` passes all 16 tasks (9 cached).
-The seven-suite run includes the parent's new consent and Thinking controller tests.
-That production diff against the parent contained only the capture preferences;
-consent and dock implementation are unchanged from that parent.
-
-### Manual speaker and headphone witness — pending, owner-held
-
-Allow about five minutes per output mode. Kostandin compares the pinned base and
-this branch using the same browser, microphone, volume, prompt and disposable
-document. Inspect effective capture settings using the browser's WebRTC diagnostics;
-record unavailable settings as unknown, not confirmation. Do not start a second
-capture just to inspect settings. Requested preferences may already be defaults.
-
-1. Connect both sessions and remain silent while a short no-tool Brunch answer
- plays. Compare speakers and headphones; inspect completed input, canonical
- admissions, delegation and commentary separately. Target zero unwanted admissions.
-2. During playback give short novel replies and quantity/negation corrections;
- hesitate, elaborate and deliberately quote the assistant. Check exact retained
- words and admission order. Lost corrections invalidate apparent improvement.
-3. Exit Voice and check silence/cleanup. If feedback persists, use headphones or
- typed input. For the existing manual fallback, end Live, explicitly start
- Realtime, disable “Interruption by speaking” and use “Your turn.” Never switch
- or replay automatically. Unknown admission requires inspecting history first.
-
-This witness can support a limited mitigation claim, not an all-device guarantee,
-native speech fidelity, tool-turn acceptance or migration readiness. Only the
-separate bounded headless probe above grants an agent-run provider allocation.
+### Authority cut
+
+Verified 2026-09-15: forced repository Markdown lint checked exactly this
+mission, its future pointer and the website pointer with zero errors;
+`git diff --check` also passed. These checks establish legible repository
+authority only; they do not establish any product behavior. The pre-cut focused
+baselines were reported as 51 Petrinaut tests and 218 website tests passing;
+they contain no FE-1722 implementation.
+
+### Deterministic product proof — established
+
+- `libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx`
+ owns the compact dock, Show/Hide conversation as visibility only, canonical
+ Stop only for `submitted`/`streaming`, separate End, common audio controls and
+ provider-capability presentation.
+- `libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.test.ts`
+ and
+ `libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx`
+ own the host contract and provider-optional action forwarding without making
+ controls mandatory for unrelated hosts.
+- `apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts`
+ and `live-conversation-control.test.tsx` own shared-track microphone mute,
+ independent playback mute/volume, per-session reset and the unchanged
+ canonical Stop-to-`stopResponse()` path without media teardown.
+- `apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts`,
+ `voice-turn-controller.test.ts` and `voice-interview-control.test.tsx` own
+ unchanged Realtime microphone gating, common speaker settings, per-session
+ reset and Realtime-only controls.
+- `apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts`
+ and `live-conversation-control.test.tsx` own status precedence and prove that
+ speaker mute or volume zero does not rewrite Speaking.
+
+Verified 2026-09-15:
+
+- Network-denied
+ `yarn workspace @hashintel/petrinaut test:unit --run`
+ over the three focused Voice files passed 160 tests after the rebase.
+- Network-denied `yarn workspace @apps/petrinaut-website test:unit` over the
+ eight focused unit files passed 349 tests after the rebase; the ninth
+ `voice-preview.integration.test.ts` file passed 5 tests under the same
+ network denial.
+- `build`, `lint:tsc` and `lint:eslint` passed independently for
+ `@hashintel/petrinaut` and `@apps/petrinaut-website`.
+- `yarn workspace @local/petrinaut-arch-docs lint:arch-docs` reported 79
+ layers, 408 edges, 866 files, 80 generated pages and 40 authored pages.
+- Root `yarn lint:format`, exact Markdown lint over the changed mission,
+ pointer, user guide and changeset, plus working and committed
+ `git diff --check` checks passed.
+
+These checks establish deterministic controls and regressions, not physical
+audio, conversational quality or visual usability.
+
+### Product witness — pending, owner-held
+
+Kostandin starts a fresh Live session and a fresh Realtime session through the
+real product door. In each, show and hide the conversation while speech and
+canonical work continue; mute and unmute the microphone; mute the speaker and
+move volume through zero during output; verify Speaking still reflects provider
+output; and Stop one submitted or streaming Brunch response without ending
+Voice. End Voice separately and confirm it does not stop canonical work. Start
+a second session and confirm speaker mute and volume reset. In Realtime only,
+also exercise read-full-response, repeat-question and
+interruption-by-speaking.
+
+This witness may accept the interaction and audible effect on the tested
+browser/device. It does not establish all-device media behavior, natural turn
+boundaries, echo mitigation or FE-1712's remaining owner-held obligations.
## Constraints
-- Preserve Realtime, default provider selection, consent, provider pinning, one
- capture feeding both sessions, and teardown on failure/Stop. No extra capture,
- session, dependency, telemetry store, retry or automatic fallback.
-- Keep `gpt-4o-transcribe`, provider item ordering, no-delegation
- admission, one waiting composer slot, frozen settled commentary and once-only
- offering unchanged. No transcript suppression, fuzzy matching or new timers.
-- Brunch remains canonical answer/tool authority. Live speech remains native and
- best-effort; settlement gates supplied context, not every audible word. Append
- acknowledgment is not consumption, speech, playback or execution completion.
-- Preserve visible full text and no truncation/chunking/replay on commentary
- rejection. Local Exit and canonical Stop remain distinct from acoustic interruption
- and from canceling already-executed effects.
-- No changes to parent branches, other issues/PRs, Brunch prompts, models,
- services or infrastructure. Push and draft creation are authorized for this child
- only; no merge, deployment or agent microphone access. The only provider exception
- is the bounded transcription probe above. Prior FE-1664
- publication permissions do not transfer to this mission.
-- Only the separate Live transcription session may switch VAD as specified above;
- do not change Realtime. The only native Live behavior change is the accepted
- patient-listening instruction; its effect is probabilistic, not enforced timing.
+- Preserve explicit consent, one-capture ownership, teardown, transcript
+ admission, delegation policy, canonical Brunch authority, provider pinning
+ and Realtime response ownership. Do not start microphone or provider sessions
+ on an agent's behalf; real media evidence remains owner-held.
+- Preserve FE-1712 browser capture preferences, semantic VAD,
+ patient-listening instruction and 500 ms output-activity hold.
+- Live microphone mute disables the one shared capture track feeding Live and
+ transcription without muting playback, ending either session or changing
+ canonical work. Realtime keeps its existing gating semantics.
+- Canonical Stop appears only for `submitted` or `streaming` and uses the
+ existing `onStop` path. Live Stop keeps media connected. End tears down Voice
+ and does not cancel canonical work.
+- Connection/error, Speaking, Thinking, microphone-muted and Listening retain
+ that precedence. Audio settings describe local audibility, not provider
+ output activity.
+- Show/Hide conversation changes visibility only. It must not change capture,
+ playback, work, session state, panel history or admission.
+- Speaker mute and normalized volume are session-local for both providers and
+ reset for every new session. Do not persist them.
+- A provider-finalized partial transcript admitted after mid-utterance mute is
+ allowed. Add no transcript suppression, fuzzy matching or timers.
+- Existing read-full-response, repeat-question and interruption-by-speaking
+ behavior stays Realtime-only.
## Fog-line
-The reported silent “No tengo.” admission demonstrates unwanted input, not whether
-echo, background audio, routing or hallucination caused it. Browser defaults may
-already apply these preferences; effective settings and the manual contrast decide
-whether this change has acoustic value. A passing configuration test does not.
-
-OpenAI's [VAD guide](https://developers.openai.com/api/docs/guides/realtime-vad)
-documents semantic VAD for supported transcription sessions and low eagerness for
-larger chunks. Actual acceptance with this model/session is the headless probe's
-first discriminator; natural human speech remains owner-witnessed.
-Semantic VAD is probabilistic and may add latency; it does not guarantee a complete
-thought or prevent echo. The earlier VAD rejection on a different model does not
-establish incompatibility for `gpt-4o-transcribe`.
-
-Live-native transcripts and client delegation remain an alternative, not a selected
-replacement. Transcript deltas lack authoritative finalization/item identity;
-delegation has an opaque ID, target and offset, not task text. Separate transcription
-finalizes audio items, not complete thoughts. Shared capture does not synchronize
-the sessions' clocks. The cross-session range-selection oracle remains unresolved:
-which immutable input range belongs to a delegation, and when is it complete?
-Latest-item pairing, arrival order, silence timeouts or fuzzy matching cannot prove
-it. This cut neither changes that policy nor claims to solve it.
-
-Mandatory half-duplex could enforce app playback/capture exclusion at the cost of
-simultaneous listening. Live lacks Realtime's response-terminal/handoff lifecycle;
-do not invent it from activity telemetry or append acknowledgments.
+The exact compact spacing, icons, volume affordance and responsive fit remain
+implementation details to validate against the existing Petrinaut design
+system and accessibility semantics. They may not move microphone mute into the
+popover, create another output surface or alter the control policy above.
+
+Muting a capture track cannot retract audio the provider already received; a
+finalized partial transcript after mute is therefore neither automatically a
+bug nor evidence of suppression. Speaker mute and zero volume change local
+audibility, not whether output is active. Deterministic browser tests cannot
+establish subjective volume feel, physical routing or whether the compact dock
+is usable on Kostandin's device.
+
+FE-1712's acoustic benefit, natural turn-boundary quality, direct spoken-user
+attribution, withheld-work recovery and comparative latency remain unresolved
+at their existing parent or future-spine owners. This mission neither reruns
+nor accepts them.
## Stop or reorient
-Stop after the bounded headless probe for Kostandin's review. Do not add filtering
-automatically. Return to the owner if preferences change neither settings nor
-failure, headphones still produce silent admissions, or genuine corrections are
-lost. Reclassify the observed failure before adding a mechanism.
-For the turn-boundary recut, stop on provider rejection, continued fragmentation,
-lost corrections or unacceptable delay. Preserve the existing visible connection
-failure without silently reverting VAD; use typed input or explicitly ended Live
-followed by Realtime. Return to the owner before selecting another setting.
-
-If deterministic prevention is required, select half-duplex/typed policy explicitly.
-If delegation-driven invocation is required, resolve finalization/range selection
-first. Any new queue, gate, prompt, model or handoff policy requires a new accepted
-cut. Premature substantive speech, reordered/lost corrections, replay of uncertain
-work or revived speech after Stop remain failures, not accepted side effects.
+Stop and return to the owner if microphone mute silences output, creates a new
+capture, changes admission, or fails to gate both Live consumers of the shared
+track; if speaker controls alter microphone state or Speaking status; if Stop
+tears down Voice or End stops canonical work; if Show/Hide changes anything
+other than visibility; if settings survive a new session; or if Live gains
+Realtime-only controls.
+
+Also stop on an inaccessible or unusable compact layout, a provider-specific
+contract that cannot be represented without weakening the common invariants,
+an unlisted persistence or timer, a new provider/media session, or a required
+change to FE-1712's protected behavior. Do not select a broader redesign from
+mechanism failure without a new owner decision.
## Deferred
-[Voice feedback follow-up](MISSION.next.md#voice-feedback-follow-up) retains the
-conditional filtering, native-delegation and half-duplex alternatives. The existing
-[Voice recovery obligation](MISSION.next.md#voice-after-the-live-transport-cut),
-parent integration witness and Mission 7c/7d obligations remain open under their
-owners; this cut does not consume their waivers or acceptance.
+[Voice control follow-up](MISSION.next.md#voice-control-follow-up) retains
+device switching, voice and speed selection, helmet animation and settings
+persistence. [Voice feedback follow-up](MISSION.next.md#voice-feedback-follow-up)
+and [Voice after the live transport cut](MISSION.next.md#voice-after-the-live-transport-cut)
+retain FE-1712's unfinished alternatives and owner-held obligations. None is
+authorized by this cut.
diff --git a/libs/@hashintel/brunch-agent/MISSION.next.md b/libs/@hashintel/brunch-agent/MISSION.next.md
index 6a745eefb65..baea2c28306 100644
--- a/libs/@hashintel/brunch-agent/MISSION.next.md
+++ b/libs/@hashintel/brunch-agent/MISSION.next.md
@@ -2,17 +2,26 @@
> Future sequence and decision register only; not execution authority. [`MISSION.md`](MISSION.md) owns live scope and progress. Successor drafts become executable only after an owner-authorized cut; archives and git history retain prior contracts.
-On this stacked voice branch, `MISSION.md` owns FE-1712. The inherited Mission 7c
-map and its mission-section references below belong to the
+On this stacked voice branch, `MISSION.md` owns FE-1722. Its pinned parent is
+[FE-1664 at `023a26b96b`](https://github.com/hashintel/hash/blob/023a26b96b51169da0acdb188697e159d001bcc0/libs/%40hashintel/brunch-agent/MISSION.md),
+the squash base incorporating merged
+[FE-1712 PR #9704](https://github.com/hashintel/hash/pull/9704). FE-1712's
+protected behavior and evidence are inherited through that base; its unfinished
+speech, acoustic and recovery obligations remain open under their existing
+owners. The inherited Mission 7c map and its mission-section references below
+belong to the
[upstream contract](https://github.com/hashintel/hash/blob/dee90599e9a07d9fa3e55d0711c14491e9ce5c7c/libs/%40hashintel/brunch-agent/MISSION.md)
on #9667, not to a second execution authority here. The stack does not close that
mission or grant its paid-run permissions to voice work.
## Voice feedback follow-up
-FE-1712 selects explicit capture preferences, semantic transcription turn detection
-and owner-held speech/acoustic comparisons as specified in the live mission. The complete
-[FE-1664 contract](https://github.com/hashintel/hash/blob/9499b9287bd69b751ebcdd61b0c6bf2586bc191e/libs/%40hashintel/brunch-agent/MISSION.md)
+FE-1712 selected explicit capture preferences, semantic transcription turn detection
+and owner-held speech/acoustic comparisons as specified in its
+[inherited FE-1664 squash base](https://github.com/hashintel/hash/blob/023a26b96b51169da0acdb188697e159d001bcc0/libs/%40hashintel/brunch-agent/MISSION.md).
+FE-1722 preserves that behavior and those unfinished obligations while changing
+only the controls admitted by its live mission. The complete
+[FE-1664 contract](https://github.com/hashintel/hash/blob/023a26b96b51169da0acdb188697e159d001bcc0/libs/%40hashintel/brunch-agent/MISSION.md)
is retained at the branch's pinned parent, not archived as accepted or replaced
on that branch. Its input/delivery contracts, first no-tool exchange, later
operation/correction/Stop and independent-tab provenance/withheld-work witnesses
@@ -43,6 +52,13 @@ The broader alternatives, rejected shortcuts and discriminating test portfolio a
planning context in [FE-1712](https://linear.app/hash/issue/FE-1712/stabilize-gpt-live-full-duplex-voice-feedback),
not authority for these deferred changes.
+## Voice control follow-up
+
+FE-1722's [live mission](MISSION.md) owns the current control cut. Device
+switching, provider voice and speech-speed selection, helmet animation and
+persistence of speaker settings remain future work and require a separate
+owner-authorized cut.
+
## How to use this spine
Read this file to answer four questions:
@@ -238,7 +254,8 @@ Immediate switching from a review or gap report into renewed elicitation remains
### Voice after the live transport cut
-The FE-1664 integration is governed by this branch's [mission](MISSION.md).
+The inherited FE-1664 integration is governed by the
+[FE-1712 behavior and evidence inherited through this branch's FE-1664 pinned parent](https://github.com/hashintel/hash/blob/023a26b96b51169da0acdb188697e159d001bcc0/libs/%40hashintel/brunch-agent/MISSION.md).
Native Live delivery and canonical transcription do not waive the recovery
obligation below or establish live provider compatibility. Historical waiver and
attribution rationale remains in the
diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md
index eaae84c985c..eedece17a23 100644
--- a/libs/@hashintel/petrinaut/docs/ai-assistant.md
+++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md
@@ -72,110 +72,110 @@ understand and selecting **Start voice mode**. Petrinaut remembers that acknowle
browser for the current disclosure version, so later uses of **Start voice mode** start directly. If
browser storage is unavailable or the disclosure changes, Petrinaut asks again.
-Some hosts offer a Brunch-backed GPT-Live voice interview instead. It uses one
-GPT-Live session for conversational audio and a separate transcription session for
-canonical user messages. Brunch remains responsible for domain answers, chat history,
-and model operations; settled Brunch prose is supplied to Live for best-effort spoken
-delivery. Consent and **Start voice** are required for each session. The permission
-panel explains both OpenAI audio streams and text retention, with a permission
-checkbox, **Start voice**, and **Cancel**. Cancel returns to text without starting a
-session. Its dock provides collapse/expand and **End voice mode**; **Thinking** means
-Brunch is working, while **Speaking** reflects local audio activity rather than a
-completed turn. The microphone stays open during playback and the input ribbon follows
-its level where the browser supports it. If the browser blocks remote playback, the
-dock keeps the warning visible and offers **Play voice audio**; selecting it retries
-playback from that user gesture. Ending Voice stops both sessions, microphone capture,
-and playback; closing the panel ends the experiment rather than pausing it. A
-connection error returns to setup without retrying. The detailed turn-taking and
-playback controls below apply to Realtime-based Brunch Voice, not the Live experiment.
-
-While a session runs, the composer is replaced by a low-profile Voice dock at the foot of the panel:
-a ribbon that fades out at both ends and one short state -- **Connecting**, **Listening**,
-**Muted**, **Thinking**, **Speaking**, **Paused**, or **Voice interrupted**. The ribbon is three
-travelling waves, one leading and two trailing it more faintly. While it listens they open with your
-voice and settle back between phrases, so an ordinary speaking level is plainly visible rather than a
-flicker above the line. While the assistant speaks the ribbon takes on a restrained self-driven
-motion instead, colour crossfading as the turn changes hands, so which side holds it is readable at a
-glance. It flattens to near a line whenever nobody holds the turn.
-
-For Brunch-backed Live sessions, **Thinking** follows an active submitted or streaming
-request, even without microphone activity. **Speaking** takes precedence during playback;
-once work and playback are idle, the dock returns to **Listening**. This status does not
-mute the microphone or announce progress aloud.
-
-Voice failures and recovery warnings, including unconfirmed submissions and input that was not
-retained, join the Voice warning indicator rather than appearing as global notifications. Hover to
-preview or click to read the complete details, including while the panel is collapsed. Distinct
-issues share one icon with a count; long details wrap and scroll in the popover, where they can be
-copied or dismissed with **Dismiss Voice issues**. Issues stay in the list until dismissed or the
-chat is cleared; dismissing them does not trigger a retry or mean that unsent input was retained.
-Temporary status notices, such as a pending handoff, replace the short dock state only while they
-apply and do not become Voice issues.
-
-Spoken turns appear in the conversation as soon as their finalized text arrives, so the transcript
-stays current while the session runs and tools that edit the net remain visible. Select **Collapse
-voice session** to reduce the panel to the Voice dock alone; this hides the AI header, transcript, and
-host Voice region without ending the session. Select **Expand voice session** to restore them. Ending
-Voice while collapsed also closes the AI panel; ending Voice while expanded returns to the text
-composer. When Voice is collapsed, the zoom and fullscreen controls sit above the compact dock
-at the right edge; the bottom toolbar stays beside it. Only finalized answers and canonical Brunch text become chat history; provisional
-transcription and Realtime audio are ephemeral. Finalized spoken user messages carry a small
-**Voice** chip in front of the words themselves, and the exact inline answer completed by speech
-carries the same chip, so Voice provenance remains visible without duplicating an answer while the
-session is mounted.
-
-**Interruption by speaking** is on by default. Start speaking while Brunch is talking to stop its
-audio and give your answer. Your interrupting words are captured; you do not need to repeat them.
-If Brunch is still finishing its previous turn, the dock shows **Answer captured. Waiting for Brunch.**
-and sends that answer when it is ready. Wait for that answer to be sent before giving another one.
-
-Open **Voice playback options** and uncheck **Interruption by speaking** to use manual handover.
-This preference is remembered in your browser. In manual mode the microphone closes during
-assistant output. Select **Your turn**, wait for cancellation to finish, then speak; audio before
-that handover is discarded. The **Your turn** control is hidden when interruption by speaking is on.
-If speaker playback causes unwanted interruptions, use headphones or switch to manual handover.
-
-Semantic voice detection finishes your answer automatically after a natural pause, so there is no
-required done-speaking action. Duplicate, empty, failed, or unavailable transcripts are not submitted.
-An empty or failed transcript asks you to try again, and an overlong answer asks for a shorter response.
-Completed interruptions that strongly repeat the transcription vocabulary or the assistant's active
-speech are silently discarded rather than sent as your answer. Short answers such as “stop”, “no”,
-and “wait” remain valid. A false speech detection can still stop playback even when its transcript
-is discarded.
-Provisional words remain display-only until the provider completes their transcript.
-
-Every session control lives in the dock: **Collapse voice session** / **Expand voice session** and
-**Voice playback options** on the left, and the available handoff, microphone, recovery, and end
-actions on the right.
-**Read full response** becomes available after the matching response and speech have both finished
-and replays every exact retained canonical segment in order. **Repeat question** uses the same
-availability gates and replays only exact question text explicitly marked by Brunch. It stays
-disabled when that marker is missing or does not match finalized assistant text rather than
-guessing that the final segment is a question.
-Playback stays unavailable during active capture, submission, cancellation, pause, and errors. **Mute microphone** becomes
-**Unmute microphone** once muted, and your latest choice applies when a handoff settles. **Resume voice mode**
-replaces the microphone action while a session is paused, and **Reconnect voice mode** replaces it
-after a failure. Nothing is added to the canvas toolbar. Sending non-empty typed text from the
-composer or first-run prompt ends Voice mode before it sends the message once through the same
-conversation; repeated send actions are ignored while that short handoff completes.
-
-The interviewer uses a warm, calm, curious, and professionally neutral voice and treats you as the authority on your system. Brunch still chooses every question and interview decision; OpenAI only transcribes your completed input and delivers Brunch's words. The question and finalized response shown in the Petrinaut conversation are authoritative. The speech request receives that exact Brunch text in part order; synthesized audio is generated from it but is not a verbatim recording. Interrupting audio does not undo the visible response or change the interview's saved history.
-
-Closing the AI panel pauses microphone capture and active speech, then hides the dock until you
-reopen the panel. The same mounted session stays paused; choose **Resume voice mode** when you are
-ready. **Clear AI chat** is unavailable while a Voice
-session is active.
-
-If voice cannot continue, the status reads **Voice interrupted** and the warning icon contains
-the microphone, connection, or Voice failure, followed by any diagnostic reference in parentheses.
-**Reconnect voice mode** replaces the microphone action until
-the session recovers. For microphone permission or device errors, allow access or connect/select a
-microphone before reconnecting. For an interrupted request, network error, or timeout, check the
-connection and reconnect. If the preview is unavailable, continue with the text composer. An invalid
-service response includes a diagnostic reference you can give to an operator. That reference and its
-diagnostic record do not contain your transcript or the response being spoken. Interview-state
-failures use a content-free `interview-correlation`, `interview-response`, or `interview-submission`
-code so an operator can distinguish them without receiving your answer.
+Some hosts offer a Brunch-backed GPT-Live voice interview. It uses one Live
+session for conversational audio and a separate transcription session for
+finalized user messages. Brunch remains responsible for domain answers, chat
+history, and changes to the net; settled Brunch prose is supplied to Live for
+best-effort spoken delivery. Consent and **Start voice** are required for each
+session. The permission panel explains both OpenAI audio streams and text
+retention, with a permission checkbox, **Start voice**, and **Cancel**.
+**Cancel** returns to text without starting one. If the browser blocks remote
+playback, the dock keeps the warning visible and offers **Play voice audio**;
+selecting it retries playback from that user gesture. Closing the panel or
+selecting **End voice mode** ends Live audio, transcription, microphone
+capture, and playback. A connection error returns to setup without retrying.
+
+While Voice runs, the composer is replaced by a compact dock at the foot of the
+panel. It shows one short state -- **Connecting**, **Listening**, **Muted**,
+**Thinking**, **Speaking**, **Paused**, or **Voice interrupted** -- and keeps
+the controls available without covering the transcript. Select **Hide
+conversation** to leave only the dock visible, and **Show conversation** to
+restore the AI header, transcript, and host Voice region. These controls change
+visibility only: they do not pause, stop, or end Voice. Ending Voice while the
+conversation is hidden also closes the AI panel; ending it while the
+conversation is visible returns to the text composer.
+When the conversation is hidden, the zoom and fullscreen controls remain above
+the compact dock at the right edge.
+
+The microphone action stays directly in the dock. **Mute microphone** becomes
+**Unmute microphone** and remains pressed while input is muted. Mute is
+independent of the activity state: if the assistant is playing audio, the dock
+continues to say **Speaking** while the pressed microphone action truthfully
+shows that input is muted. When no output or Brunch work takes precedence, a
+muted session can instead show **Muted**. For Live sessions, **Thinking** means
+Brunch has a submitted or streaming response, while **Speaking** means audio is
+currently playing. Neither state announces progress aloud or changes the
+microphone setting. The microphone action remains visible but disabled while
+Voice is connecting, paused, or interrupted by an error.
+The latest microphone-mute choice is reapplied when a handoff settles.
+
+**Stop AI response** appears next to the separate **End voice mode** action
+only while Brunch has submitted or streaming work. Stop cancels that current
+canonical Brunch response; it does not reverse changes that already completed.
+During a Live session, Stop leaves both media sessions and the microphone
+available for the next turn. **End voice mode** tears down Voice but does not
+cancel canonical Brunch work already in progress, so select Stop first when
+you also need to cancel that work.
+
+Open **Audio options** for session-local speaker controls. Both Live and
+Realtime Voice provide **Mute speaker** / **Unmute speaker** and **Speaker
+volume**. These controls affect assistant playback only: they do not affect
+microphone input, Brunch work, or the **Speaking** state. When the speaker is
+muted during playback, the dock therefore continues to say **Speaking** and
+Audio options shows the pressed speaker state. Speaker mute and volume reset
+for each new Voice session. Speaker controls are unavailable while Voice is
+connecting or interrupted by an error; they remain available while Realtime is
+paused. They are the only Audio options preferences that reset per Voice
+session: Realtime remembers **Interruption by speaking** in this browser. The
+dock does not promise device switching, voice or speed selection, helmet
+animation, or persistence of the speaker settings.
+
+Realtime-based Brunch Voice additionally provides **Repeat question**, **Read
+full response**, and **Interruption by speaking** in Audio options. Live Voice
+does not show these controls. **Read full response** becomes available after
+the matching response and speech have both finished and replays every exact
+retained canonical segment in order. **Repeat question** uses the same
+availability gates and replays only exact question text explicitly marked by
+Brunch. It stays disabled when that marker is missing or does not match
+finalized assistant text.
+Both replay controls stay unavailable while capture, submission, cancellation,
+pause, or an error makes playback unsafe.
+
+With **Interruption by speaking** enabled, start speaking while Brunch is
+talking to stop its audio and give your answer. Your interrupting words are
+captured; you do not need to repeat them. If Brunch is still finishing its
+previous turn, the dock shows **Answer captured. Waiting for Brunch.** and
+sends that answer when it is ready. Wait for it to be sent before giving
+another one. Disable **Interruption by speaking** to use manual handover. In
+manual mode, select **Your turn**, wait for cancellation to finish, then speak;
+audio before that handover is discarded. **Your turn** is hidden while
+interruption by speaking is enabled.
+
+Semantic voice detection finishes an answer automatically after a natural
+pause, so there is no required done-speaking action. Duplicate, empty, failed,
+or unavailable transcripts are not submitted. Provisional words remain
+display-only until the provider finalizes their transcript. Spoken turns then
+appear in the conversation, and finalized spoken user messages carry a
+**Voice** chip. Only finalized answers and canonical Brunch text become chat
+history; provisional transcription and provider audio are ephemeral.
+Completed interruptions that strongly repeat the assistant's active speech may
+be silently discarded instead of sent as an answer. Short answers such as
+“stop” and “no” remain valid.
+
+Voice failures and recovery warnings, including unconfirmed submissions and
+input that was not retained, appear behind the Voice warning indicator instead
+of as global notifications. Hover to preview or select it to read the complete
+details, including while the conversation is hidden. Distinct issues share one
+icon with a count. Dismissing them does not retry a request or mean unsent
+input was retained. Temporary notices replace the short dock state only while
+they apply. Diagnostic references and their records contain neither your
+transcript nor the response being spoken.
+
+Realtime Voice pauses microphone capture and active speech when the AI panel
+closes. Reopen the panel and select **Resume voice mode** to continue. Live
+Voice ends when the panel closes. If Realtime Voice is interrupted, allow
+microphone access or check the connection, then select **Reconnect voice
+mode**. **Clear AI chat** is unavailable while a Voice session is active.
When no interview is active and the host permits clearing, **Clear AI chat** via the delete button in the top right of the panel wipes the local conversation, stops any in-flight stream, and tells the host app to forget the messages if it persists them. Hosts with canonical history may disable this control. The Brunch panel disables it because clearing only the browser view would not delete Flue history and the conversation would return on rehydration.
diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/store.ts b/libs/@hashintel/petrinaut/src/react/voice-session/store.ts
index 6bb3ad5230d..37ac1148765 100644
--- a/libs/@hashintel/petrinaut/src/react/voice-session/store.ts
+++ b/libs/@hashintel/petrinaut/src/react/voice-session/store.ts
@@ -14,6 +14,8 @@ export type VoiceSessionActions = {
resume?: () => void;
setInterruptionBySpeaking?: (enabled: boolean) => void;
setMicrophoneMuted?: (muted: boolean) => void;
+ setSpeakerMuted?: (muted: boolean) => void;
+ setSpeakerVolume?: (volume: number) => void;
takeTurn?: () => Promise | void;
};
diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts
index be5e1b5fce5..a8f089f9217 100644
--- a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts
+++ b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts
@@ -35,6 +35,10 @@ export type PetrinautAiVoiceSessionState = {
/** Temporary operational status shown in place of the current phase. */
notice?: string | null;
phase: PetrinautAiVoiceSessionPhase;
+ /** Whether assistant audio is muted independently of its retained volume. */
+ speakerMuted?: boolean;
+ /** Normalized 0–1 assistant audio volume. */
+ speakerVolume?: number;
/** Recoverable issue retained behind the Voice warning indicator. */
warningMessage?: string | null;
};
diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts b/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts
index 3fa16ee72c7..2d1d9c1f7b9 100644
--- a/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts
+++ b/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts
@@ -43,6 +43,26 @@ export const useVoiceSessionMicrophoneMuted = (): boolean => {
);
};
+export const useVoiceSessionSpeakerMuted = (): boolean => {
+ const store = use(VoiceSessionContext);
+
+ return useSyncExternalStore(
+ store.subscribe,
+ () => store.getSnapshot().state?.speakerMuted ?? false,
+ () => false,
+ );
+};
+
+export const useVoiceSessionSpeakerVolume = (): number => {
+ const store = use(VoiceSessionContext);
+
+ return useSyncExternalStore(
+ store.subscribe,
+ () => store.getSnapshot().state?.speakerVolume ?? 1,
+ () => 1,
+ );
+};
+
export const useVoiceSessionErrorMessage = (): string | null => {
const store = use(VoiceSessionContext);
diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.test.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.test.ts
index 37ec0af7bbc..5b101be7640 100644
--- a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.test.ts
+++ b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.test.ts
@@ -1,5 +1,6 @@
import { expectTypeOf, test } from "vitest";
+import type { VoiceSessionActions } from "../../react/voice-session/store";
import type {
PetrinautAiVoiceModeContext,
PetrinautAiVoiceModeControls,
@@ -43,3 +44,24 @@ test("accepts an existing context implementation with complete control registrat
expectTypeOf(currentContext).toEqualTypeOf();
});
+
+test("exposes optional provider-neutral speaker controls", () => {
+ expectTypeOf().toEqualTypeOf<
+ ((muted: boolean) => void) | undefined
+ >();
+ expectTypeOf<
+ PetrinautAiVoiceModeControls["setSpeakerVolume"]
+ >().toEqualTypeOf<((volume: number) => void) | undefined>();
+ expectTypeOf<
+ PetrinautAiVoiceModeSessionControls["setSpeakerMuted"]
+ >().toEqualTypeOf<((muted: boolean) => void) | undefined>();
+ expectTypeOf<
+ PetrinautAiVoiceModeSessionControls["setSpeakerVolume"]
+ >().toEqualTypeOf<((volume: number) => void) | undefined>();
+ expectTypeOf().toEqualTypeOf<
+ ((muted: boolean) => void) | undefined
+ >();
+ expectTypeOf().toEqualTypeOf<
+ ((volume: number) => void) | undefined
+ >();
+});
diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts
index 1716a4ca3e2..aa621985f6e 100644
--- a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts
+++ b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts
@@ -79,6 +79,10 @@ export type PetrinautAiVoiceModeControls = {
setMicrophoneMuted: (muted: boolean) => void;
/** Allows speech to interrupt assistant playback without clearing input. */
setInterruptionBySpeaking?: (enabled: boolean) => void;
+ /** Mutes assistant audio without changing its retained volume. */
+ setSpeakerMuted?: (muted: boolean) => void;
+ /** Sets normalized 0–1 assistant audio volume without changing mute state. */
+ setSpeakerVolume?: (volume: number) => void;
/** Cancels Voice output and hands the live microphone turn to the user. */
takeTurn?: () => Promise | void;
};
diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts
index b5b170792ca..48969c791b3 100644
--- a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts
+++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts
@@ -28,20 +28,25 @@ export const voiceSessionStatusLabel = (
};
export const voiceSessionActionLabels = {
- collapse: "Collapse voice session",
+ audioControls: "Audio controls",
+ audioOptions: "Audio options",
+ collapse: "Hide conversation",
end: "End voice mode",
- expand: "Expand voice session",
+ expand: "Show conversation",
interruptionBySpeaking: "Interruption by speaking",
mute: "Mute microphone",
+ muteSpeaker: "Mute speaker",
pause: "Pause voice mode",
- playbackOptions: "Voice playback options",
readFullResponse: "Read full response",
reconnect: "Reconnect voice mode",
repeatQuestion: "Repeat question",
retryPlayback: "Play voice audio",
resume: "Resume voice mode",
+ speakerVolume: "Speaker volume",
+ stop: "Stop AI response",
takeTurn: "Your turn",
unmute: "Unmute microphone",
+ unmuteSpeaker: "Unmute speaker",
} as const;
export const voiceSetupLabels = {
diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx
index 9198b5c6641..b799f733597 100644
--- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx
+++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx
@@ -2371,6 +2371,8 @@ describe("AiAssistantPanel composer submissions", () => {
const takeTurn = vi.fn();
const repeatQuestion = vi.fn();
const readFullResponse = vi.fn();
+ const setSpeakerMuted = vi.fn();
+ const setSpeakerVolume = vi.fn();
const VoiceMode = ({
context,
replayAllowed,
@@ -2390,6 +2392,8 @@ describe("AiAssistantPanel composer submissions", () => {
repeatQuestion,
resume: vi.fn(),
setMicrophoneMuted: vi.fn(),
+ setSpeakerMuted,
+ setSpeakerVolume,
takeTurn,
}),
[registerVoiceModeControls],
@@ -2403,6 +2407,8 @@ describe("AiAssistantPanel composer submissions", () => {
microphoneLevel: 0,
microphoneMuted: false,
phase: "speaking",
+ speakerMuted: false,
+ speakerVolume: 0.25,
});
return () => reportVoiceSessionState(null);
}, [replayAllowed, reportVoiceSessionState]);
@@ -2424,61 +2430,67 @@ describe("AiAssistantPanel composer submissions", () => {
fireEvent.click(await screen.findByRole("button", { name: "Your turn" }));
expect(takeTurn).toHaveBeenCalledOnce();
- fireEvent.click(
- screen.getByRole("button", { name: "Voice playback options" }),
- );
- const repeatQuestionItem = await screen.findByRole("menuitem", {
- name: "Repeat question",
+ const audioOptions = screen.getByRole("button", {
+ name: "Audio options",
});
- expect(repeatQuestionItem.getAttribute("aria-disabled")).not.toBe("true");
- const repeatQuestionMenu = screen.getByRole("menu");
- fireEvent.keyDown(repeatQuestionMenu, { key: "ArrowDown" });
+ expect(audioOptions.getAttribute("aria-haspopup")).toBe("dialog");
+ fireEvent.click(audioOptions);
await waitFor(() =>
- expect(repeatQuestionMenu.getAttribute("aria-activedescendant")).toBe(
- repeatQuestionItem.id,
+ expect(screen.getByRole("dialog").contains(document.activeElement)).toBe(
+ true,
),
);
- fireEvent.keyDown(repeatQuestionMenu, { key: "Enter" });
- await waitFor(() => expect(repeatQuestion).toHaveBeenCalledOnce());
+ const repeatQuestionItem = screen.getByRole("button", {
+ name: "Repeat question",
+ });
+ expect((repeatQuestionItem as HTMLButtonElement).disabled).toBe(false);
+ fireEvent.click(repeatQuestionItem);
+ expect(repeatQuestion).toHaveBeenCalledOnce();
- fireEvent.click(
- screen.getByRole("button", { name: "Voice playback options" }),
- );
- const readFullResponseItem = await screen.findByRole("menuitem", {
+ fireEvent.click(screen.getByRole("button", { name: "Mute speaker" }));
+ expect(setSpeakerMuted).toHaveBeenCalledWith(true);
+ const volume = screen.getByRole("slider", { name: "Speaker volume" });
+ volume.focus();
+ fireEvent.keyDown(volume, { key: "ArrowRight" });
+ await waitFor(() => expect(setSpeakerVolume).toHaveBeenCalledWith(0.3));
+
+ const readFullResponseItem = screen.getByRole("button", {
name: "Read full response",
});
- expect(readFullResponseItem.getAttribute("aria-disabled")).not.toBe("true");
- const readFullResponseMenu = screen.getByRole("menu");
- fireEvent.keyDown(readFullResponseMenu, { key: "End" });
- await waitFor(() =>
- expect(readFullResponseMenu.getAttribute("aria-activedescendant")).toBe(
- readFullResponseItem.id,
- ),
- );
- fireEvent.keyDown(readFullResponseMenu, { key: "Enter" });
- await waitFor(() => expect(readFullResponse).toHaveBeenCalledOnce());
+ expect((readFullResponseItem as HTMLButtonElement).disabled).toBe(false);
+ fireEvent.click(readFullResponseItem);
+ expect(readFullResponse).toHaveBeenCalledOnce();
+ fireEvent.keyDown(document.activeElement ?? document, { key: "Escape" });
+ await waitFor(() => {
+ expect(audioOptions.getAttribute("aria-expanded")).toBe("false");
+ expect(document.activeElement).toBe(audioOptions);
+ });
rendered.rerenderPanel(aiAssistant(false), editorContextValue);
- fireEvent.click(
- await screen.findByRole("button", { name: "Voice playback options" }),
- );
+ fireEvent.click(audioOptions);
expect(
(
- await screen.findByRole("menuitem", { name: "Repeat question" })
- ).getAttribute("aria-disabled"),
- ).toBe("true");
+ await screen.findByRole("button", { name: "Repeat question" })
+ ).hasAttribute("disabled"),
+ ).toBe(true);
expect(
- screen
- .getByRole("menuitem", { name: "Read full response" })
- .getAttribute("aria-disabled"),
- ).toBe("true");
+ (
+ screen.getByRole("button", {
+ name: "Read full response",
+ }) as HTMLButtonElement
+ ).disabled,
+ ).toBe(true);
});
test("retires missing and unmounted optional host Voice actions", async () => {
+ const setSpeakerMuted = vi.fn();
+ const setSpeakerVolume = vi.fn();
const VoiceMode = ({
context,
+ speakerControls,
}: {
context: PetrinautAiVoiceModeContext;
+ speakerControls: boolean;
}) => {
const { registerVoiceModeSessionControls, reportVoiceSessionState } =
context;
@@ -2488,8 +2500,9 @@ describe("AiAssistantPanel composer submissions", () => {
return registerVoiceModeSessionControls({
end: async () => undefined,
pause: vi.fn(),
+ ...(speakerControls ? { setSpeakerMuted, setSpeakerVolume } : {}),
});
- }, [registerVoiceModeSessionControls]);
+ }, [registerVoiceModeSessionControls, speakerControls]);
useEffect(() => {
reportVoiceSessionState({
canReadFullResponse: true,
@@ -2499,32 +2512,51 @@ describe("AiAssistantPanel composer submissions", () => {
microphoneLevel: 0,
microphoneMuted: false,
phase: "speaking",
+ speakerMuted: false,
+ speakerVolume: 0.5,
});
return () => reportVoiceSessionState(null);
}, [reportVoiceSessionState]);
return null;
};
- const aiAssistant = (mounted: boolean): PetrinautAiAssistant => ({
+ const aiAssistant = (
+ mounted: boolean,
+ speakerControls: boolean,
+ ): PetrinautAiAssistant => ({
renderVoiceMode: (context) =>
- mounted ? : null,
+ mounted ? (
+
+ ) : null,
transport: {
reconnectToStream: () => Promise.resolve(null),
sendMessages: vi.fn(),
},
});
- const rendered = renderTestPanel({ aiAssistant: aiAssistant(true) });
+ const rendered = renderTestPanel({
+ aiAssistant: aiAssistant(true, true),
+ });
expect(screen.queryByRole("button", { name: "Your turn" })).toBeNull();
await screen.findByRole("region", { name: "Voice session" });
- expect(
- screen.queryByRole("button", { name: "Voice playback options" }),
- ).toBeNull();
+ fireEvent.click(screen.getByRole("button", { name: "Audio options" }));
+ fireEvent.click(
+ await screen.findByRole("button", { name: "Mute speaker" }),
+ );
+ expect(setSpeakerMuted).toHaveBeenCalledWith(true);
+ fireEvent.keyDown(document.activeElement ?? document, { key: "Escape" });
+
+ rendered.rerenderPanel(aiAssistant(true, false), editorContextValue);
+ await waitFor(() =>
+ expect(
+ screen.queryByRole("button", { name: "Audio options" }),
+ ).toBeNull(),
+ );
expect(
screen.queryByRole("button", { name: "Mute microphone" }),
).toBeNull();
- rendered.rerenderPanel(aiAssistant(false), editorContextValue);
+ rendered.rerenderPanel(aiAssistant(false, false), editorContextValue);
await waitFor(() =>
expect(
diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx
index bc85f1fb6ac..ad17c65c424 100644
--- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx
+++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx
@@ -723,6 +723,18 @@ const ConversationAiAssistantPanel = ({
controls.setMicrophoneMuted?.(muted),
}
: {}),
+ ...(controls.setSpeakerMuted
+ ? {
+ setSpeakerMuted: (muted: boolean) =>
+ controls.setSpeakerMuted?.(muted),
+ }
+ : {}),
+ ...(controls.setSpeakerVolume
+ ? {
+ setSpeakerVolume: (volume: number) =>
+ controls.setSpeakerVolume?.(volume),
+ }
+ : {}),
...(controls.takeTurn ? { takeTurn: () => controls.takeTurn?.() } : {}),
});
diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx
index 22947e76be2..b86816136ef 100644
--- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx
+++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx
@@ -1,12 +1,16 @@
import { type ReactNode, useState } from "react";
-import { expect, userEvent, within } from "storybook/test";
+import { expect, userEvent, waitFor, within } from "storybook/test";
import { Button } from "@hashintel/ds-components";
import { css } from "@hashintel/ds-helpers/css";
import { NotificationsProvider } from "../../../../../react/notifications/provider";
import { VoiceSessionContext } from "../../../../../react/voice-session/context";
-import { createVoiceSessionStore } from "../../../../../react/voice-session/store";
+import {
+ createVoiceSessionStore,
+ type VoiceSessionActions,
+ type VoiceSessionStore,
+} from "../../../../../react/voice-session/store";
import { AiAssistantContents } from "./ai-assistant-contents";
import type { PetrinautAiVoiceSessionState } from "../../../../types/ai-assistant-composer-control";
@@ -225,6 +229,22 @@ const hostSlotTitleStyle = css({
fontWeight: "semibold",
});
+const frameStyle = css({
+ height: "[720px]",
+ position: "relative",
+ width: "full",
+});
+
+const narrowFrameStyle = css({
+ height: "[720px]",
+ maxWidth: "full",
+ position: "relative",
+ width: "[390px]",
+ "& > aside": {
+ maxWidth: "[calc(100% - 32px)]",
+ },
+});
+
/**
* Stands in for a host's pre-session slot. Once a session is running the host
* reports state instead of rendering, and Petrinaut's own dock takes over.
@@ -243,8 +263,65 @@ const HostVoiceSlotPreview = () => (
);
+type VoiceProvider = "live" | "realtime";
+
+const updateVoiceSessionState = (
+ store: VoiceSessionStore,
+ update: Partial,
+) => {
+ const current = store.getSnapshot().state;
+ if (current) {
+ store.setState({ ...current, ...update });
+ }
+};
+
+const createStoryVoiceSessionStore = (
+ provider: VoiceProvider,
+ state?: PetrinautAiVoiceSessionState,
+): VoiceSessionStore => {
+ const store = createVoiceSessionStore();
+ if (!state) {
+ return store;
+ }
+
+ const speakerActions = {
+ setSpeakerMuted: (speakerMuted: boolean) =>
+ updateVoiceSessionState(store, { speakerMuted }),
+ setSpeakerVolume: (speakerVolume: number) =>
+ updateVoiceSessionState(store, {
+ speakerVolume: Math.min(1, Math.max(0, speakerVolume)),
+ }),
+ };
+ const liveActions: VoiceSessionActions = {
+ end: () => {},
+ pause: () => {},
+ setMicrophoneMuted: (microphoneMuted) =>
+ updateVoiceSessionState(store, { microphoneMuted }),
+ ...speakerActions,
+ };
+
+ store.setActions(
+ provider === "live"
+ ? liveActions
+ : {
+ ...liveActions,
+ readFullResponse: () => {},
+ reconnect: () => {},
+ repeatQuestion: () => {},
+ resume: () => {},
+ setInterruptionBySpeaking: (interruptionBySpeaking) =>
+ updateVoiceSessionState(store, { interruptionBySpeaking }),
+ takeTurn: () => {},
+ },
+ );
+ store.setState(state);
+
+ return store;
+};
+
const Frame = ({
error,
+ fixedNarrowWidth = false,
initialVoiceDockCollapsed = false,
inputMode = "text",
messages,
@@ -252,9 +329,11 @@ const Frame = ({
stopped = false,
voiceMode,
voiceModeAvailable = false,
+ voiceProvider = "live",
voiceSession,
}: {
error?: Error;
+ fixedNarrowWidth?: boolean;
initialVoiceDockCollapsed?: boolean;
inputMode?: "text" | "voice";
messages: PetrinautAiMessage[];
@@ -262,6 +341,7 @@ const Frame = ({
stopped?: boolean;
voiceMode?: ReactNode;
voiceModeAvailable?: boolean;
+ voiceProvider?: VoiceProvider;
voiceSession?: PetrinautAiVoiceSessionState;
}) => {
const [input, setInput] = useState("");
@@ -270,23 +350,16 @@ const Frame = ({
);
// Stands in for the host, which reports session state rather than rendering
// the live surfaces itself.
- const [voiceSessionStore] = useState(() => {
- const store = createVoiceSessionStore();
- store.setActions({
- end: () => {},
- pause: () => {},
- reconnect: () => {},
- resume: () => {},
- setMicrophoneMuted: () => {},
- });
- store.setState(voiceSession ?? null);
-
- return store;
- });
+ const [voiceSessionStore] = useState(() =>
+ createStoryVoiceSessionStore(voiceProvider, voiceSession),
+ );
return (
-