From b7ee161478b621cedb24752d2521d01c49479537 Mon Sep 17 00:00:00 2001 From: Mark Stuart <323302876+markstuart-oai@users.noreply.github.com> Date: Tue, 27 Oct 2026 14:59:03 +0000 Subject: [PATCH 01/11] fix(examples): recognize Live replies during audio upload --- examples/live/transcript-demo.ts | 19 ++- tests/lib/live/transcript-demo.test.ts | 184 +++++++++++++++++++++++++ 2 files changed, 198 insertions(+), 5 deletions(-) create mode 100644 tests/lib/live/transcript-demo.test.ts diff --git a/examples/live/transcript-demo.ts b/examples/live/transcript-demo.ts index c3471521e7..139d9e20c0 100644 --- a/examples/live/transcript-demo.ts +++ b/examples/live/transcript-demo.ts @@ -218,14 +218,23 @@ export async function runTranscriptDemo( 'the first answer to settle', ); } + // The answer can arrive while audio is still being sent. Do not mistake the + // first answer's delayed closure for a reply to the second utterance. + const previousSegments = new Set(latest.keys()); await sendPCM(secondAudio); - const beforeAnswer = assistantCharacters; - const beforeClose = closedAssistant; await waitWithAudio( () => - assistantCharacters > beforeAnswer && - closedAssistant > beforeClose && - performance.now() - lastTextAt >= 2500, + performance.now() - lastTextAt >= 2500 && + result.segments.some(({ segment }, index, segments) => { + const previous = segments[index - 1]?.segment; + return ( + segment.speaker === 'assistant' && + previous?.speaker === 'user' && + !previousSegments.has(previous.id) && + segment.previousId === previous.id && + segment.endMs >= previous.startMs + ); + }), 'the final answer to settle', ); finishing = true; diff --git a/tests/lib/live/transcript-demo.test.ts b/tests/lib/live/transcript-demo.test.ts new file mode 100644 index 0000000000..cb63efbc72 --- /dev/null +++ b/tests/lib/live/transcript-demo.test.ts @@ -0,0 +1,184 @@ +import { once } from 'node:events'; +import { expect, it } from 'vitest'; +import { WebSocketServer } from 'ws'; + +import OpenAI from 'openai'; +import { runTranscriptDemo } from '../../../examples/live/transcript-demo'; + +const firstUser = 'Plan our picnic.'; +const secondUser = 'Make it breakfast.'; +const firstAnswer = 'There are sandwiches, fruit and a vegetarian salad.'; +const finalAnswer = 'Bring breakfast rolls and coffee.'; + +it.concurrent.each([ + { timing: 'early', interrupt: false }, + { timing: 'early', interrupt: true }, + { timing: 'delayed', interrupt: false }, + { timing: 'delayed', interrupt: true }, + { timing: 'overlap', interrupt: false }, + { timing: 'overlap', interrupt: true }, + { timing: 'stale', interrupt: true }, + { timing: 'no-answer', interrupt: true }, + { timing: 'error', interrupt: false }, + { timing: 'eof', interrupt: true }, +])('handles a $timing answer (interrupt=$interrupt)', async ({ timing, interrupt }) => { + const server = new WebSocketServer({ port: 0, host: '127.0.0.1' }); + await once(server, 'listening'); + const address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('Expected an ephemeral TCP address'); + } + const baseURL = `http://127.0.0.1:${address.port}/v1`; + const client = new OpenAI({ apiKey: 'local-test-key', baseURL }); + const timers: ReturnType[] = []; + let secondChunks = 0; + let answerAtChunk = 0; + let sentSessionClose = false; + + server.on('connection', (socket) => { + const startedAt = performance.now(); + const now = () => Math.round(performance.now() - startedAt); + const sendText = (speaker: 'input' | 'output', text: string, start: number, end: number) => + socket.send( + JSON.stringify({ + type: `session.${speaker}_transcript.delta`, + event_id: `${speaker}-${now()}`, + delta: text, + start_ms: start, + end_ms: end, + }), + ); + const sendAudio = () => + socket.send( + JSON.stringify({ + type: 'session.output_audio.delta', + event_id: `audio-${now()}`, + delta: Buffer.alloc(1920).toString('base64'), + }), + ); + let heardFirst = false; + let firstEnd = 0; + socket.on('message', (raw) => { + const event: { type: string; audio: string } = JSON.parse(raw.toString()); + if (event.type === 'session.start') { + socket.send( + JSON.stringify({ type: 'session.started', session: { id: 'fixture', status: 'active' } }), + ); + } + if (event.type === 'session.input_audio.append') { + const [marker] = Buffer.from(event.audio, 'base64'); + if (marker === 1 && !heardFirst) { + heardFirst = true; + const mark = now(); + sendText('input', firstUser, Math.max(0, mark - 20), mark); + timers.push( + setTimeout(() => { + firstEnd = now(); + sendText('output', firstAnswer, firstEnd - 50, firstEnd); + sendAudio(); + }, 720), + ); + } + if (marker === 2) { + secondChunks += 1; + if (secondChunks === 1) { + const mark = now(); + if (timing !== 'overlap') { + sendText('input', secondUser, Math.max(0, mark - 20), mark); + } + if (timing === 'stale' || timing === 'no-answer') { + timers.push( + setTimeout(() => { + // This older fragment is delivered after the new user input. + sendText('output', ' Plus lemonade.', firstEnd - 50, firstEnd); + sendAudio(); + }, 140), + ); + } + const answerDelay = timing === 'delayed' || timing === 'overlap' ? 2050 : 720; + if (timing === 'early' || timing === 'delayed' || timing === 'stale' || timing === 'overlap') { + timers.push( + setTimeout(() => { + answerAtChunk = secondChunks; + const answerEnd = now(); + if (timing === 'overlap') { + // The answer occurred during the user's long second interval, + // even though the peer delivers both transcripts afterwards. + sendText('input', secondUser, Math.max(0, mark - 20), answerEnd); + sendText('output', finalAnswer, mark + 650, mark + 1000); + } else { + sendText('output', finalAnswer, answerEnd - 50, answerEnd); + } + sendAudio(); + }, answerDelay), + ); + } + // Allow a real settled answer, then close the peer to bound a failed + // demo without waiting for its 45-second production timeout. + timers.push( + setTimeout(() => { + if (timing === 'error') { + socket.send( + JSON.stringify({ + type: 'error', + error: { type: 'invalid_request_error', code: 'fixture', message: 'Synthetic error' }, + }), + ); + } else { + socket.close(); + } + }, answerDelay + 3700), + ); + } + } + } + if (event.type === 'session.close') { + sentSessionClose = true; + socket.send( + JSON.stringify({ type: 'session.closed', reason: 'close_requested', session: { id: 'fixture' } }), + ); + socket.close(); + } + }); + }); + + try { + const result = runTranscriptDemo(client, [Buffer.alloc(1920, 1), Buffer.alloc(1920 * 40, 2)], { + interrupt, + }); + if (timing === 'no-answer' || timing === 'error' || timing === 'eof') { + await expect(result).rejects.toBeInstanceOf(Error); + expect(sentSessionClose).toBe(false); + } else { + const demo = await result; + expect(sentSessionClose).toBe(true); + expect(secondChunks).toBe(40); + if (timing === 'delayed' || timing === 'overlap') { + expect(answerAtChunk).toBe(secondChunks); + } else { + expect(answerAtChunk).toBeLessThan(secondChunks); + } + expect(demo.rawUserText).toBe(firstUser + secondUser); + expect(demo.interruptionSentDuringOutput).toBe(interrupt); + expect(demo.segments.map(({ segment }) => [segment.speaker, segment.text])).toEqual([ + ['user', firstUser], + ['assistant', firstAnswer], + ['user', secondUser], + ['assistant', (timing === 'stale' ? ' Plus lemonade.' : '') + finalAnswer], + ]); + for (const [index, { segment }] of demo.segments.entries()) { + expect(segment.previousId).toBe(demo.segments[index - 1]?.segment.id ?? null); + } + } + } finally { + for (const timer of timers) { + clearTimeout(timer); + } + const closed = once(server, 'close'); + for (const peer of server.clients) { + peer.terminate(); + } + server.close(); + await closed; + } +}); From d79d8ea766c76aa9e8b7d0ba0ed97f40e825a93a Mon Sep 17 00:00:00 2001 From: Mark Stuart <323302876+markstuart-oai@users.noreply.github.com> Date: Wed, 28 Oct 2026 02:59:22 +0000 Subject: [PATCH 02/11] fix(examples): respect transcript source timing and audio activity --- examples/live/transcript-demo.ts | 34 +++++++++--- tests/lib/live/transcript-demo.test.ts | 74 ++++++++++++++++++-------- 2 files changed, 77 insertions(+), 31 deletions(-) diff --git a/examples/live/transcript-demo.ts b/examples/live/transcript-demo.ts index 139d9e20c0..14130a8ac6 100644 --- a/examples/live/transcript-demo.ts +++ b/examples/live/transcript-demo.ts @@ -49,6 +49,11 @@ export async function prepareTranscriptDemo(client: OpenAI): Promise { * Send pre-generated PCM through the public Live WebSocket and group only its * public transcript events. This makes paid API requests; requires Live access. * No microphone, local TTS latency, internal API, or server turn events are used. + * Completion is a best-effort observation of speech after the second input, + * followed by transcript and audio inactivity. Live's public transcript does not + * identify managed backend turns. A pause after an acknowledgment can still look + * like completion, so this demo cannot guarantee that a final answer has arrived. + * In an interactive application, leave the session open until the user closes it. */ export async function runTranscriptDemo( client: OpenAI, @@ -93,6 +98,9 @@ export async function runTranscriptDemo( let lastTextAt = Number.NEGATIVE_INFINITY; let assistantCharacters = 0; let closedAssistant = 0; + let secondAudioStarted = false; + let secondInputStartMs: number | undefined; + let secondOutputStartMs = Number.NEGATIVE_INFINITY; const latest = new Map(); const closedIds = new Set(); transcript.on('segment.updated', (segment) => { @@ -128,11 +136,17 @@ export async function runTranscriptDemo( if (event.type === 'session.input_transcript.delta') { result.rawTranscriptEvents += 1; result.rawUserText += event.delta; + if (secondAudioStarted && event.delta) { + secondInputStartMs ??= event.start_ms; + } } if (event.type === 'session.output_transcript.delta') { result.rawTranscriptEvents += 1; assistantCharacters += event.delta.length; lastTextAt = performance.now(); + if (secondAudioStarted && event.delta) { + secondOutputStartMs = Math.max(secondOutputStartMs, event.start_ms); + } } if (event.type === 'session.output_audio.delta') { lastAudioAt = performance.now(); @@ -214,28 +228,32 @@ export async function runTranscriptDemo( result.interruptionSentDuringOutput = true; } else { await waitWithAudio( - () => closedAssistant > 0 && performance.now() - lastTextAt >= 2500, + () => closedAssistant > 0 && performance.now() - Math.max(lastTextAt, lastAudioAt) >= 2500, 'the first answer to settle', ); } - // The answer can arrive while audio is still being sent. Do not mistake the - // first answer's delayed closure for a reply to the second utterance. - const previousSegments = new Set(latest.keys()); + // Snapshot text, not just IDs: the second utterance can extend an active + // user bubble. A delayed old output interval must not count merely because + // its end overlaps the new input. + const previousSegments = new Map(latest); + secondAudioStarted = true; await sendPCM(secondAudio); await waitWithAudio( () => - performance.now() - lastTextAt >= 2500 && + performance.now() - Math.max(lastTextAt, lastAudioAt) >= 2500 && + secondInputStartMs !== undefined && + secondOutputStartMs >= secondInputStartMs && result.segments.some(({ segment }, index, segments) => { const previous = segments[index - 1]?.segment; return ( segment.speaker === 'assistant' && previous?.speaker === 'user' && - !previousSegments.has(previous.id) && + previousSegments.get(previous.id) !== previous.text && segment.previousId === previous.id && - segment.endMs >= previous.startMs + segment.endMs >= secondOutputStartMs ); }), - 'the final answer to settle', + 'speech after the second input to settle', ); finishing = true; live.send({ type: 'session.close' }); diff --git a/tests/lib/live/transcript-demo.test.ts b/tests/lib/live/transcript-demo.test.ts index cb63efbc72..b1a893f68e 100644 --- a/tests/lib/live/transcript-demo.test.ts +++ b/tests/lib/live/transcript-demo.test.ts @@ -19,9 +19,13 @@ it.concurrent.each([ { timing: 'overlap', interrupt: true }, { timing: 'stale', interrupt: true }, { timing: 'no-answer', interrupt: true }, + { timing: 'straddling-stale-only', interrupt: true }, + { timing: 'reused-user', interrupt: true }, + { timing: 'audio-tail', interrupt: false }, { timing: 'error', interrupt: false }, { timing: 'eof', interrupt: true }, ])('handles a $timing answer (interrupt=$interrupt)', async ({ timing, interrupt }) => { + const expectsAnswer = !['no-answer', 'straddling-stale-only', 'error', 'eof'].includes(timing); const server = new WebSocketServer({ port: 0, host: '127.0.0.1' }); await once(server, 'listening'); const address = server.address(); @@ -34,6 +38,7 @@ it.concurrent.each([ let secondChunks = 0; let answerAtChunk = 0; let sentSessionClose = false; + let audioFrames = 0; server.on('connection', (socket) => { const startedAt = performance.now(); @@ -48,7 +53,8 @@ it.concurrent.each([ end_ms: end, }), ); - const sendAudio = () => + const sendAudio = () => { + audioFrames += 1; socket.send( JSON.stringify({ type: 'session.output_audio.delta', @@ -56,6 +62,7 @@ it.concurrent.each([ delta: Buffer.alloc(1920).toString('base64'), }), ); + }; let heardFirst = false; let firstEnd = 0; socket.on('message', (raw) => { @@ -70,12 +77,17 @@ it.concurrent.each([ if (marker === 1 && !heardFirst) { heardFirst = true; const mark = now(); - sendText('input', firstUser, Math.max(0, mark - 20), mark); + sendText('input', timing === 'reused-user' ? 'Plan our ' : firstUser, Math.max(0, mark - 20), mark); timers.push( setTimeout(() => { firstEnd = now(); sendText('output', firstAnswer, firstEnd - 50, firstEnd); sendAudio(); + if (timing === 'reused-user') { + // A delayed user fragment leaves the same user bubble active + // across the start of the second clip. + sendText('input', 'picnic.', firstEnd + 1, firstEnd + 2); + } }, 720), ); } @@ -86,17 +98,22 @@ it.concurrent.each([ if (timing !== 'overlap') { sendText('input', secondUser, Math.max(0, mark - 20), mark); } - if (timing === 'stale' || timing === 'no-answer') { + if (timing === 'stale' || timing === 'no-answer' || timing === 'straddling-stale-only') { timers.push( setTimeout(() => { // This older fragment is delivered after the new user input. - sendText('output', ' Plus lemonade.', firstEnd - 50, firstEnd); + sendText( + 'output', + ' Plus lemonade.', + firstEnd - 50, + timing === 'straddling-stale-only' ? mark + 500 : firstEnd, + ); sendAudio(); }, 140), ); } const answerDelay = timing === 'delayed' || timing === 'overlap' ? 2050 : 720; - if (timing === 'early' || timing === 'delayed' || timing === 'stale' || timing === 'overlap') { + if (expectsAnswer) { timers.push( setTimeout(() => { answerAtChunk = secondChunks; @@ -113,21 +130,29 @@ it.concurrent.each([ }, answerDelay), ); } + if (timing === 'audio-tail') { + for (const delay of [1800, 2400, 3200, 4000, 4800]) { + timers.push(setTimeout(sendAudio, delay)); + } + } // Allow a real settled answer, then close the peer to bound a failed // demo without waiting for its 45-second production timeout. timers.push( - setTimeout(() => { - if (timing === 'error') { - socket.send( - JSON.stringify({ - type: 'error', - error: { type: 'invalid_request_error', code: 'fixture', message: 'Synthetic error' }, - }), - ); - } else { - socket.close(); - } - }, answerDelay + 3700), + setTimeout( + () => { + if (timing === 'error') { + socket.send( + JSON.stringify({ + type: 'error', + error: { type: 'invalid_request_error', code: 'fixture', message: 'Synthetic error' }, + }), + ); + } else { + socket.close(); + } + }, + timing === 'audio-tail' ? 9000 : answerDelay + 3700, + ), ); } } @@ -146,10 +171,7 @@ it.concurrent.each([ const result = runTranscriptDemo(client, [Buffer.alloc(1920, 1), Buffer.alloc(1920 * 40, 2)], { interrupt, }); - if (timing === 'no-answer' || timing === 'error' || timing === 'eof') { - await expect(result).rejects.toBeInstanceOf(Error); - expect(sentSessionClose).toBe(false); - } else { + if (expectsAnswer) { const demo = await result; expect(sentSessionClose).toBe(true); expect(secondChunks).toBe(40); @@ -160,15 +182,21 @@ it.concurrent.each([ } expect(demo.rawUserText).toBe(firstUser + secondUser); expect(demo.interruptionSentDuringOutput).toBe(interrupt); + if (timing === 'audio-tail') { + expect(audioFrames).toBe(7); + } expect(demo.segments.map(({ segment }) => [segment.speaker, segment.text])).toEqual([ - ['user', firstUser], + ['user', timing === 'reused-user' ? 'Plan our ' : firstUser], ['assistant', firstAnswer], - ['user', secondUser], + ['user', (timing === 'reused-user' ? 'picnic.' : '') + secondUser], ['assistant', (timing === 'stale' ? ' Plus lemonade.' : '') + finalAnswer], ]); for (const [index, { segment }] of demo.segments.entries()) { expect(segment.previousId).toBe(demo.segments[index - 1]?.segment.id ?? null); } + } else { + await expect(result).rejects.toBeInstanceOf(Error); + expect(sentSessionClose).toBe(false); } } finally { for (const timer of timers) { From edcda564f78a7150e11ac0c79127e18d0a37d408 Mon Sep 17 00:00:00 2001 From: Mark Stuart <323302876+markstuart-oai@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:02:17 +0000 Subject: [PATCH 03/11] examples: await Live transcript finalization --- examples/live/transcript-demo.ts | 14 ++++-- tests/lib/live/transcript-demo.test.ts | 68 ++++++++++++++++++++++---- 2 files changed, 69 insertions(+), 13 deletions(-) diff --git a/examples/live/transcript-demo.ts b/examples/live/transcript-demo.ts index 14130a8ac6..7e449071c4 100644 --- a/examples/live/transcript-demo.ts +++ b/examples/live/transcript-demo.ts @@ -94,6 +94,7 @@ export async function runTranscriptDemo( }; let ready = false; let finishing = false; + let sessionClosed = false; let lastAudioAt = Number.NEGATIVE_INFINITY; let lastTextAt = Number.NEGATIVE_INFINITY; let assistantCharacters = 0; @@ -125,7 +126,7 @@ export async function runTranscriptDemo( live.on('error', (error) => abort.abort(error)); live.on('close', () => { - if (!finishing) { + if (!finishing || !sessionClosed) { abort.abort(new Error('Live transport closed before the demo completed')); } }); @@ -157,6 +158,10 @@ export async function runTranscriptDemo( } transcript.push(event); if (event.type === 'session.closed') { + if (!finishing) { + abort.abort(new Error('Live session ended before the demo completed')); + } + sessionClosed = true; live.off('event', onEvent); } } catch (error) { @@ -257,8 +262,11 @@ export async function runTranscriptDemo( ); finishing = true; live.send({ type: 'session.close' }); - // Give the public session.closed event a chance to finalize remaining text. - await sleep(300, undefined, { signal: abort.signal }); + // Confirm finalization before closing the transport or returning the transcript. + // oxlint-disable-next-line no-unmodified-loop-condition -- The session.closed event listener sets sessionClosed. + while (!sessionClosed) { + await sleep(20, undefined, { signal: abort.signal }); + } } finally { finishing = true; clearTimeout(timeout); diff --git a/tests/lib/live/transcript-demo.test.ts b/tests/lib/live/transcript-demo.test.ts index b1a893f68e..c59f2f2928 100644 --- a/tests/lib/live/transcript-demo.test.ts +++ b/tests/lib/live/transcript-demo.test.ts @@ -22,10 +22,23 @@ it.concurrent.each([ { timing: 'straddling-stale-only', interrupt: true }, { timing: 'reused-user', interrupt: true }, { timing: 'audio-tail', interrupt: false }, + { timing: 'delayed-finalization', interrupt: false }, + { timing: 'eof-finalization', interrupt: false }, + { timing: 'premature-finalization', interrupt: false }, + { timing: 'premature-finalization-open', interrupt: false }, { timing: 'error', interrupt: false }, { timing: 'eof', interrupt: true }, ])('handles a $timing answer (interrupt=$interrupt)', async ({ timing, interrupt }) => { - const expectsAnswer = !['no-answer', 'straddling-stale-only', 'error', 'eof'].includes(timing); + const expectsAnswer = ![ + 'no-answer', + 'straddling-stale-only', + 'error', + 'eof', + 'eof-finalization', + 'premature-finalization', + 'premature-finalization-open', + ].includes(timing); + const receivesAnswer = expectsAnswer || timing === 'eof-finalization'; const server = new WebSocketServer({ port: 0, host: '127.0.0.1' }); await once(server, 'listening'); const address = server.address(); @@ -38,6 +51,8 @@ it.concurrent.each([ let secondChunks = 0; let answerAtChunk = 0; let sentSessionClose = false; + let sentSessionClosed = false; + let forcedPrematureClose = false; let audioFrames = 0; server.on('connection', (socket) => { @@ -63,14 +78,31 @@ it.concurrent.each([ }), ); }; + const startSession = () => { + if (timing === 'premature-finalization' || timing === 'premature-finalization-open') { + socket.send( + JSON.stringify({ type: 'session.closed', reason: 'close_requested', session: { id: 'fixture' } }), + ); + if (timing === 'premature-finalization-open') { + timers.push( + setTimeout(() => { + forcedPrematureClose = true; + socket.close(); + }, 3000), + ); + } else { + socket.close(); + } + return; + } + socket.send(JSON.stringify({ type: 'session.started', session: { id: 'fixture', status: 'active' } })); + }; let heardFirst = false; let firstEnd = 0; socket.on('message', (raw) => { const event: { type: string; audio: string } = JSON.parse(raw.toString()); if (event.type === 'session.start') { - socket.send( - JSON.stringify({ type: 'session.started', session: { id: 'fixture', status: 'active' } }), - ); + startSession(); } if (event.type === 'session.input_audio.append') { const [marker] = Buffer.from(event.audio, 'base64'); @@ -113,7 +145,7 @@ it.concurrent.each([ ); } const answerDelay = timing === 'delayed' || timing === 'overlap' ? 2050 : 720; - if (expectsAnswer) { + if (receivesAnswer) { timers.push( setTimeout(() => { answerAtChunk = secondChunks; @@ -159,10 +191,24 @@ it.concurrent.each([ } if (event.type === 'session.close') { sentSessionClose = true; - socket.send( - JSON.stringify({ type: 'session.closed', reason: 'close_requested', session: { id: 'fixture' } }), - ); - socket.close(); + const finalize = () => { + if (timing !== 'eof-finalization') { + sentSessionClosed = true; + socket.send( + JSON.stringify({ + type: 'session.closed', + reason: 'close_requested', + session: { id: 'fixture' }, + }), + ); + } + socket.close(); + }; + if (timing === 'delayed-finalization') { + timers.push(setTimeout(finalize, 650)); + } else { + finalize(); + } } }); }); @@ -174,6 +220,7 @@ it.concurrent.each([ if (expectsAnswer) { const demo = await result; expect(sentSessionClose).toBe(true); + expect(sentSessionClosed).toBe(true); expect(secondChunks).toBe(40); if (timing === 'delayed' || timing === 'overlap') { expect(answerAtChunk).toBe(secondChunks); @@ -196,7 +243,8 @@ it.concurrent.each([ } } else { await expect(result).rejects.toBeInstanceOf(Error); - expect(sentSessionClose).toBe(false); + expect(sentSessionClose).toBe(timing === 'eof-finalization'); + expect(forcedPrematureClose).toBe(false); } } finally { for (const timer of timers) { From 9e5dbc740c0a5e1656d6a02cf868891f364cc0f7 Mon Sep 17 00:00:00 2001 From: Mark Stuart <323302876+markstuart-oai@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:11:27 +0000 Subject: [PATCH 04/11] docs(live): clarify limits when inferring transcript completion --- examples/live/transcript-demo.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/live/transcript-demo.ts b/examples/live/transcript-demo.ts index 7e449071c4..b24ef7f857 100644 --- a/examples/live/transcript-demo.ts +++ b/examples/live/transcript-demo.ts @@ -49,10 +49,11 @@ export async function prepareTranscriptDemo(client: OpenAI): Promise { * Send pre-generated PCM through the public Live WebSocket and group only its * public transcript events. This makes paid API requests; requires Live access. * No microphone, local TTS latency, internal API, or server turn events are used. - * Completion is a best-effort observation of speech after the second input, - * followed by transcript and audio inactivity. Live's public transcript does not - * identify managed backend turns. A pause after an acknowledgment can still look - * like completion, so this demo cannot guarantee that a final answer has arrived. + * Completion is a best-effort guess from received events and a period of transcript + * and audio inactivity. Live's public transcript does not identify managed turns + * or map each fragment to an audio append. Delayed earlier speech can be mistaken + * for the new clip, and an acknowledgment can be mistaken for a final answer. + * The demo can stop before the second transcript or final answer arrives. * In an interactive application, leave the session open until the user closes it. */ export async function runTranscriptDemo( From 1cb75add2fab4b04b609067782cb0477288b5d28 Mon Sep 17 00:00:00 2001 From: Mark Stuart <323302876+markstuart-oai@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:24:29 +0000 Subject: [PATCH 05/11] fix(live): let callers finish transcript capture explicitly --- examples/live/README.md | 4 +- examples/live/transcript-demo.ts | 58 ++++------ examples/live/transcript-grouper.ts | 13 +++ tests/lib/live/transcript-demo.test.ts | 118 ++++++++++++++++++++- tests/live/transcript-grouper.live.test.ts | 3 + 5 files changed, 151 insertions(+), 45 deletions(-) diff --git a/examples/live/README.md b/examples/live/README.md index 8527616c31..483ec300a5 100644 --- a/examples/live/README.md +++ b/examples/live/README.md @@ -56,7 +56,9 @@ pnpm tsn -T examples/live/transcript-grouper.ts ``` Audio playback requires `ffplay` on your `PATH`. To run without playback, add -`--no-audio`. +`--no-audio`. Press Enter to finish capturing the transcript. The example +requests closure and waits for the server's confirmation. It does not infer that +the model finished an answer from a pause or transcript fragment. ## WebRTC with a backend sideband diff --git a/examples/live/transcript-demo.ts b/examples/live/transcript-demo.ts index b24ef7f857..17eeba4c11 100644 --- a/examples/live/transcript-demo.ts +++ b/examples/live/transcript-demo.ts @@ -49,17 +49,16 @@ export async function prepareTranscriptDemo(client: OpenAI): Promise { * Send pre-generated PCM through the public Live WebSocket and group only its * public transcript events. This makes paid API requests; requires Live access. * No microphone, local TTS latency, internal API, or server turn events are used. - * Completion is a best-effort guess from received events and a period of transcript - * and audio inactivity. Live's public transcript does not identify managed turns - * or map each fragment to an audio append. Delayed earlier speech can be mistaken - * for the new clip, and an acknowledgment can be mistaken for a final answer. - * The demo can stop before the second transcript or final answer arrives. - * In an interactive application, leave the session open until the user closes it. + * The caller ends capture with finishSignal; transcript activity never ends it. + * This controls capture duration, not whether the model finished an answer. + * The result is returned only after the server confirms session.closed. */ export async function runTranscriptDemo( client: OpenAI, audio: readonly Buffer[], options: { + /** End capture when the caller or user decides. This is separate from operation failures. */ + finishSignal: AbortSignal; /** Optional transport options, for example a local opt-in compatibility header. */ websocketOptions?: LiveWSClientOptions | undefined; /** Interrupt the first answer instead of waiting for its projected completion. */ @@ -72,7 +71,7 @@ export async function runTranscriptDemo( onInputAudio?: (pcm: Buffer) => void; /** Observe incoming mono 24 kHz PCM independently of transcript grouping. */ onOutputAudio?: (pcm: Buffer) => void; - } = {}, + }, ): Promise { const [firstAudio, secondAudio] = audio; if (!firstAudio?.length || !secondAudio?.length) { @@ -100,9 +99,6 @@ export async function runTranscriptDemo( let lastTextAt = Number.NEGATIVE_INFINITY; let assistantCharacters = 0; let closedAssistant = 0; - let secondAudioStarted = false; - let secondInputStartMs: number | undefined; - let secondOutputStartMs = Number.NEGATIVE_INFINITY; const latest = new Map(); const closedIds = new Set(); transcript.on('segment.updated', (segment) => { @@ -138,17 +134,11 @@ export async function runTranscriptDemo( if (event.type === 'session.input_transcript.delta') { result.rawTranscriptEvents += 1; result.rawUserText += event.delta; - if (secondAudioStarted && event.delta) { - secondInputStartMs ??= event.start_ms; - } } if (event.type === 'session.output_transcript.delta') { result.rawTranscriptEvents += 1; assistantCharacters += event.delta.length; lastTextAt = performance.now(); - if (secondAudioStarted && event.delta) { - secondOutputStartMs = Math.max(secondOutputStartMs, event.start_ms); - } } if (event.type === 'session.output_audio.delta') { lastAudioAt = performance.now(); @@ -190,6 +180,9 @@ export async function runTranscriptDemo( const startedAt = performance.now(); for (let offset = 0; offset < pcm.length; offset += CHUNK_BYTES) { abort.signal.throwIfAborted(); + if (options.finishSignal.aborted) { + return; + } const chunk = pcm.subarray(offset, offset + CHUNK_BYTES); live.send({ type: 'session.input_audio.append', @@ -205,7 +198,8 @@ export async function runTranscriptDemo( const silence = Buffer.alloc(CHUNK_BYTES * 5); async function waitWithAudio(predicate: () => boolean, description: string): Promise { const deadline = performance.now() + 45_000; - while (!predicate()) { + while (!predicate() && !options.finishSignal.aborted) { + abort.signal.throwIfAborted(); if (performance.now() >= deadline) { throw new Error( `Timed out waiting for ${description} (${assistantCharacters} assistant characters, ${closedAssistant} closed assistant segments, ${Math.round(performance.now() - lastTextAt)} ms since the last output transcript)`, @@ -231,36 +225,20 @@ export async function runTranscriptDemo( performance.now() - lastTextAt < 1000, 'the assistant to speak before interrupting', ); - result.interruptionSentDuringOutput = true; + result.interruptionSentDuringOutput = !options.finishSignal.aborted; } else { await waitWithAudio( () => closedAssistant > 0 && performance.now() - Math.max(lastTextAt, lastAudioAt) >= 2500, 'the first answer to settle', ); } - // Snapshot text, not just IDs: the second utterance can extend an active - // user bubble. A delayed old output interval must not count merely because - // its end overlaps the new input. - const previousSegments = new Map(latest); - secondAudioStarted = true; await sendPCM(secondAudio); - await waitWithAudio( - () => - performance.now() - Math.max(lastTextAt, lastAudioAt) >= 2500 && - secondInputStartMs !== undefined && - secondOutputStartMs >= secondInputStartMs && - result.segments.some(({ segment }, index, segments) => { - const previous = segments[index - 1]?.segment; - return ( - segment.speaker === 'assistant' && - previous?.speaker === 'user' && - previousSegments.get(previous.id) !== previous.text && - segment.previousId === previous.id && - segment.endMs >= secondOutputStartMs - ); - }), - 'speech after the second input to settle', - ); + // Live has no public turn-final marker. Keep the duplex input running until + // the caller explicitly ends capture instead of guessing from transcript gaps. + while (!options.finishSignal.aborted) { + await sendPCM(silence); + } + abort.signal.throwIfAborted(); finishing = true; live.send({ type: 'session.close' }); // Confirm finalization before closing the transport or returning the transcript. diff --git a/examples/live/transcript-grouper.ts b/examples/live/transcript-grouper.ts index 6e8a956efc..499cfc762f 100644 --- a/examples/live/transcript-grouper.ts +++ b/examples/live/transcript-grouper.ts @@ -1,3 +1,5 @@ +import { createInterface } from 'node:readline'; +import type { Interface } from 'node:readline'; import OpenAI from 'openai'; import { prepareTranscriptDemo, runTranscriptDemo } from './transcript-demo'; import { createPCMPlayback } from './transcript-playback'; @@ -6,13 +8,21 @@ async function main(): Promise { const playback = process.argv.includes('--no-audio') ? undefined : { user: createPCMPlayback(), assistant: createPCMPlayback() }; + const finish = new AbortController(); + let input: Interface | undefined; + const stop = () => finish.abort(); try { const client = new OpenAI({ logLevel: 'off' }); console.log('All voices are AI-generated. Preparing both TTS clips before connecting to Live…'); const audio = await prepareTranscriptDemo(client); console.log(playback ? 'Playing both voices through playAudio (ffplay).' : 'Audio playback disabled.'); console.log('TTS ready. Starting a picnic conversation, then interrupting with a breakfast request.'); + input = createInterface({ input: process.stdin, output: process.stdout }); + input.once('line', stop); + input.once('close', stop); + console.log('Press Enter to finish capturing the transcript and close the session.'); const result = await runTranscriptDemo(client, audio, { + finishSignal: finish.signal, interrupt: true, onInputAudio: (pcm) => playback?.user.write(pcm), onOutputAudio: (pcm) => playback?.assistant.write(pcm), @@ -28,6 +38,9 @@ async function main(): Promise { console.error(key ? message.split(key).join('[REDACTED]') : message); process.exitCode = 1; } finally { + input?.removeListener('line', stop); + input?.removeListener('close', stop); + input?.close(); await Promise.all([playback?.user.close(), playback?.assistant.close()]); } } diff --git a/tests/lib/live/transcript-demo.test.ts b/tests/lib/live/transcript-demo.test.ts index c59f2f2928..a07ab4122a 100644 --- a/tests/lib/live/transcript-demo.test.ts +++ b/tests/lib/live/transcript-demo.test.ts @@ -9,6 +9,7 @@ const firstUser = 'Plan our picnic.'; const secondUser = 'Make it breakfast.'; const firstAnswer = 'There are sandwiches, fruit and a vegetarian salad.'; const finalAnswer = 'Bring breakfast rolls and coffee.'; +const acknowledgment = 'I understand the change and will think about a breakfast picnic.'; it.concurrent.each([ { timing: 'early', interrupt: false }, @@ -21,6 +22,8 @@ it.concurrent.each([ { timing: 'no-answer', interrupt: true }, { timing: 'straddling-stale-only', interrupt: true }, { timing: 'reused-user', interrupt: true }, + { timing: 'delayed-first-input', interrupt: true }, + { timing: 'long-acknowledgment', interrupt: true }, { timing: 'audio-tail', interrupt: false }, { timing: 'delayed-finalization', interrupt: false }, { timing: 'eof-finalization', interrupt: false }, @@ -54,6 +57,13 @@ it.concurrent.each([ let sentSessionClosed = false; let forcedPrematureClose = false; let audioFrames = 0; + let answerSent = false; + const finish = new AbortController(); + const finishAfterExpectedInputAndAnswer = (): void => { + if (secondChunks === 40 && answerSent && timing !== 'audio-tail') { + finish.abort(); + } + }; server.on('connection', (socket) => { const startedAt = performance.now(); @@ -99,6 +109,15 @@ it.concurrent.each([ }; let heardFirst = false; let firstEnd = 0; + let firstMark = 0; + const answerDelay = + new Map([ + ['delayed', 2050], + ['overlap', 2050], + ['delayed-first-input', 4300], + ['long-acknowledgment', 4300], + ]).get(timing) ?? 720; + const firstInputFragmented = ['reused-user', 'delayed-first-input'].includes(timing); socket.on('message', (raw) => { const event: { type: string; audio: string } = JSON.parse(raw.toString()); if (event.type === 'session.start') { @@ -109,7 +128,8 @@ it.concurrent.each([ if (marker === 1 && !heardFirst) { heardFirst = true; const mark = now(); - sendText('input', timing === 'reused-user' ? 'Plan our ' : firstUser, Math.max(0, mark - 20), mark); + firstMark = mark; + sendText('input', firstInputFragmented ? 'Plan our ' : firstUser, Math.max(0, mark - 20), mark); timers.push( setTimeout(() => { firstEnd = now(); @@ -127,7 +147,11 @@ it.concurrent.each([ secondChunks += 1; if (secondChunks === 1) { const mark = now(); - if (timing !== 'overlap') { + if (timing === 'delayed-first-input') { + sendText('input', 'picnic.', firstMark, firstMark + 10); + sendText('output', ' Plus lemonade.', firstEnd + 1, firstEnd + 40); + sendAudio(); + } else if (timing !== 'overlap') { sendText('input', secondUser, Math.max(0, mark - 20), mark); } if (timing === 'stale' || timing === 'no-answer' || timing === 'straddling-stale-only') { @@ -144,7 +168,15 @@ it.concurrent.each([ }, 140), ); } - const answerDelay = timing === 'delayed' || timing === 'overlap' ? 2050 : 720; + if (timing === 'long-acknowledgment') { + timers.push( + setTimeout(() => { + const ackEnd = now(); + sendText('output', acknowledgment, ackEnd - 200, ackEnd); + sendAudio(); + }, 400), + ); + } if (receivesAnswer) { timers.push( setTimeout(() => { @@ -156,9 +188,14 @@ it.concurrent.each([ sendText('input', secondUser, Math.max(0, mark - 20), answerEnd); sendText('output', finalAnswer, mark + 650, mark + 1000); } else { + if (timing === 'delayed-first-input') { + sendText('input', secondUser, mark, mark + 1600); + } sendText('output', finalAnswer, answerEnd - 50, answerEnd); } sendAudio(); + answerSent = true; + finishAfterExpectedInputAndAnswer(); }, answerDelay), ); } @@ -166,6 +203,7 @@ it.concurrent.each([ for (const delay of [1800, 2400, 3200, 4000, 4800]) { timers.push(setTimeout(sendAudio, delay)); } + timers.push(setTimeout(() => finish.abort(), 5200)); } // Allow a real settled answer, then close the peer to bound a failed // demo without waiting for its 45-second production timeout. @@ -187,6 +225,7 @@ it.concurrent.each([ ), ); } + finishAfterExpectedInputAndAnswer(); } } if (event.type === 'session.close') { @@ -215,6 +254,7 @@ it.concurrent.each([ try { const result = runTranscriptDemo(client, [Buffer.alloc(1920, 1), Buffer.alloc(1920 * 40, 2)], { + finishSignal: finish.signal, interrupt, }); if (expectsAnswer) { @@ -222,13 +262,22 @@ it.concurrent.each([ expect(sentSessionClose).toBe(true); expect(sentSessionClosed).toBe(true); expect(secondChunks).toBe(40); - if (timing === 'delayed' || timing === 'overlap') { + if (['delayed', 'overlap', 'delayed-first-input', 'long-acknowledgment'].includes(timing)) { expect(answerAtChunk).toBe(secondChunks); } else { expect(answerAtChunk).toBeLessThan(secondChunks); } expect(demo.rawUserText).toBe(firstUser + secondUser); expect(demo.interruptionSentDuringOutput).toBe(interrupt); + if (timing === 'delayed-first-input' || timing === 'long-acknowledgment') { + expect(answerSent).toBe(true); + const assistant = demo.segments + .filter(({ segment }) => segment.speaker === 'assistant') + .map(({ segment }) => segment.text) + .join(''); + expect(assistant).toContain(finalAnswer); + return; + } if (timing === 'audio-tail') { expect(audioFrames).toBe(7); } @@ -258,3 +307,64 @@ it.concurrent.each([ await closed; } }); + +it('finishes during the first input without sending the second clip, and waits for session.closed', async () => { + const server = new WebSocketServer({ port: 0, host: '127.0.0.1' }); + await once(server, 'listening'); + const address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('Expected an ephemeral TCP address'); + } + const client = new OpenAI({ apiKey: 'local-test-key', baseURL: `http://127.0.0.1:${address.port}/v1` }); + const finish = new AbortController(); + const markers: number[] = []; + let sawSessionClose = false; + let sawSessionClosed = false; + const timers: ReturnType[] = []; + server.on('connection', (socket) => { + socket.on('message', (raw) => { + const event: { type: string; audio: string } = JSON.parse(raw.toString()); + if (event.type === 'session.start') { + socket.send(JSON.stringify({ type: 'session.started', session: { id: 'early', status: 'active' } })); + } else if (event.type === 'session.input_audio.append') { + const [marker] = Buffer.from(event.audio, 'base64'); + expect(marker).toBeDefined(); + if (marker !== undefined) { + markers.push(marker); + } + finish.abort(); + } else if (event.type === 'session.close') { + sawSessionClose = true; + timers.push( + setTimeout(() => { + sawSessionClosed = true; + socket.send( + JSON.stringify({ type: 'session.closed', session: { id: 'early', status: 'active' } }), + ); + socket.close(); + }, 200), + ); + } + }); + }); + try { + const result = await runTranscriptDemo(client, [Buffer.alloc(1920 * 40, 1), Buffer.alloc(1920 * 40, 2)], { + interrupt: true, + finishSignal: finish.signal, + }); + expect(markers).toEqual([1]); + expect(sawSessionClose).toBe(true); + expect(sawSessionClosed).toBe(true); + expect(result.interruptionSentDuringOutput).toBe(false); + } finally { + for (const timer of timers) { + clearTimeout(timer); + } + const closed = once(server, 'close'); + for (const socket of server.clients) { + socket.terminate(); + } + server.close(); + await closed; + } +}); diff --git a/tests/live/transcript-grouper.live.test.ts b/tests/live/transcript-grouper.live.test.ts index 9c33cfc982..ee0fc0ff63 100644 --- a/tests/live/transcript-grouper.live.test.ts +++ b/tests/live/transcript-grouper.live.test.ts @@ -21,6 +21,9 @@ describeLive('Transcript grouper with TTS input over the public Live API', () => 'groups a real conversation (interrupt: %s)', async (interrupt) => { const result = await runTranscriptDemo(client, audio, { + // This test controls its observation window; it does not infer a final + // answer from live speech. Assert what was actually captured below. + finishSignal: AbortSignal.timeout(45_000), interrupt, websocketOptions: process.env['OPENAI_LIVE_COMPATIBILITY_HEADER'] ? { headers: { 'OpenAI-Alpha': process.env['OPENAI_LIVE_COMPATIBILITY_HEADER'] } } From a2a618eabe80ca484c5fd5f9e1002525a5ecef86 Mon Sep 17 00:00:00 2001 From: Mark Stuart <323302876+markstuart-oai@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:30:58 +0000 Subject: [PATCH 06/11] fix: reject unexpected Live transcript session closure --- examples/live/transcript-demo.ts | 2 + tests/lib/live/transcript-demo.test.ts | 125 ++++++++++++++----------- 2 files changed, 70 insertions(+), 57 deletions(-) diff --git a/examples/live/transcript-demo.ts b/examples/live/transcript-demo.ts index 17eeba4c11..84150b9dcb 100644 --- a/examples/live/transcript-demo.ts +++ b/examples/live/transcript-demo.ts @@ -151,6 +151,8 @@ export async function runTranscriptDemo( if (event.type === 'session.closed') { if (!finishing) { abort.abort(new Error('Live session ended before the demo completed')); + } else if (event.reason !== 'close_requested') { + abort.abort(new Error('Live session ended without confirming the requested close')); } sessionClosed = true; live.off('event', onEvent); diff --git a/tests/lib/live/transcript-demo.test.ts b/tests/lib/live/transcript-demo.test.ts index a07ab4122a..359d387875 100644 --- a/tests/lib/live/transcript-demo.test.ts +++ b/tests/lib/live/transcript-demo.test.ts @@ -308,63 +308,74 @@ it.concurrent.each([ } }); -it('finishes during the first input without sending the second clip, and waits for session.closed', async () => { - const server = new WebSocketServer({ port: 0, host: '127.0.0.1' }); - await once(server, 'listening'); - const address = server.address(); - if (!address || typeof address === 'string') { - throw new Error('Expected an ephemeral TCP address'); - } - const client = new OpenAI({ apiKey: 'local-test-key', baseURL: `http://127.0.0.1:${address.port}/v1` }); - const finish = new AbortController(); - const markers: number[] = []; - let sawSessionClose = false; - let sawSessionClosed = false; - const timers: ReturnType[] = []; - server.on('connection', (socket) => { - socket.on('message', (raw) => { - const event: { type: string; audio: string } = JSON.parse(raw.toString()); - if (event.type === 'session.start') { - socket.send(JSON.stringify({ type: 'session.started', session: { id: 'early', status: 'active' } })); - } else if (event.type === 'session.input_audio.append') { - const [marker] = Buffer.from(event.audio, 'base64'); - expect(marker).toBeDefined(); - if (marker !== undefined) { - markers.push(marker); +it.concurrent.each(['close_requested', 'expired', 'content', 'remote_hangup', 'connection_lost'])( + 'finishes during the first input and checks the session.closed reason: %s', + async (reason) => { + const server = new WebSocketServer({ port: 0, host: '127.0.0.1' }); + await once(server, 'listening'); + const address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('Expected an ephemeral TCP address'); + } + const client = new OpenAI({ apiKey: 'local-test-key', baseURL: `http://127.0.0.1:${address.port}/v1` }); + const finish = new AbortController(); + const markers: number[] = []; + let sawSessionClose = false; + let sawSessionClosed = false; + const timers: ReturnType[] = []; + server.on('connection', (socket) => { + socket.on('message', (raw) => { + const event: { type: string; audio: string } = JSON.parse(raw.toString()); + if (event.type === 'session.start') { + socket.send( + JSON.stringify({ type: 'session.started', session: { id: 'early', status: 'active' } }), + ); + } else if (event.type === 'session.input_audio.append') { + const [marker] = Buffer.from(event.audio, 'base64'); + expect(marker).toBeDefined(); + if (marker !== undefined) { + markers.push(marker); + } + finish.abort(); + } else if (event.type === 'session.close') { + sawSessionClose = true; + timers.push( + setTimeout(() => { + sawSessionClosed = true; + socket.send( + JSON.stringify({ + type: 'session.closed', + reason, + session: { id: 'early', status: 'active' }, + }), + ); + socket.close(); + }, 200), + ); } - finish.abort(); - } else if (event.type === 'session.close') { - sawSessionClose = true; - timers.push( - setTimeout(() => { - sawSessionClosed = true; - socket.send( - JSON.stringify({ type: 'session.closed', session: { id: 'early', status: 'active' } }), - ); - socket.close(); - }, 200), - ); - } + }); }); - }); - try { - const result = await runTranscriptDemo(client, [Buffer.alloc(1920 * 40, 1), Buffer.alloc(1920 * 40, 2)], { - interrupt: true, - finishSignal: finish.signal, - }); - expect(markers).toEqual([1]); - expect(sawSessionClose).toBe(true); - expect(sawSessionClosed).toBe(true); - expect(result.interruptionSentDuringOutput).toBe(false); - } finally { - for (const timer of timers) { - clearTimeout(timer); - } - const closed = once(server, 'close'); - for (const socket of server.clients) { - socket.terminate(); + try { + const result = runTranscriptDemo(client, [Buffer.alloc(1920 * 40, 1), Buffer.alloc(1920 * 40, 2)], { + interrupt: true, + finishSignal: finish.signal, + }); + await (reason === 'close_requested' + ? expect(result).resolves.toMatchObject({ interruptionSentDuringOutput: false }) + : expect(result).rejects.toThrow()); + expect(markers).toEqual([1]); + expect(sawSessionClose).toBe(true); + expect(sawSessionClosed).toBe(true); + } finally { + for (const timer of timers) { + clearTimeout(timer); + } + const closed = once(server, 'close'); + for (const socket of server.clients) { + socket.terminate(); + } + server.close(); + await closed; } - server.close(); - await closed; - } -}); + }, +); From f474293c18efb077e2fe7a0d70da7de8ab9bbb26 Mon Sep 17 00:00:00 2001 From: Mark Stuart <323302876+markstuart-oai@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:39:04 +0000 Subject: [PATCH 07/11] fix: stop transcript capture during Live startup --- examples/live/transcript-demo.ts | 10 ++++- tests/lib/live/transcript-demo.test.ts | 59 ++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/examples/live/transcript-demo.ts b/examples/live/transcript-demo.ts index 84150b9dcb..d18b3a7d93 100644 --- a/examples/live/transcript-demo.ts +++ b/examples/live/transcript-demo.ts @@ -52,6 +52,7 @@ export async function prepareTranscriptDemo(client: OpenAI): Promise { * The caller ends capture with finishSignal; transcript activity never ends it. * This controls capture duration, not whether the model finished an answer. * The result is returned only after the server confirms session.closed. + * Finishing before session startup rejects without returning an unconfirmed capture. */ export async function runTranscriptDemo( client: OpenAI, @@ -77,6 +78,9 @@ export async function runTranscriptDemo( if (!firstAudio?.length || !secondAudio?.length) { throw new Error('Generate both TTS clips before opening Live.'); } + if (options.finishSignal.aborted) { + throw new Error('Live transcript capture stopped before session startup'); + } const live = new LiveWS(client, options.websocketOptions); const abort = new AbortController(); @@ -215,9 +219,13 @@ export async function runTranscriptDemo( // The generated transport queues this until the socket opens. live.send({ type: 'session.start', session }); // oxlint-disable-next-line no-unmodified-loop-condition -- The session.started event listener sets ready. - while (!ready) { + while (!ready && !options.finishSignal.aborted) { await sleep(20, undefined, { signal: abort.signal }); } + abort.signal.throwIfAborted(); + if (!ready) { + throw new Error('Live transcript capture stopped before session startup'); + } await sendPCM(firstAudio); if (options.interrupt) { await waitWithAudio( diff --git a/tests/lib/live/transcript-demo.test.ts b/tests/lib/live/transcript-demo.test.ts index 359d387875..4517571b77 100644 --- a/tests/lib/live/transcript-demo.test.ts +++ b/tests/lib/live/transcript-demo.test.ts @@ -379,3 +379,62 @@ it.concurrent.each(['close_requested', 'expired', 'content', 'remote_hangup', 'c } }, ); + +it.concurrent.each(['before-opening', 'during-startup'])( + 'stops capture %s without sending an unstarted session.close', + async (stage) => { + const server = new WebSocketServer({ port: 0, host: '127.0.0.1' }); + await once(server, 'listening'); + const address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('Expected an ephemeral TCP address'); + } + const client = new OpenAI({ apiKey: 'local-test-key', baseURL: `http://127.0.0.1:${address.port}/v1` }); + const finish = new AbortController(); + const events: string[] = []; + let connections = 0; + let startupTimedOut = false; + const timers: ReturnType[] = []; + if (stage === 'before-opening') { + finish.abort('This caller reason must not enter the error'); + } + server.on('connection', (socket) => { + connections += 1; + socket.on('message', (raw) => { + const event: { type: string } = JSON.parse(raw.toString()); + events.push(event.type); + if (event.type === 'session.start') { + finish.abort('This caller reason must not enter the error'); + // Do not confirm startup. An early stop must not wait for this error + // or send a command which requires a started session. + timers.push( + setTimeout(() => { + startupTimedOut = true; + socket.send(JSON.stringify({ type: 'error', message: 'Synthetic startup timeout' })); + }, 1200), + ); + } + }); + }); + try { + await expect( + runTranscriptDemo(client, [Buffer.alloc(1920, 1), Buffer.alloc(1920, 2)], { + finishSignal: finish.signal, + }), + ).rejects.toThrow('Live transcript capture stopped before session startup'); + expect(startupTimedOut).toBe(false); + expect(events).toEqual(stage === 'before-opening' ? [] : ['session.start']); + expect(connections).toBe(stage === 'before-opening' ? 0 : 1); + } finally { + for (const timer of timers) { + clearTimeout(timer); + } + const closed = once(server, 'close'); + for (const socket of server.clients) { + socket.terminate(); + } + server.close(); + await closed; + } + }, +); From beb1b6da6fef40115db6b65f6149e69646e4d6ab Mon Sep 17 00:00:00 2001 From: markstuart-oai Date: Wed, 28 Oct 2026 06:08:12 +0000 Subject: [PATCH 08/11] fix(live): stop on interruption and allow bounded unattended demos --- examples/live/README.md | 9 +- examples/live/transcript-demo.ts | 20 +-- examples/live/transcript-grouper.ts | 40 ++++-- examples/live/transcript-playback.ts | 40 ++++-- tests/lib/live/transcript-demo.test.ts | 74 ++++++++++ tests/lib/live/transcript-playback.test.ts | 157 ++++++++++++--------- 6 files changed, 238 insertions(+), 102 deletions(-) diff --git a/examples/live/README.md b/examples/live/README.md index 483ec300a5..b25dac5229 100644 --- a/examples/live/README.md +++ b/examples/live/README.md @@ -56,9 +56,12 @@ pnpm tsn -T examples/live/transcript-grouper.ts ``` Audio playback requires `ffplay` on your `PATH`. To run without playback, add -`--no-audio`. Press Enter to finish capturing the transcript. The example -requests closure and waits for the server's confirmation. It does not infer that -the model finished an answer from a pause or transcript fragment. +`--no-audio`. In a terminal, press Enter to finish capturing the transcript, or +Ctrl-C to cancel and stop playback. Without a terminal, the example uses a +45-second demo budget after preparing the TTS clips, including session startup. +Both normal finish paths request closure and wait for the server's confirmation. +The budget is an observation window, not a claim that the model finished its +answer. No path infers answer completion from a pause or transcript fragment. ## WebRTC with a backend sideband diff --git a/examples/live/transcript-demo.ts b/examples/live/transcript-demo.ts index d18b3a7d93..8f466f73c5 100644 --- a/examples/live/transcript-demo.ts +++ b/examples/live/transcript-demo.ts @@ -60,6 +60,8 @@ export async function runTranscriptDemo( options: { /** End capture when the caller or user decides. This is separate from operation failures. */ finishSignal: AbortSignal; + /** Cancel capture and pending socket waits without returning a transcript. */ + signal?: AbortSignal; /** Optional transport options, for example a local opt-in compatibility header. */ websocketOptions?: LiveWSClientOptions | undefined; /** Interrupt the first answer instead of waiting for its projected completion. */ @@ -81,9 +83,11 @@ export async function runTranscriptDemo( if (options.finishSignal.aborted) { throw new Error('Live transcript capture stopped before session startup'); } + options.signal?.throwIfAborted(); const live = new LiveWS(client, options.websocketOptions); const abort = new AbortController(); + const signal = options.signal ? AbortSignal.any([abort.signal, options.signal]) : abort.signal; const timeout = setTimeout( () => abort.abort(new Error('Live transcript demo exceeded 120 seconds')), 120_000, @@ -185,7 +189,7 @@ export async function runTranscriptDemo( async function sendPCM(pcm: Buffer): Promise { const startedAt = performance.now(); for (let offset = 0; offset < pcm.length; offset += CHUNK_BYTES) { - abort.signal.throwIfAborted(); + signal.throwIfAborted(); if (options.finishSignal.aborted) { return; } @@ -196,7 +200,7 @@ export async function runTranscriptDemo( }); options.onInputAudio?.(chunk); const sentMs = (Math.min(offset + CHUNK_BYTES, pcm.length) / (SAMPLE_RATE * 2)) * 1000; - await sleep(Math.max(0, startedAt + sentMs - performance.now()), undefined, { signal: abort.signal }); + await sleep(Math.max(0, startedAt + sentMs - performance.now()), undefined, { signal }); } } @@ -205,7 +209,7 @@ export async function runTranscriptDemo( async function waitWithAudio(predicate: () => boolean, description: string): Promise { const deadline = performance.now() + 45_000; while (!predicate() && !options.finishSignal.aborted) { - abort.signal.throwIfAborted(); + signal.throwIfAborted(); if (performance.now() >= deadline) { throw new Error( `Timed out waiting for ${description} (${assistantCharacters} assistant characters, ${closedAssistant} closed assistant segments, ${Math.round(performance.now() - lastTextAt)} ms since the last output transcript)`, @@ -220,9 +224,9 @@ export async function runTranscriptDemo( live.send({ type: 'session.start', session }); // oxlint-disable-next-line no-unmodified-loop-condition -- The session.started event listener sets ready. while (!ready && !options.finishSignal.aborted) { - await sleep(20, undefined, { signal: abort.signal }); + await sleep(20, undefined, { signal }); } - abort.signal.throwIfAborted(); + signal.throwIfAborted(); if (!ready) { throw new Error('Live transcript capture stopped before session startup'); } @@ -248,13 +252,13 @@ export async function runTranscriptDemo( while (!options.finishSignal.aborted) { await sendPCM(silence); } - abort.signal.throwIfAborted(); + signal.throwIfAborted(); finishing = true; live.send({ type: 'session.close' }); // Confirm finalization before closing the transport or returning the transcript. // oxlint-disable-next-line no-unmodified-loop-condition -- The session.closed event listener sets sessionClosed. while (!sessionClosed) { - await sleep(20, undefined, { signal: abort.signal }); + await sleep(20, undefined, { signal }); } } finally { finishing = true; @@ -266,6 +270,6 @@ export async function runTranscriptDemo( live.close(); } } - abort.signal.throwIfAborted(); + signal.throwIfAborted(); return result; } diff --git a/examples/live/transcript-grouper.ts b/examples/live/transcript-grouper.ts index 499cfc762f..7b59a76954 100644 --- a/examples/live/transcript-grouper.ts +++ b/examples/live/transcript-grouper.ts @@ -9,37 +9,57 @@ async function main(): Promise { ? undefined : { user: createPCMPlayback(), assistant: createPCMPlayback() }; const finish = new AbortController(); + const cancel = new AbortController(); let input: Interface | undefined; const stop = () => finish.abort(); + const interrupt = () => { + cancel.abort(); + void playback?.user.close(); + void playback?.assistant.close(); + }; try { const client = new OpenAI({ logLevel: 'off' }); console.log('All voices are AI-generated. Preparing both TTS clips before connecting to Live…'); const audio = await prepareTranscriptDemo(client); - console.log(playback ? 'Playing both voices through playAudio (ffplay).' : 'Audio playback disabled.'); + console.log(playback ? 'Playing both voices through ffplay.' : 'Audio playback disabled.'); console.log('TTS ready. Starting a picnic conversation, then interrupting with a breakfast request.'); - input = createInterface({ input: process.stdin, output: process.stdout }); - input.once('line', stop); - input.once('close', stop); - console.log('Press Enter to finish capturing the transcript and close the session.'); + process.once('SIGINT', interrupt); + if (process.stdin.isTTY) { + input = createInterface({ input: process.stdin, output: process.stdout }); + input.once('line', stop); + input.once('close', stop); + input.once('SIGINT', interrupt); + console.log('Press Enter to finish capturing the transcript and close the session.'); + } else { + console.log('Capturing for up to 45 seconds, then closing the session.'); + } const result = await runTranscriptDemo(client, audio, { - finishSignal: finish.signal, + finishSignal: input ? finish.signal : AbortSignal.timeout(45_000), + signal: cancel.signal, interrupt: true, onInputAudio: (pcm) => playback?.user.write(pcm), onOutputAudio: (pcm) => playback?.assistant.write(pcm), onClosed: ({ segment, reason }) => console.log(`[${segment.speaker}; ${reason}] ${segment.text}`), }); await Promise.all([playback?.user.finish(), playback?.assistant.finish()]); + cancel.signal.throwIfAborted(); console.log( `${result.rawTranscriptEvents} raw transcript fragments → ${result.updateCount} bubble updates → ${result.segments.length} final bubbles.`, ); } catch (error) { - const message = error instanceof Error ? error.message : 'The Live example failed'; - const key = process.env['OPENAI_API_KEY']; - console.error(key ? message.split(key).join('[REDACTED]') : message); - process.exitCode = 1; + if (cancel.signal.aborted) { + process.exitCode = 130; + } else { + const message = error instanceof Error ? error.message : 'The Live example failed'; + const key = process.env['OPENAI_API_KEY']; + console.error(key ? message.split(key).join('[REDACTED]') : message); + process.exitCode = 1; + } } finally { + process.removeListener('SIGINT', interrupt); input?.removeListener('line', stop); input?.removeListener('close', stop); + input?.removeListener('SIGINT', interrupt); input?.close(); await Promise.all([playback?.user.close(), playback?.assistant.close()]); } diff --git a/examples/live/transcript-playback.ts b/examples/live/transcript-playback.ts index 9aefc13972..bc4e0575c2 100644 --- a/examples/live/transcript-playback.ts +++ b/examples/live/transcript-playback.ts @@ -1,7 +1,6 @@ -import { PassThrough } from 'node:stream'; -import { playAudio } from 'openai/helpers/audio'; +import { spawn } from 'node:child_process'; -/** Play the demo's mono, 24 kHz, 16-bit PCM using the SDK's existing ffplay helper. */ +/** Play the demo's mono, 24 kHz, 16-bit PCM and retain the child for cancellation. */ export function createPCMPlayback() { // A streaming WAV header lets playAudio recognize raw Live/TTS PCM. The // unknown lengths are resolved by EOF, so playback can start before completion. @@ -19,18 +18,30 @@ export function createPCMPlayback() { header.write('data', 36); header.writeUInt32LE(0xff_ff_ff_ff, 40); - const stream = new PassThrough(); - stream.write(header); + // We own the child so close() can stop queued playback even after its input has ended. + const player = spawn('ffplay', ['-autoexit', '-nodisp', '-i', 'pipe:0']); + player.stdout.resume(); + player.stderr.resume(); + const stream = player.stdin; let failure: Error | undefined; - // Attach rejection handling immediately, including when ffplay is missing. - const finished = (async () => { - try { - await playAudio(stream); - } catch (error) { - failure = error instanceof Error ? error : new Error('Audio playback failed'); - stream.destroy(); - } - })(); + // Record failures immediately and settle only when the child has exited. + // oxlint-disable-next-line promise/avoid-new -- Await child close even after spawn or stdin errors, so cleanup never leaves a player running. + const finished = new Promise((resolve) => { + player.once('error', (error) => { + failure ??= error; + }); + stream.once('error', (error) => { + failure ??= error; + player.kill(); + }); + player.once('close', (code) => { + if (code !== 0) { + failure ??= new Error(`ffplay process exited with code ${code}`); + } + resolve(); + }); + }); + stream.write(header); return { /** Queue a PCM chunk without blocking the duplex transport. */ @@ -54,6 +65,7 @@ export function createPCMPlayback() { /** Stop playback on teardown, including after a failed Live session. */ async close(): Promise { stream.destroy(); + player.kill(); await finished; }, }; diff --git a/tests/lib/live/transcript-demo.test.ts b/tests/lib/live/transcript-demo.test.ts index 4517571b77..ed52c384bf 100644 --- a/tests/lib/live/transcript-demo.test.ts +++ b/tests/lib/live/transcript-demo.test.ts @@ -438,3 +438,77 @@ it.concurrent.each(['before-opening', 'during-startup'])( } }, ); + +it.concurrent.each(['startup', 'input', 'finalization'])( + 'cancels %s promptly without publishing an unconfirmed transcript', + async (stage) => { + const server = new WebSocketServer({ port: 0, host: '127.0.0.1' }); + await once(server, 'listening'); + const address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('Expected an ephemeral TCP address'); + } + + const finish = new AbortController(); + const cancel = new AbortController(); + const events: string[] = []; + const connected = once(server, 'connection'); + server.once('connection', (socket) => { + socket.on('message', (message) => { + const event = JSON.parse(message.toString()); + events.push(event.type); + if (event.type === 'session.start') { + if (stage === 'startup') { + cancel.abort(); + } else { + socket.send( + JSON.stringify({ type: 'session.started', session: { id: 'fixture', status: 'active' } }), + ); + } + } else if (event.type === 'session.input_audio.append') { + if (stage === 'input') { + cancel.abort(); + } else { + finish.abort(); + } + } else if (event.type === 'session.close') { + // A silent peer must not hold cancellation until the 120 s demo deadline. + cancel.abort(); + } + }); + }); + const client = new OpenAI({ apiKey: 'local-test-key', baseURL: `http://127.0.0.1:${address.port}/v1` }); + try { + const demo = runTranscriptDemo(client, [Buffer.alloc(48_000), Buffer.alloc(48_000)], { + finishSignal: finish.signal, + signal: cancel.signal, + interrupt: true, + }); + const [socket] = await connected; + const closed = once(socket, 'close'); + await expect(demo).rejects.toThrow(); + await closed; + const expected = ['session.start']; + if (stage !== 'startup') { + expected.push('session.input_audio.append'); + } + if (stage === 'finalization') { + expected.push('session.close'); + } + // Extra input frames can already be in flight before the server's abort reaches the client. + expect(new Set(events)).toEqual(new Set(expected)); + expect(events[0]).toBe('session.start'); + if (stage === 'finalization') { + expect(events.at(-1)).toBe('session.close'); + } + } finally { + const closed = once(server, 'close'); + for (const socket of server.clients) { + socket.terminate(); + } + server.close(); + await closed; + } + }, + 2000, +); diff --git a/tests/lib/live/transcript-playback.test.ts b/tests/lib/live/transcript-playback.test.ts index f833d0822a..573dc0e3e0 100644 --- a/tests/lib/live/transcript-playback.test.ts +++ b/tests/lib/live/transcript-playback.test.ts @@ -1,70 +1,93 @@ -import type { PassThrough } from 'node:stream'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { playAudio } from 'openai/helpers/audio'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { expect, it, vi } from 'vitest'; import { createPCMPlayback } from '../../../examples/live/transcript-playback'; -// oxlint-disable-next-line anti-slop/no-module-mocking -- Control player failure and stream lifetime without starting ffplay or accessing audio hardware. -vi.mock('openai/helpers/audio', () => ({ playAudio: vi.fn() })); - -const chunks: Buffer[] = []; -let source: PassThrough | undefined; - -beforeEach(() => { - chunks.length = 0; - source = undefined; - vi.mocked(playAudio).mockReset(); - vi.mocked(playAudio).mockImplementation(async (input) => { - // SAFETY: createPCMPlayback supplies its owned PassThrough to the mocked player; retain that stream to check playback completion and destruction. - source = input as PassThrough; - for await (const chunk of source) { - chunks.push(chunk); +it.each(['before-eof', 'after-eof', 'complete', 'missing', 'nonzero'])( + 'handles %s playback without leaving a child running', + async (stage) => { + const dir = await mkdtemp(path.join(tmpdir(), 'transcript-player-')); + const ready = path.join(dir, 'ready'); + const eof = path.join(dir, 'eof'); + const oldPath = process.env['PATH']; + let player: ReturnType | undefined; + if (stage !== 'missing') { + await writeFile( + path.join(dir, 'ffplay'), + `#!/bin/sh\nexec "${process.execPath}" "${path.join(dir, 'player.mjs')}"\n`, + { mode: 0o700 }, + ); + await writeFile( + path.join(dir, 'player.mjs'), + ` + import fs from 'node:fs'; + fs.writeFileSync(${JSON.stringify(ready)}, String(process.pid)); + const chunks = []; + process.stdin.on('data', (chunk) => chunks.push(chunk)); + process.stdin.on('end', () => { + fs.writeFileSync(${JSON.stringify(eof)}, Buffer.concat(chunks)); + setTimeout(() => process.exit(${stage === 'nonzero' ? 21 : 0}), + ${stage === 'after-eof' ? 2400 : 40}); + }); + `, + ); } - }); -}); - -describe('Live example PCM playback', () => { - it('streams a mono 24 kHz WAV header and unchanged PCM through the existing helper', async () => { - const playback = createPCMPlayback(); - const pcm = Buffer.from([0, 0, 255, 127]); - playback.write(pcm); - await vi.waitFor(() => expect(chunks.length).toBeGreaterThan(0)); - expect(source?.readableEnded).toBe(false); - await playback.finish(); - - const audio = Buffer.concat(chunks); - expect(audio.toString('ascii', 0, 4)).toBe('RIFF'); - expect(audio.readUInt32LE(4)).toBe(0xff_ff_ff_ff); - expect(audio.toString('ascii', 8, 16)).toBe('WAVEfmt '); - expect(audio.readUInt32LE(16)).toBe(16); - expect(audio.readUInt16LE(20)).toBe(1); - expect(audio.readUInt16LE(22)).toBe(1); - expect(audio.readUInt32LE(24)).toBe(24_000); - expect(audio.readUInt32LE(28)).toBe(48_000); - expect(audio.readUInt16LE(32)).toBe(2); - expect(audio.readUInt16LE(34)).toBe(16); - expect(audio.toString('ascii', 36, 40)).toBe('data'); - expect(audio.readUInt32LE(40)).toBe(0xff_ff_ff_ff); - expect(audio.subarray(44)).toEqual(pcm); - expect(playAudio).toHaveBeenCalledTimes(1); - expect(() => playback.write(pcm)).toThrow('after closing'); - await playback.close(); - }); - - it('destroys an unfinished stream on teardown and allows repeated close', async () => { - const playback = createPCMPlayback(); - await playback.close(); - await playback.close(); - expect(source?.destroyed).toBe(true); - expect(() => playback.write(Buffer.alloc(2))).toThrow(); - }); - - it('handles a missing or failed player immediately and reports it on write and finish', async () => { - const error = new Error('ffplay was not found'); - vi.mocked(playAudio).mockRejectedValueOnce(error); - const playback = createPCMPlayback(); - await Promise.resolve(); - expect(() => playback.write(Buffer.alloc(2))).toThrow(error); - await expect(playback.finish()).rejects.toBe(error); - await playback.close(); - }); -}); + // The example must find the controlled executable, never a machine's real audio player. + process.env['PATH'] = dir; + try { + player = createPCMPlayback(); + if (stage === 'missing') { + await expect(player.finish()).rejects.toThrow(); + expect(() => player?.write(Buffer.alloc(2))).toThrow(); + return; + } + player.write(Buffer.alloc(480, 3)); + const pid = Number(await vi.waitFor(() => readFile(ready, 'utf-8'))); + if (stage === 'before-eof') { + await player.close(); + expect(await readFile(eof).catch(() => {})).toBeUndefined(); + } else { + const result = player.finish().then( + () => 'completed', + () => 'failed', + ); + const input = await vi.waitFor(() => readFile(eof)); + expect(input.toString('ascii', 0, 4)).toBe('RIFF'); + expect(input.readUInt32LE(4)).toBe(0xff_ff_ff_ff); + expect(input.toString('ascii', 8, 16)).toBe('WAVEfmt '); + expect(input.readUInt32LE(16)).toBe(16); + expect(input.readUInt16LE(20)).toBe(1); + expect(input.readUInt16LE(22)).toBe(1); + expect(input.readUInt32LE(24)).toBe(24_000); + expect(input.readUInt32LE(28)).toBe(48_000); + expect(input.readUInt16LE(32)).toBe(2); + expect(input.readUInt16LE(34)).toBe(16); + expect(input.toString('ascii', 36, 40)).toBe('data'); + expect(input.readUInt32LE(40)).toBe(0xff_ff_ff_ff); + expect(input.subarray(44)).toEqual(Buffer.alloc(480, 3)); + if (stage === 'after-eof') { + const start = performance.now(); + await player.close(); + expect(performance.now() - start).toBeLessThan(900); + expect(await result).toBe('failed'); + } else { + expect(await result).toBe(stage === 'complete' ? 'completed' : 'failed'); + } + } + await player.close(); + await player.close(); + expect(() => player?.write(Buffer.alloc(2))).toThrow(); + expect(() => process.kill(pid, 0)).toThrow(); + } finally { + await player?.close(); + if (oldPath === undefined) { + delete process.env['PATH']; + } else { + process.env['PATH'] = oldPath; + } + await rm(dir, { recursive: true, force: true }); + } + }, + 5000, +); From e506abde90681a9ed08631a405f0ad55b7b36a04 Mon Sep 17 00:00:00 2001 From: Mark Stuart <323302876+markstuart-oai@users.noreply.github.com> Date: Wed, 28 Oct 2026 06:16:09 +0000 Subject: [PATCH 09/11] test(live): cover CLI cancellation and fix ES2020 assertion --- tests/lib/live/transcript-demo-cli.test.ts | 166 +++++++++++++++++++++ tests/lib/live/transcript-demo.test.ts | 2 +- 2 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 tests/lib/live/transcript-demo-cli.test.ts diff --git a/tests/lib/live/transcript-demo-cli.test.ts b/tests/lib/live/transcript-demo-cli.test.ts new file mode 100644 index 0000000000..198ab3b542 --- /dev/null +++ b/tests/lib/live/transcript-demo-cli.test.ts @@ -0,0 +1,166 @@ +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { createServer } from 'node:http'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { WebSocketServer } from 'ws'; +import { compiledFixture, compiledFixtureConfig } from '../../utils/compiled-fixtures'; + +const quote = (part: string) => `'${part.split("'").join("'\\''")}'`; + +async function runCLI(action: 'deadline' | 'signal' | 'terminal') { + const server = createServer((request, response) => { + if (request.method !== 'POST' || request.url !== '/v1/audio/speech') { + response.writeHead(404).end(); + return; + } + response.writeHead(200, { 'content-type': 'application/octet-stream' }); + response.end(Buffer.alloc(1920)); + }); + const webSocket = new WebSocketServer({ server }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + const address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('Expected local TCP address'); + } + const arguments_ = [ + '-r', + path.join(process.cwd(), 'node_modules/tsconfig-paths/register.js'), + compiledFixture('examples/live/transcript-grouper.ts'), + '--no-audio', + ]; + const terminal = action === 'terminal'; + // script(1) supplies real terminal mode, so Node's readline processes Ctrl-C. + const child = spawn( + terminal ? '/usr/bin/script' : process.execPath, + terminal + ? ['-q', '-e', '-c', `exec ${[process.execPath, ...arguments_].map(quote).join(' ')}`, '/dev/null'] + : arguments_, + { + cwd: process.cwd(), + env: { + OPENAI_API_KEY: 'synthetic-live-cli-key', + OPENAI_BASE_URL: `http://127.0.0.1:${address.port}/v1`, + TS_NODE_PROJECT: compiledFixtureConfig(), + DISABLE_V8_COMPILE_CACHE: '1', + NODE_COMPILE_CACHE: process.env['NODE_COMPILE_CACHE'], + SystemRoot: process.env['SystemRoot'], + SHELL: '/bin/sh', + }, + stdio: [terminal ? 'pipe' : 'ignore', 'pipe', 'pipe'], + timeout: action === 'deadline' ? 65_000 : 12_000, + }, + ); + if (!child.stdout || !child.stderr) { + throw new Error('Expected CLI stdout and stderr pipes'); + } + let stdout = ''; + let stderr = ''; + child.stdout.setEncoding('utf-8').on('data', (data: string) => (stdout += data)); + child.stderr.setEncoding('utf-8').on('data', (data: string) => (stderr += data)); + let receivedStart = false; + let receivedAudio = false; + let receivedClose = false; + let confirmedClose = false; + let startedAt = 0; + let closedAt = 0; + webSocket.on('connection', (socket) => { + socket.on('message', (data) => { + // SAFETY: this local peer reads serialized messages from the actual SDK. + const event = JSON.parse(data.toString()) as { type: string }; + if (event.type === 'session.start') { + receivedStart = true; + startedAt = performance.now(); + socket.send( + JSON.stringify({ type: 'session.started', session: { id: 'fixture', status: 'active' } }), + ); + } else if (event.type === 'session.input_audio.append' && !receivedAudio) { + receivedAudio = true; + if (action === 'deadline') { + socket.send( + JSON.stringify({ + type: 'session.input_transcript.delta', + event_id: 'synthetic-input', + delta: 'A vegetarian picnic.', + start_ms: 0, + end_ms: 40, + }), + ); + } + if (action === 'signal') { + child.kill('SIGINT'); + } + if (terminal) { + child.stdin?.write('\u0003'); + } + } else if (event.type === 'session.close') { + receivedClose = true; + closedAt = performance.now(); + socket.send(JSON.stringify({ type: 'session.closed', reason: 'close_requested' })); + confirmedClose = true; + } + }); + }); + try { + const [exitCode, signal] = await once(child, 'close'); + expect(stdout + stderr).not.toContain('synthetic-live-cli-key'); + return { + exitCode, + signal, + stdout, + receivedStart, + receivedAudio, + receivedClose, + confirmedClose, + captureMs: closedAt - startedAt, + }; + } finally { + child.kill(); + for (const socket of webSocket.clients) { + socket.terminate(); + } + const wsClosed = once(webSocket, 'close'); + webSocket.close(); + await wsClosed; + const httpClosed = once(server, 'close'); + server.close(); + await httpClosed; + } +} + +describe('Live transcript CLI capture and terminal input', () => { + it.concurrent('captures to its default deadline with stdin redirected from /dev/null', async () => { + const result = await runCLI('deadline'); + expect(result).toMatchObject({ + exitCode: 0, + signal: null, + receivedStart: true, + receivedAudio: true, + receivedClose: true, + confirmedClose: true, + }); + expect(result.captureMs).toBeGreaterThan(44_000); + expect(result.stdout).toContain('A vegetarian picnic.'); + expect(result.stdout).toContain('final bubbles.'); + }, 70_000); + + // Windows subprocess.kill('SIGINT') terminates unconditionally, unlike a real console Ctrl-C. + it.skipIf(process.platform === 'win32').concurrent( + 'cancels with exit 130 on process SIGINT during capture', + async () => { + const result = await runCLI('signal'); + expect(result).toMatchObject({ exitCode: 130, signal: null, receivedAudio: true }); + expect(result.stdout).not.toContain('final bubbles.'); + }, + ); + + it.skipIf(process.platform !== 'linux').concurrent( + 'cancels readline Ctrl-C in an actual terminal', + async () => { + const result = await runCLI('terminal'); + expect(result).toMatchObject({ exitCode: 130, signal: null, receivedAudio: true }); + expect(result.stdout).not.toContain('final bubbles.'); + }, + ); +}); diff --git a/tests/lib/live/transcript-demo.test.ts b/tests/lib/live/transcript-demo.test.ts index ed52c384bf..2445180164 100644 --- a/tests/lib/live/transcript-demo.test.ts +++ b/tests/lib/live/transcript-demo.test.ts @@ -499,7 +499,7 @@ it.concurrent.each(['startup', 'input', 'finalization'])( expect(new Set(events)).toEqual(new Set(expected)); expect(events[0]).toBe('session.start'); if (stage === 'finalization') { - expect(events.at(-1)).toBe('session.close'); + expect(events.slice(-1)).toEqual(['session.close']); } } finally { const closed = once(server, 'close'); From 311351c153930d2c0a1a9a4ac8b00b40ab8c0981 Mon Sep 17 00:00:00 2001 From: Mark Stuart <323302876+markstuart-oai@users.noreply.github.com> Date: Wed, 28 Oct 2026 06:21:49 +0000 Subject: [PATCH 10/11] test(live): avoid EOF race during playback cancellation --- tests/lib/live/transcript-playback.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lib/live/transcript-playback.test.ts b/tests/lib/live/transcript-playback.test.ts index 573dc0e3e0..2943f7b47d 100644 --- a/tests/lib/live/transcript-playback.test.ts +++ b/tests/lib/live/transcript-playback.test.ts @@ -45,8 +45,8 @@ it.each(['before-eof', 'after-eof', 'complete', 'missing', 'nonzero'])( player.write(Buffer.alloc(480, 3)); const pid = Number(await vi.waitFor(() => readFile(ready, 'utf-8'))); if (stage === 'before-eof') { - await player.close(); expect(await readFile(eof).catch(() => {})).toBeUndefined(); + await player.close(); } else { const result = player.finish().then( () => 'completed', From 52bfb4f296fa97bd3f7cec1ed3813aed9ca64e1e Mon Sep 17 00:00:00 2001 From: Mark Stuart <323302876+markstuart-oai@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:42:34 +0000 Subject: [PATCH 11/11] fix(examples): preserve automatic Live capture completion --- examples/live/README.md | 8 ++-- examples/live/transcript-demo.ts | 3 +- examples/live/transcript-grouper.ts | 7 ++-- tests/lib/live/transcript-demo-cli.test.ts | 49 +++++++++++++++++++--- tests/lib/live/transcript-demo.test.ts | 13 ++++-- tests/lib/live/transcript-playback.test.ts | 10 +++-- 6 files changed, 70 insertions(+), 20 deletions(-) diff --git a/examples/live/README.md b/examples/live/README.md index b25dac5229..e3a05216f2 100644 --- a/examples/live/README.md +++ b/examples/live/README.md @@ -56,10 +56,10 @@ pnpm tsn -T examples/live/transcript-grouper.ts ``` Audio playback requires `ffplay` on your `PATH`. To run without playback, add -`--no-audio`. In a terminal, press Enter to finish capturing the transcript, or -Ctrl-C to cancel and stop playback. Without a terminal, the example uses a -45-second demo budget after preparing the TTS clips, including session startup. -Both normal finish paths request closure and wait for the server's confirmation. +`--no-audio`. The example uses a 45-second demo budget after preparing the TTS +clips, including session startup. In a terminal, press Enter to finish sooner, +or Ctrl-C to cancel and stop playback. Both normal finish paths request closure +and wait for the server's confirmation. The budget is an observation window, not a claim that the model finished its answer. No path infers answer completion from a pause or transcript fragment. diff --git a/examples/live/transcript-demo.ts b/examples/live/transcript-demo.ts index 8f466f73c5..de1a7219dc 100644 --- a/examples/live/transcript-demo.ts +++ b/examples/live/transcript-demo.ts @@ -136,7 +136,8 @@ export async function runTranscriptDemo( } }); const onEvent = (event: ServerEvent) => { - if (event.type === 'session.started') { + // A late start must not turn an earlier finish into a successful capture. + if (event.type === 'session.started' && !options.finishSignal.aborted) { ready = true; } if (event.type === 'session.input_transcript.delta') { diff --git a/examples/live/transcript-grouper.ts b/examples/live/transcript-grouper.ts index 7b59a76954..a261e059cf 100644 --- a/examples/live/transcript-grouper.ts +++ b/examples/live/transcript-grouper.ts @@ -24,17 +24,16 @@ async function main(): Promise { console.log(playback ? 'Playing both voices through ffplay.' : 'Audio playback disabled.'); console.log('TTS ready. Starting a picnic conversation, then interrupting with a breakfast request.'); process.once('SIGINT', interrupt); + console.log('Capturing for up to 45 seconds, then closing the session.'); if (process.stdin.isTTY) { input = createInterface({ input: process.stdin, output: process.stdout }); input.once('line', stop); input.once('close', stop); input.once('SIGINT', interrupt); - console.log('Press Enter to finish capturing the transcript and close the session.'); - } else { - console.log('Capturing for up to 45 seconds, then closing the session.'); + console.log('Press Enter to finish capturing the transcript and close the session sooner.'); } const result = await runTranscriptDemo(client, audio, { - finishSignal: input ? finish.signal : AbortSignal.timeout(45_000), + finishSignal: AbortSignal.any([finish.signal, AbortSignal.timeout(45_000)]), signal: cancel.signal, interrupt: true, onInputAudio: (pcm) => playback?.user.write(pcm), diff --git a/tests/lib/live/transcript-demo-cli.test.ts b/tests/lib/live/transcript-demo-cli.test.ts index 198ab3b542..0a2dbe44d1 100644 --- a/tests/lib/live/transcript-demo-cli.test.ts +++ b/tests/lib/live/transcript-demo-cli.test.ts @@ -8,7 +8,7 @@ import { compiledFixture, compiledFixtureConfig } from '../../utils/compiled-fix const quote = (part: string) => `'${part.split("'").join("'\\''")}'`; -async function runCLI(action: 'deadline' | 'signal' | 'terminal') { +async function runCLI(action: 'deadline' | 'signal' | 'terminal' | 'terminal-deadline' | 'terminal-enter') { const server = createServer((request, response) => { if (request.method !== 'POST' || request.url !== '/v1/audio/speech') { response.writeHead(404).end(); @@ -30,7 +30,8 @@ async function runCLI(action: 'deadline' | 'signal' | 'terminal') { compiledFixture('examples/live/transcript-grouper.ts'), '--no-audio', ]; - const terminal = action === 'terminal'; + const terminal = action.startsWith('terminal'); + const deadline = action === 'deadline' || action === 'terminal-deadline'; // script(1) supplies real terminal mode, so Node's readline processes Ctrl-C. const child = spawn( terminal ? '/usr/bin/script' : process.execPath, @@ -49,7 +50,7 @@ async function runCLI(action: 'deadline' | 'signal' | 'terminal') { SHELL: '/bin/sh', }, stdio: [terminal ? 'pipe' : 'ignore', 'pipe', 'pipe'], - timeout: action === 'deadline' ? 65_000 : 12_000, + timeout: deadline ? 65_000 : 12_000, }, ); if (!child.stdout || !child.stderr) { @@ -77,7 +78,7 @@ async function runCLI(action: 'deadline' | 'signal' | 'terminal') { ); } else if (event.type === 'session.input_audio.append' && !receivedAudio) { receivedAudio = true; - if (action === 'deadline') { + if (deadline || action === 'terminal-enter') { socket.send( JSON.stringify({ type: 'session.input_transcript.delta', @@ -91,9 +92,12 @@ async function runCLI(action: 'deadline' | 'signal' | 'terminal') { if (action === 'signal') { child.kill('SIGINT'); } - if (terminal) { + if (action === 'terminal') { child.stdin?.write('\u0003'); } + if (action === 'terminal-enter') { + child.stdin?.write('\n'); + } } else if (event.type === 'session.close') { receivedClose = true; closedAt = performance.now(); @@ -145,6 +149,41 @@ describe('Live transcript CLI capture and terminal input', () => { expect(result.stdout).toContain('final bubbles.'); }, 70_000); + it.skipIf(process.platform !== 'linux').concurrent( + 'captures to its default deadline in an unattended terminal', + async () => { + const result = await runCLI('terminal-deadline'); + expect(result).toMatchObject({ + exitCode: 0, + signal: null, + receivedStart: true, + receivedAudio: true, + receivedClose: true, + confirmedClose: true, + }); + expect(result.captureMs).toBeGreaterThan(44_000); + expect(result.stdout).toContain('A vegetarian picnic.'); + expect(result.stdout).toContain('final bubbles.'); + }, + 70_000, + ); + + it.skipIf(process.platform !== 'linux').concurrent( + 'allows Enter to finish before the terminal deadline', + async () => { + const result = await runCLI('terminal-enter'); + expect(result).toMatchObject({ + exitCode: 0, + signal: null, + receivedAudio: true, + receivedClose: true, + confirmedClose: true, + }); + expect(result.captureMs).toBeLessThan(10_000); + expect(result.stdout).toContain('final bubbles.'); + }, + ); + // Windows subprocess.kill('SIGINT') terminates unconditionally, unlike a real console Ctrl-C. it.skipIf(process.platform === 'win32').concurrent( 'cancels with exit 130 on process SIGINT during capture', diff --git a/tests/lib/live/transcript-demo.test.ts b/tests/lib/live/transcript-demo.test.ts index 2445180164..87939144a6 100644 --- a/tests/lib/live/transcript-demo.test.ts +++ b/tests/lib/live/transcript-demo.test.ts @@ -380,7 +380,7 @@ it.concurrent.each(['close_requested', 'expired', 'content', 'remote_hangup', 'c }, ); -it.concurrent.each(['before-opening', 'during-startup'])( +it.concurrent.each(['before-opening', 'during-startup', 'before-delayed-start'])( 'stops capture %s without sending an unstarted session.close', async (stage) => { const server = new WebSocketServer({ port: 0, host: '127.0.0.1' }); @@ -405,14 +405,21 @@ it.concurrent.each(['before-opening', 'during-startup'])( events.push(event.type); if (event.type === 'session.start') { finish.abort('This caller reason must not enter the error'); - // Do not confirm startup. An early stop must not wait for this error - // or send a command which requires a started session. + if (stage === 'before-delayed-start') { + socket.send( + JSON.stringify({ type: 'session.started', session: { id: 'fixture', status: 'active' } }), + ); + } + // An early stop must not wait for this error or send a command which + // requires an accepted startup. timers.push( setTimeout(() => { startupTimedOut = true; socket.send(JSON.stringify({ type: 'error', message: 'Synthetic startup timeout' })); }, 1200), ); + } else if (event.type === 'session.close') { + socket.send(JSON.stringify({ type: 'session.closed', reason: 'close_requested' })); } }); }); diff --git a/tests/lib/live/transcript-playback.test.ts b/tests/lib/live/transcript-playback.test.ts index 2943f7b47d..3326c745a2 100644 --- a/tests/lib/live/transcript-playback.test.ts +++ b/tests/lib/live/transcript-playback.test.ts @@ -4,9 +4,14 @@ import path from 'node:path'; import { expect, it, vi } from 'vitest'; import { createPCMPlayback } from '../../../examples/live/transcript-playback'; -it.each(['before-eof', 'after-eof', 'complete', 'missing', 'nonzero'])( +it.for(['before-eof', 'after-eof', 'complete', 'missing', 'nonzero'])( 'handles %s playback without leaving a child running', - async (stage) => { + { timeout: 5000 }, + async (stage, { skip }) => { + // The controlled player is a POSIX shell script, which Windows cannot execute. + if (process.platform === 'win32' && stage !== 'missing') { + skip(); + } const dir = await mkdtemp(path.join(tmpdir(), 'transcript-player-')); const ready = path.join(dir, 'ready'); const eof = path.join(dir, 'eof'); @@ -89,5 +94,4 @@ it.each(['before-eof', 'after-eof', 'complete', 'missing', 'nonzero'])( await rm(dir, { recursive: true, force: true }); } }, - 5000, );