Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions apps/petrinaut-website/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -261,14 +261,14 @@ gets through, open DevTools, enable the **Verbose** console level and filter by
it.
- `input.ignored` with `reason: "short-during-output"` marks a short transcript
that started during output and was not sent to Brunch.
- `filter.shadow` with `stage: "echo"` marks a transcript the echo check would
- `filter.shadow` with `reason: "echo"` marks a transcript the echo check would
skip. The check runs in shadow, so the transcript is still handled as before
and can also appear as `input.ignored`. It compares speech that overlapped
output, in memory only, with Live's output transcript from three seconds
before the speech started until it stopped. Six or more words count as a
repeat when they mostly match Live's words in order; fewer count only when
they appear together, in order, in Live's words.
- `filter.shadow` with `stage: "doubtful-short-during-output"` tries a narrower
- `filter.shadow` with `reason: "doubtful-short-during-output"` tries a narrower
three-word rule: short speech during output counts only when its least
likely token has a log probability below -1.9 (`minLogprob`) or it repeats
Live's words as the echo check defines them. The three-word rule still
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1384,13 +1384,9 @@ test("an input repeating Live's overlapping words is traced by the echo stage in
expect(
records.filter((record) => record.event === "filter.shadow"),
).toMatchObject([
{ inputId: "echo", stage: "echo", reason: "echo" },
{ inputId: "short-echo", stage: "echo", reason: "echo" },
{
inputId: "short-echo",
stage: "doubtful-short-during-output",
reason: "doubtful-short-during-output",
},
{ inputId: "echo", reason: "echo" },
{ inputId: "short-echo", reason: "echo" },
{ inputId: "short-echo", reason: "doubtful-short-during-output" },
]);
expect(
records.filter((record) => record.event === "input.ignored"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,19 @@ const input = { id: "one", text: "PRIVATE okay", startedDuringOutput: true };

test("sends when no stage skips", () => {
expect(
routeUtterance(input, [{ name: "probe", mode: "on", skip: () => null }]),
routeUtterance(input, [
{ reason: "empty", mode: "on", skips: () => false },
]),
).toBeNull();
});

test("the first stage that is on and skips decides; later stages never run", () => {
const later = vi.fn(() => "empty" as const);
const later = vi.fn(() => true);
expect(
routeUtterance(input, [
{ name: "quiet", mode: "on", skip: () => null },
{ name: "decides", mode: "on", skip: () => "short-during-output" },
{ name: "later", mode: "on", skip: later },
{ reason: "echo", mode: "on", skips: () => false },
{ reason: "short-during-output", mode: "on", skips: () => true },
{ reason: "empty", mode: "on", skips: later },
]),
).toBe("short-during-output");
expect(later).not.toHaveBeenCalled();
Expand All @@ -45,20 +47,19 @@ test("a shadow stage traces what it would skip without deciding", () => {
const debug = vi.spyOn(console, "debug").mockImplementation(() => {});
expect(
routeUtterance(input, [
{ name: "trial", mode: "shadow", skip: () => "empty" },
{ name: "decides", mode: "on", skip: () => "short-during-output" },
{ reason: "empty", mode: "shadow", skips: () => true },
{ reason: "short-during-output", mode: "on", skips: () => true },
]),
).toBe("short-during-output");
const records = traceRecords(debug.mock.calls);
expect(records).toMatchObject([
{ event: "filter.shadow", inputId: "one", stage: "trial", reason: "empty" },
{ event: "filter.shadow", inputId: "one", reason: "empty" },
]);
expect(Object.keys(records[0] ?? {}).sort()).toEqual([
"at",
"event",
"inputId",
"reason",
"stage",
]);
expect(JSON.stringify(debug.mock.calls)).not.toContain("PRIVATE");
});
Expand All @@ -68,32 +69,32 @@ test("a shadow stage that would not skip leaves no trace", () => {
const debug = vi.spyOn(console, "debug").mockImplementation(() => {});
expect(
routeUtterance(input, [
{ name: "trial", mode: "shadow", skip: () => null },
{ reason: "empty", mode: "shadow", skips: () => false },
]),
).toBeNull();
expect(debug).not.toHaveBeenCalled();
});

test("a stage that is off never runs", () => {
const off = vi.fn(() => "empty" as const);
const off = vi.fn(() => true);
expect(
routeUtterance(input, [{ name: "off", mode: "off", skip: off }]),
routeUtterance(input, [{ reason: "empty", mode: "off", skips: off }]),
).toBeNull();
expect(off).not.toHaveBeenCalled();
});

test("stages see Live's words when the speech overlapped its output", () => {
const skip = vi.fn(() => null);
const skips = vi.fn(() => false);
routeUtterance(
{
id: "one",
text: "Okay",
startedDuringOutput: true,
liveOutputText: "Okay?",
},
[{ name: "probe", mode: "on", skip }],
[{ reason: "empty", mode: "on", skips }],
);
expect(skip).toHaveBeenCalledWith({
expect(skips).toHaveBeenCalledWith({
inputId: "one",
text: "Okay",
words: 1,
Expand All @@ -109,11 +110,11 @@ test.each([
])(
"stages see %j with its word count, counting contractions as one word",
(text, startedDuringOutput, words) => {
const skip = vi.fn(() => null);
const skips = vi.fn(() => false);
routeUtterance({ id: "one", text, startedDuringOutput }, [
{ name: "probe", mode: "on", skip },
{ reason: "empty", mode: "on", skips },
]);
expect(skip).toHaveBeenCalledWith({
expect(skips).toHaveBeenCalledWith({
inputId: "one",
text,
words,
Expand All @@ -123,12 +124,12 @@ test.each([
);

test("stages see the transcript's lowest token log probability", () => {
const skip = vi.fn(() => null);
const skips = vi.fn(() => false);
routeUtterance(
{ id: "one", text: "Okay", startedDuringOutput: false, minLogprob: -2.5 },
[{ name: "probe", mode: "on", skip }],
[{ reason: "empty", mode: "on", skips }],
);
expect(skip).toHaveBeenCalledWith({
expect(skips).toHaveBeenCalledWith({
inputId: "one",
text: "Okay",
words: 1,
Expand All @@ -141,7 +142,7 @@ const doubtfulShadows = (calls: readonly (readonly unknown[])[]) =>
traceRecords(calls).filter(
(record) =>
record.event === "filter.shadow" &&
record.stage === "doubtful-short-during-output",
record.reason === "doubtful-short-during-output",
);

test.each([
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@ export interface Utterance {
}

export interface UtteranceStage {
readonly name: string;
readonly reason: SkipReason;
/** A shadow stage traces what it would skip and never decides. */
readonly mode: "on" | "shadow" | "off";
readonly skip: (utterance: Utterance) => SkipReason | null;
readonly skips: (utterance: Utterance) => boolean;
}

export const doubtfulBelowLogprob = -1.9;
Expand All @@ -40,35 +40,32 @@ const isShortDuringOutput = ({ startedDuringOutput, words }: Utterance) =>
export const liveUtteranceStages: readonly UtteranceStage[] = [
// Leaked Live audio can finalize as a longer repeat of Live's own words.
{
name: "echo",
reason: "echo",
mode: "shadow",
skip: (utterance) => (repeatsOverlappingOutput(utterance) ? "echo" : null),
skips: repeatsOverlappingOutput,
},
// Only short speech during output that also looks invented or repeats Live;
// a shadow stage must run before the on stage that skips the same input.
{
name: "doubtful-short-during-output",
reason: "doubtful-short-during-output",
mode: "shadow",
skip: (utterance) =>
skips: (utterance) =>
isShortDuringOutput(utterance) &&
((utterance.minLogprob !== undefined &&
utterance.minLogprob < doubtfulBelowLogprob) ||
repeatsOverlappingOutput(utterance))
? "doubtful-short-during-output"
: null,
repeatsOverlappingOutput(utterance)),
},
// Leaked Live audio finalizes as phantoms of a few words.
{
name: "short-during-output",
reason: "short-during-output",
mode: "on",
skip: (utterance) =>
isShortDuringOutput(utterance) ? "short-during-output" : null,
skips: isShortDuringOutput,
},
// Transcription can finalize noise as punctuation alone, such as ".".
{
name: "empty",
reason: "empty",
mode: "on",
skip: ({ words }) => (words === 0 ? "empty" : null),
skips: ({ words }) => words === 0,
},
];

Expand All @@ -91,14 +88,11 @@ export const routeUtterance = (
minLogprob: input.minLogprob,
};
for (const stage of stages) {
if (stage.mode === "off") continue;
const reason = stage.skip(utterance);
if (reason === null) continue;
if (stage.mode === "on") return reason;
if (stage.mode === "off" || !stage.skips(utterance)) continue;
if (stage.mode === "on") return stage.reason;
logLiveDiagnostic("filter.shadow", {
inputId: utterance.inputId,
stage: stage.name,
reason,
reason: stage.reason,
});
}
return null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1485,7 +1485,7 @@ test("telemetry shows activity but silence and late samples never settle or revi
await vi.advanceTimersByTimeAsync(1);
expect(fixture.onState.mock.lastCall?.[0].activity?.outputActive).toBe(false);
expect(fixture.onFinalizedInput).not.toHaveBeenCalled();
getStats.mockRejectedValueOnce(new Error("Optional telemetry failed"));
getStats.mockRejectedValueOnce(new Error("Stats unavailable"));
await vi.advanceTimersByTimeAsync(100);
expect(fixture.input.stop).not.toHaveBeenCalled();
getStats.mockResolvedValueOnce(
Expand Down Expand Up @@ -1756,66 +1756,6 @@ test("summarises echo evidence for each stretch of audible Live output without p
await stopped;
});

test("does not count speech as started during output while the speaker is muted or at zero volume", async () => {
vi.useFakeTimers();
const fixture = setup();
Object.assign(fixture.peers[0]!, {
getStats: vi.fn(
async (): Promise<Map<string, unknown>> =>
new Map([
["output", { type: "inbound-rtp", kind: "audio", audioLevel: 0.2 }],
]),
),
});
await connect(fixture);
fixture.peers[0]!.dispatchEvent(
Object.assign(new Event("track"), {
track: fixture.outputs[0],
streams: [fixture.stream],
}),
);
const speakers = [
{ itemId: "muted", muted: true, volume: 1 },
{ itemId: "silent", muted: false, volume: 0 },
{ itemId: "audible", muted: false, volume: 1 },
];
for (const { itemId, muted, volume } of speakers) {
fixture.conversation.setSpeakerMuted(muted);
fixture.conversation.setSpeakerVolume(volume);
await vi.advanceTimersByTimeAsync(100);
fixture.emit(1, {
type: "input_audio_buffer.speech_started",
item_id: itemId,
});
}
let previousItemId: string | null = null;
for (const { itemId } of speakers) {
fixture.emit(1, {
type: "input_audio_buffer.committed",
item_id: itemId,
previous_item_id: previousItemId,
});
previousItemId = itemId;
}
for (const { itemId } of speakers) {
fixture.emit(1, {
type: "conversation.item.input_audio_transcription.completed",
item_id: itemId,
content_index: 0,
transcript: "Yes.",
});
}

expect(
fixture.onFinalizedInput.mock.calls.map(
([input]) => input.startedDuringOutput,
),
).toEqual([false, false, true]);
const stopped = fixture.conversation.stop();
fixture.emit(0, { type: "session.closed" });
await stopped;
});

test("records transcription confidence on input.finalized from numbers only", async () => {
vi.stubEnv("DEV", true);
const debug = vi.spyOn(console, "debug").mockImplementation(() => {});
Expand Down Expand Up @@ -2067,7 +2007,7 @@ test("passes Live's recent words only with speech that overlapped its audible ou
await stopped;
});

test("does not count speech as overlapping output while the speaker is muted or at zero volume", async () => {
test("does not count speech as started during or overlapping output while the speaker is muted or at zero volume", async () => {
vi.useFakeTimers();
const fixture = setup();
Object.assign(fixture.peers[0]!, {
Expand Down Expand Up @@ -2123,6 +2063,11 @@ test("does not count speech as overlapping output while the speaker is muted or
});
}

expect(
fixture.onFinalizedInput.mock.calls.map(
([input]) => input.startedDuringOutput,
),
).toEqual([false, false, true]);
expect(
fixture.onFinalizedInput.mock.calls.map(([input]) => input.liveOutputText),
).toEqual([undefined, undefined, "Yes."]);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,9 @@ export const createLiveConversation = (
if (report.type === "inbound-rtp") outputLevel = report.audioLevel;
});
} catch {
// Optional telemetry must not affect the session lifetime.
// A failed sample reads as inaudible output. The short-during-output
// filter and echo checks depend on this sampling, but its failure must
// not end the session.
}
if (abort.signal.aborted) return;
activityTimer = setTimeout(() => void sampleActivity(), 100);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ const turnSchema = z.object({
submissionId: z.string().optional(),
canonicalId: z.string().optional(),
responseIds: z.array(z.string()),
anchorOnly: z.boolean().optional(),
fields: z.record(z.string(), z.string()).optional(),
preparationFailed: z.boolean().optional(),
reply: lineSchema.optional(),
Expand Down Expand Up @@ -52,7 +51,7 @@ export class VoiceMediationHistory {
.max(100)
.parse(JSON.parse(storage?.getItem(this.#key) ?? "[]"));
for (const turn of saved) {
if (!turn.submissionId && !turn.anchorOnly) continue;
if (!turn.submissionId) continue;
if (turn.reply) turn.reply.state = "done";
if (turn.wrapUp) turn.wrapUp.state = "done";
this.#turns.set(turn.id, turn);
Expand All @@ -77,7 +76,7 @@ export class VoiceMediationHistory {
this.#key,
JSON.stringify(
[...this.#turns.values()]
.filter((turn) => turn.submissionId || turn.anchorOnly)
.filter((turn) => turn.submissionId)
.slice(-100),
),
);
Expand Down
Loading