fix(voice): scope forwardAudio playback-started listener to its own segment (#1786)

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: chenghao-mou

.changeset/scope-forward-audio-playback-started.md

@@ -0,0 +1,15 @@
+---
+'@livekit/agents': patch
+---
+
+fix(voice): scope forwardAudio's playback-started listener to its own segment
+
+When a speech is interrupted, the scheduling loop immediately authorizes the next
+speech, so the new segment's `forwardAudio` registers its `playback_started`
+listener on the shared audio output while the interrupted segment is still
+emitting events during teardown. The stray event resolved the new segment's
+`firstFrameFut` before its first frame was captured, which skipped resampler
+creation and pushed an unresampled frame straight to the `AudioSource`
+(`RtcError: sample_rate and num_channels don't match`) and corrupted playback
+bookkeeping. The listener now only resolves `firstFrameFut` after the segment has
+captured its own first frame.

agents/src/voice/generation.ts

@@ -836,8 +836,14 @@ async function forwardAudio(
   const reader = ttsStream.getReader();
   let resampler: AudioResampler | null = null;
 
+  // The audio output is shared across overlapping segments, so ignore a
+  // PLAYBACK_STARTED from another segment until we capture our own first frame.
+  // Resolving `firstFrameFut` early skips resampler creation and pushes an
+  // unresampled frame (`RtcError: sample_rate and num_channels don't match`).
+  let hasCapturedOwnFrame = false;
+
   const onPlaybackStarted = (ev: { createdAt: number }) => {
-    if (!out.firstFrameFut.done) {
+    if (hasCapturedOwnFrame && !out.firstFrameFut.done) {
       out.firstFrameFut.resolve(ev.createdAt);
     }
   };
@@ -868,6 +874,10 @@ async function forwardAudio(
         resampler = new AudioResampler(frame.sampleRate, audioOutput.sampleRate, 1);
       }
 
+      // Mark before capturing so the PLAYBACK_STARTED emitted synchronously inside
+      // the first captureFrame is attributed to this segment.
+      hasCapturedOwnFrame = true;
+
       if (resampler) {
         for (const f of resampler.push(frame)) {
           await audioOutput.captureFrame(f);

agents/src/voice/generation_tts_timeout.test.ts

@@ -105,6 +105,60 @@ describe('TTS stream idle timeout', () => {
     expect(audioOut.firstFrameFut.done).toBe(true);
   });
 
+  it('ignores PLAYBACK_STARTED from another segment before its own first frame', async () => {
+    // Stalled stream so the forwarder is still waiting on its first read when a
+    // stray event (from an interrupted overlapping segment) arrives; the idle
+    // timeout then ends the loop without this segment ever capturing a frame.
+    const stalledStream = new ReadableStream<AudioFrame>({ start() {} });
+
+    const audioOutput = new MockAudioOutput();
+    const controller = new AbortController();
+    const [task, audioOut] = performAudioForwarding(stalledStream, audioOutput, controller, 500);
+
+    // Reject path is expected (no first frame ever captured).
+    audioOut.firstFrameFut.await.catch(() => {});
+
+    vi.useFakeTimers();
+
+    audioOutput.onPlaybackStarted(Date.now());
+    expect(audioOut.firstFrameFut.done).toBe(false);
+
+    const taskPromise = task.result;
+    await vi.advanceTimersByTimeAsync(600);
+    await taskPromise;
+
+    vi.useRealTimers();
+
+    expect(audioOutput.capturedFrames.length).toBe(0);
+    expect(audioOut.firstFrameFut.rejected).toBe(true);
+  });
+
+  it('resamples a rate-mismatched frame even after a stray PLAYBACK_STARTED', async () => {
+    // Output is 24kHz; frames are 16kHz and must be resampled regardless of any
+    // stray PLAYBACK_STARTED resolving firstFrameFut early.
+    const stream = new ReadableStream<AudioFrame>({
+      start(controller) {
+        controller.enqueue(createSilentFrame(16000));
+        controller.enqueue(createSilentFrame(16000));
+        controller.close();
+      },
+    });
+
+    const audioOutput = new MockAudioOutput();
+    const controller = new AbortController();
+    const [task, audioOut] = performAudioForwarding(stream, audioOutput, controller);
+
+    audioOutput.onPlaybackStarted(Date.now());
+
+    await task.result;
+
+    expect(audioOut.firstFrameFut.done).toBe(true);
+    expect(audioOutput.capturedFrames.length).toBeGreaterThan(0);
+    for (const f of audioOutput.capturedFrames) {
+      expect(f.sampleRate).toBe(24000);
+    }
+  });
+
   it('performTTSInference completes when TTS node returns stalled stream', async () => {
     const stalledTtsStream = new ReadableStream<AudioFrame>({
       start(controller) {