Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { act, cleanup, renderHook, waitFor } from '@testing-library/react'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'

import type { BargeMonitorCallbacks } from '@/lib/voice-barge-in'
import { setVoicePlaybackState } from '@/store/voice-playback'

import type { MicRecording } from './use-mic-recorder'
import { useVoiceConversation } from './use-voice-conversation'
Expand All @@ -24,13 +25,16 @@ vi.mock('@/lib/voice-barge-in', () => ({
}
}))

const markVoicePlaybackInterrupted = vi.fn()
const stopVoicePlayback = vi.fn()
const { markVoicePlaybackInterrupted, startSpeechStream, stopVoicePlayback } = vi.hoisted(() => ({
markVoicePlaybackInterrupted: vi.fn(),
startSpeechStream: vi.fn(),
stopVoicePlayback: vi.fn()
}))

vi.mock('@/lib/voice-playback', () => ({
markVoicePlaybackInterrupted: () => markVoicePlaybackInterrupted(),
playSpeechText: vi.fn(async () => true),
startSpeechStream: vi.fn(async () => null),
startSpeechStream,
stopVoicePlayback: () => stopVoicePlayback()
}))

Expand Down Expand Up @@ -140,6 +144,14 @@ describe('useVoiceConversation full-duplex barge-in', () => {
vi.clearAllMocks()
micHandle.start.mockResolvedValue(undefined)
micHandle.stop.mockResolvedValue(null)
startSpeechStream.mockResolvedValue(null)
setVoicePlaybackState({
audioElement: null,
messageId: null,
sequence: 0,
source: null,
status: 'idle'
})
})

afterEach(cleanup)
Expand Down Expand Up @@ -263,4 +275,70 @@ describe('useVoiceConversation full-duplex barge-in', () => {

expect(monitorCalls.length).toBe(armed)
})

it('re-arms the microphone after a normal streaming reply', async () => {
let reply: { id: string; pending: boolean; text: string } | null = null
const onBusyChange: { current: (busy: boolean) => void } = { current: () => undefined }
const onSubmit = vi.fn(async () => onBusyChange.current(true))

const session = {
append: vi.fn(),
done: Promise.resolve<'done'>('done'),
finish: vi.fn()
}

const hook = renderHook(
({ busy }: HookProps) =>
useVoiceConversation({
busy,
consumePendingResponse: vi.fn(),
enabled: true,
onSubmit,
onTranscribeAudio: vi.fn(async () => 'first request'),
pendingResponse: () => reply
}),
{ initialProps: { busy: false } }
)

onBusyChange.current = busy => hook.rerender({ busy })

await act(async () => {
await hook.result.current.start()
})
await waitFor(() => expect(hook.result.current.status).toBe('listening'))

micHandle.stop.mockResolvedValueOnce({
audio: new Blob(['q'], { type: 'audio/webm' }),
durationMs: 900,
heardSpeech: true
})
await act(async () => {
hook.result.current.stopTurn()
})
await waitFor(() => expect(hook.result.current.status).toBe('thinking'))

// `startSpeechStream()` begins by clearing stale playback, which advances
// the shared sequence. That internal change must not be mistaken for the
// user pressing Stop at the end of the reply.
startSpeechStream.mockImplementationOnce(async () => {
setVoicePlaybackState({
audioElement: null,
messageId: null,
sequence: 1,
source: 'voice-conversation',
status: 'preparing'
})

return session
})
reply = { id: 'reply-1', pending: false, text: 'First answer.' }
hook.rerender({ busy: false })

await waitFor(() => expect(startSpeechStream).toHaveBeenCalledTimes(1))
await act(async () => {
await new Promise(resolve => window.setTimeout(resolve, 20))
})
expect(startSpeechStream).toHaveBeenCalledTimes(1)
await waitFor(() => expect(micHandle.start).toHaveBeenCalledTimes(2))
})
})
13 changes: 11 additions & 2 deletions apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -458,9 +458,14 @@ export function useVoiceConversation({
// this is a safety net for read-aloud-style entries into the loop.
ensureBargeMonitor()

// `playSpeechText()` synchronously clears any previous clip before it
// returns its promise, advancing the playback sequence itself. Capture
// the baseline after that internal reset so it is not mistaken for the
// user pressing Stop; only a later sequence advance suppresses re-arm.
const playback = playSpeechText(response.text, { source: 'voice-conversation' })
speechStartSequenceRef.current = $voicePlayback.get().sequence

void playSpeechText(response.text, { source: 'voice-conversation' })
void playback
.catch(error => notifyError(error, voiceCopy.playbackFailed))
.finally(() => {
if (responseIdRef.current === responseId) {
Expand All @@ -484,7 +489,6 @@ export function useVoiceConversation({
(responseId: string) => {
responseIdRef.current = responseId
spokenSourceLengthRef.current = 0
speechStartSequenceRef.current = $voicePlayback.get().sequence
setStatus('speaking')

// VAD barge-in: the user talking over the reply cuts playback, drops
Expand All @@ -505,6 +509,11 @@ export function useVoiceConversation({
return

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

startSpeechStream() awaits URL discovery before it performs its one expected stopVoicePlayback() increment. A user Stop during that await is absorbed by this post-await baseline, so later settling cannot distinguish it from normal setup and re-arms the mic. Preserve the entry sequence and treat more than the expected one increment as an external Stop.

}

// `startSpeechStream()` clears stale playback internally. Its sequence
// advance establishes the new clip, not a user-requested stop, so the
// baseline must be taken only after the stream has opened.
speechStartSequenceRef.current = $voicePlayback.get().sequence

if (!session) {
// No streaming backend/provider: speak the whole reply once it lands.
speechSessionRef.current = null
Expand Down