From 4d5517b8b5243dcf368c52a6e470d6c201603146 Mon Sep 17 00:00:00 2001 From: Charlie Anderson <36671527+canderson26@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:22:13 -0400 Subject: [PATCH] fix(desktop): resume listening after voice playback --- .../hooks/use-voice-conversation.test.tsx | 330 ++++++++++++++++++ .../composer/hooks/use-voice-conversation.ts | 31 +- 2 files changed, 356 insertions(+), 5 deletions(-) create mode 100644 apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.test.tsx diff --git a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.test.tsx b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.test.tsx new file mode 100644 index 000000000000..71d3b5dd0c80 --- /dev/null +++ b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.test.tsx @@ -0,0 +1,330 @@ +import { act, cleanup, renderHook, waitFor } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { setVoicePlaybackState } from '@/store/voice-playback' + +const micMocks = vi.hoisted(() => { + const start = vi.fn() + const stop = vi.fn() + const cancel = vi.fn() + + return { cancel, handle: { cancel, start, stop }, start, stop } +}) + +const playbackMocks = vi.hoisted(() => ({ + markVoicePlaybackInterrupted: vi.fn(), + playSpeechText: vi.fn(), + startSpeechStream: vi.fn(), + stopVoicePlayback: vi.fn() +})) + +vi.mock('@/i18n', () => ({ + useI18n: () => ({ + t: { + notifications: { + voice: { + configureSpeechToText: 'Configure speech to text.', + couldNotStartSession: 'Could not start voice session.', + microphoneFailed: 'Microphone failed.', + playbackFailed: 'Playback failed.', + transcriptionFailed: 'Transcription failed.', + unavailable: 'Voice unavailable.' + } + } + } + }) +})) + +vi.mock('@/lib/voice-barge-in', () => ({ + monitorSpeechDuringPlayback: vi.fn(() => vi.fn()) +})) + +vi.mock('@/lib/voice-playback', () => playbackMocks) + +vi.mock('@/store/notifications', () => ({ + notify: vi.fn(), + notifyError: vi.fn() +})) + +vi.mock('./use-mic-recorder', () => ({ + useMicRecorder: () => ({ handle: micMocks.handle, level: 0, recording: false }) +})) + +import { useVoiceConversation } from './use-voice-conversation' + +interface PendingResponse { + id: string + pending: boolean + text: string +} + +function deferred() { + let resolve!: (value: T) => void + + const promise = new Promise(res => { + resolve = res + }) + + return { promise, resolve } +} + +interface ConversationProps { + busy: boolean + enabled: boolean + response: PendingResponse | null +} + +function renderConversation() { + const consumePendingResponse = vi.fn() + const submitted = deferred() + const onSubmit = vi.fn(() => submitted.promise) + const onTranscribeAudio = vi.fn(async () => 'hello') + const initialProps: ConversationProps = { busy: false, enabled: false, response: null } + + const hook = renderHook( + ({ busy, enabled, response }: ConversationProps) => + useVoiceConversation({ + busy, + consumePendingResponse, + enabled, + onSubmit, + onTranscribeAudio, + pendingResponse: () => response + }), + { initialProps } + ) + + return { consumePendingResponse, hook, onSubmit, onTranscribeAudio, resolveSubmit: submitted.resolve } +} + +async function submitFirstVoiceTurn( + hook: ReturnType['hook'], + onSubmit: ReturnType['onSubmit'], + resolveSubmit: ReturnType['resolveSubmit'], + response: PendingResponse +) { + hook.rerender({ busy: false, enabled: true, response: null }) + await waitFor(() => expect(hook.result.current.status).toBe('listening')) + expect(micMocks.start).toHaveBeenCalledTimes(1) + + const startOptions = micMocks.start.mock.calls[0]?.[0] as { onSilence?: () => void } + + act(() => startOptions.onSilence?.()) + await waitFor(() => expect(onSubmit).toHaveBeenCalledTimes(1)) + hook.rerender({ busy: true, enabled: true, response: null }) + await act(async () => resolveSubmit()) + await waitFor(() => expect(hook.result.current.status).toBe('thinking')) + hook.rerender({ busy: true, enabled: true, response }) + await waitFor(() => expect(playbackMocks.startSpeechStream).toHaveBeenCalledTimes(1)) +} + +describe('useVoiceConversation relisten loop', () => { + let sequence: number + + beforeEach(() => { + sequence = 7 + setVoicePlaybackState({ + audioElement: null, + messageId: null, + sequence, + source: null, + status: 'idle' + }) + + vi.clearAllMocks() + micMocks.start.mockResolvedValue(undefined) + micMocks.stop.mockResolvedValue({ + audio: new Blob(['voice'], { type: 'audio/webm' }), + durationMs: 500, + heardSpeech: true + }) + playbackMocks.stopVoicePlayback.mockImplementation(() => { + sequence += 1 + setVoicePlaybackState({ + audioElement: null, + messageId: null, + sequence, + source: null, + status: 'idle' + }) + }) + }) + + afterEach(async () => { + cleanup() + vi.useRealTimers() + }) + + it('listens again after streaming speech finishes normally', async () => { + const speech = deferred<'done' | 'fallback'>() + + const session = { + append: vi.fn(), + done: speech.promise, + finish: vi.fn() + } + + playbackMocks.startSpeechStream.mockImplementation(async () => { + playbackMocks.stopVoicePlayback() + + return session + }) + + const response: PendingResponse = { id: 'assistant-1', pending: true, text: 'Hello there.' } + const { hook, onSubmit, resolveSubmit } = renderConversation() + + await submitFirstVoiceTurn(hook, onSubmit, resolveSubmit, response) + hook.rerender({ busy: false, enabled: true, response }) + + await act(async () => speech.resolve('done')) + await waitFor(() => expect(micMocks.start).toHaveBeenCalledTimes(2), { timeout: 500 }) + + await act(async () => hook.result.current.end()) + }) + + it('listens again after fallback speech finishes normally', async () => { + const speech = deferred() + + playbackMocks.startSpeechStream.mockResolvedValue(null) + playbackMocks.playSpeechText.mockImplementation(() => { + playbackMocks.stopVoicePlayback() + + return speech.promise + }) + + const response: PendingResponse = { id: 'assistant-1', pending: false, text: 'Hello there.' } + const { hook, onSubmit, resolveSubmit } = renderConversation() + + await submitFirstVoiceTurn(hook, onSubmit, resolveSubmit, response) + hook.rerender({ busy: false, enabled: true, response }) + await waitFor(() => expect(playbackMocks.playSpeechText).toHaveBeenCalledTimes(1)) + + await act(async () => speech.resolve(true)) + await waitFor(() => expect(micMocks.start).toHaveBeenCalledTimes(2), { timeout: 500 }) + + await act(async () => hook.result.current.end()) + }) + + it('stays idle when the user explicitly stops streaming speech', async () => { + const speech = deferred<'done' | 'fallback'>() + + const session = { + append: vi.fn(), + done: speech.promise, + finish: vi.fn() + } + + playbackMocks.startSpeechStream.mockImplementation(async () => { + playbackMocks.stopVoicePlayback() + + return session + }) + + const response: PendingResponse = { id: 'assistant-1', pending: true, text: 'Hello there.' } + const { hook, onSubmit, resolveSubmit } = renderConversation() + + await submitFirstVoiceTurn(hook, onSubmit, resolveSubmit, response) + hook.rerender({ busy: false, enabled: true, response }) + + playbackMocks.stopVoicePlayback() + await act(async () => speech.resolve('done')) + await waitFor(() => expect(hook.result.current.status).toBe('idle')) + expect(micMocks.start).toHaveBeenCalledTimes(1) + + await act(async () => hook.result.current.end()) + }) + + it('does not start fallback speech after Stop during the streaming handoff', async () => { + const streamed = deferred<'done' | 'fallback'>() + + const session = { + append: vi.fn(), + done: streamed.promise, + finish: vi.fn() + } + + playbackMocks.startSpeechStream.mockImplementation(async () => { + playbackMocks.stopVoicePlayback() + + return session + }) + playbackMocks.playSpeechText.mockImplementation(() => { + playbackMocks.stopVoicePlayback() + + return Promise.resolve(true) + }) + + const response: PendingResponse = { id: 'assistant-1', pending: true, text: 'Hello there.' } + const { hook, onSubmit, resolveSubmit } = renderConversation() + + await submitFirstVoiceTurn(hook, onSubmit, resolveSubmit, response) + hook.rerender({ busy: false, enabled: true, response }) + + await act(async () => streamed.resolve('fallback')) + playbackMocks.stopVoicePlayback() + response.pending = false + hook.rerender({ busy: false, enabled: true, response }) + + await act(async () => { + await new Promise(resolve => window.setTimeout(resolve, 350)) + }) + + expect(playbackMocks.playSpeechText).not.toHaveBeenCalled() + expect(micMocks.start).toHaveBeenCalledTimes(1) + expect(hook.result.current.status).toBe('idle') + + await act(async () => hook.result.current.end()) + }) + + it('stays idle when the user explicitly stops fallback speech', async () => { + const speech = deferred() + + playbackMocks.startSpeechStream.mockResolvedValue(null) + playbackMocks.playSpeechText.mockImplementation(() => { + playbackMocks.stopVoicePlayback() + + return speech.promise + }) + + const response: PendingResponse = { id: 'assistant-1', pending: false, text: 'Hello there.' } + const { hook, onSubmit, resolveSubmit } = renderConversation() + + await submitFirstVoiceTurn(hook, onSubmit, resolveSubmit, response) + hook.rerender({ busy: false, enabled: true, response }) + await waitFor(() => expect(playbackMocks.playSpeechText).toHaveBeenCalledTimes(1)) + + playbackMocks.stopVoicePlayback() + await act(async () => speech.resolve(true)) + await waitFor(() => expect(hook.result.current.status).toBe('idle')) + expect(micMocks.start).toHaveBeenCalledTimes(1) + + await act(async () => hook.result.current.end()) + }) + + it('does not reopen the microphone after voice mode ends during speech', async () => { + const speech = deferred<'done' | 'fallback'>() + + const session = { + append: vi.fn(), + done: speech.promise, + finish: vi.fn() + } + + playbackMocks.startSpeechStream.mockImplementation(async () => { + playbackMocks.stopVoicePlayback() + + return session + }) + + const response: PendingResponse = { id: 'assistant-1', pending: true, text: 'Hello there.' } + const { hook, onSubmit, resolveSubmit } = renderConversation() + + await submitFirstVoiceTurn(hook, onSubmit, resolveSubmit, response) + hook.rerender({ busy: false, enabled: true, response }) + hook.rerender({ busy: false, enabled: false, response }) + await waitFor(() => expect(hook.result.current.status).toBe('idle')) + + await act(async () => speech.resolve('done')) + expect(micMocks.start).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts index 64b1c86c0ae8..03cd6253dd30 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts @@ -55,7 +55,7 @@ export function useVoiceConversation({ const speechSessionRef = useRef(null) const stopBargeMonitorRef = useRef<(() => void) | null>(null) const bargeCapturePendingRef = useRef(false) - const speechStartSequenceRef = useRef(0) + const speechStartSequenceRef = useRef(null) const enabledRef = useRef(enabled) const mutedRef = useRef(muted) const busyRef = useRef(busy) @@ -216,10 +216,11 @@ export function useVoiceConversation({ // If stopVoicePlayback() was called externally (Stop button, end), the // voice-playback sequence has advanced past what we captured at speech // start — don't auto-start the next sentence, the user chose to stop. - const stoppedByUser = - speechStartSequenceRef.current > 0 && $voicePlayback.get().sequence > speechStartSequenceRef.current + const speechStartSequence = speechStartSequenceRef.current - speechStartSequenceRef.current = 0 + const stoppedByUser = speechStartSequence !== null && $voicePlayback.get().sequence > speechStartSequence + + speechStartSequenceRef.current = null if (enabledRef.current && !stoppedByUser) { pendingStartRef.current = true @@ -330,6 +331,18 @@ export function useVoiceConversation({ return } + const speechStartSequence = speechStartSequenceRef.current + + if (speechStartSequence !== null && $voicePlayback.get().sequence > speechStartSequence) { + // Preserve an explicit Stop made while a failed stream waits to hand + // off to whole-text playback. Starting fallback would otherwise bump + // the sequence again and erase the user's stop signal. + awaitingSpokenResponseRef.current = false + settleAfterSpeech(false) + + return + } + const response = pendingResponse() if (!response || response.id !== responseId) { @@ -351,9 +364,14 @@ export function useVoiceConversation({ barged = true }) + const playback = playSpeechText(response.text, { source: 'voice-conversation' }) + + // playSpeechText() first stops any prior audio, which advances the + // sequence internally. Capture after that expected bump so only a + // later, external stop suppresses the next listening turn. speechStartSequenceRef.current = $voicePlayback.get().sequence - void playSpeechText(response.text, { source: 'voice-conversation' }) + void playback .catch(error => notifyError(error, voiceCopy.playbackFailed)) .finally(() => { if (responseIdRef.current === responseId) { @@ -410,6 +428,9 @@ export function useVoiceConversation({ return } + // startSpeechStream() stops prior playback before opening its session. + // Treat that sequence bump as startup, not a user-requested stop. + speechStartSequenceRef.current = $voicePlayback.get().sequence speechSessionRef.current = session // Timer-driven feed: reply text flows into the session at delta rate