diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts index 5184232f87b9..3321f909148f 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts @@ -7,8 +7,8 @@ import { triggerHaptic } from '@/lib/haptics' import { $voiceConversationStartRequest, takeVoiceConversationStart } from '@/store/composer' import { resetBrowseState } from '@/store/composer-input-history' import { $gateway } from '@/store/gateway' -import { notifyError } from '@/store/notifications' -import { $autoSpeakReplies, setAutoSpeakReplies } from '@/store/voice-prefs' +import { notify, notifyError } from '@/store/notifications' +import { $autoSpeakReplies, $voiceStopPhrase, setAutoSpeakReplies } from '@/store/voice-prefs' import { resumeWakeAfterVoice } from '@/store/wake-word' import type { ComposerTarget } from '../focus' @@ -208,6 +208,26 @@ export function useComposerVoice({ } }, [pauseWakeForVoice, resumeWakeIfPaused, voiceConversationActive]) + // 'Say "stop" to end the voice chat.' notice when the conversation starts. + // Phrase comes from voice.stop_phrases (first entry) so a custom phrase + // renders correctly; a null phrase (stop_phrases: []) shows no notice. + useEffect(() => { + if (!voiceConversationActive) { + return + } + + const phrase = $voiceStopPhrase.get() + + if (phrase) { + notify({ + id: 'voice-stop-hint', + kind: 'info', + icon: 'mic', + message: t.notifications.voice.sayStopToEnd(phrase) + }) + } + }, [t, voiceConversationActive]) + useEffect(() => resumeWakeIfPaused, [resumeWakeIfPaused]) // Explicit start/end for the on-screen conversation controls (the hotkey uses diff --git a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts index b2e54da3f2e2..9ac3c7432989 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-voice-conversation.ts @@ -1,6 +1,7 @@ import { useCallback, useEffect, useRef, useState } from 'react' import { useI18n } from '@/i18n' +import { startThinkingSound, stopThinkingSound } from '@/lib/thinking-sound' import { monitorSpeechDuringPlayback } from '@/lib/voice-barge-in' import { markVoicePlaybackInterrupted, @@ -574,6 +575,22 @@ export function useVoiceConversation({ return () => window.removeEventListener('keydown', onKeyDown, { capture: true }) }, [enabled, stopTurn]) + // Ambient "thinking" sound: while the agent works (status 'thinking') no + // audio flows, which reads as dead air mid-conversation. Calm bubble blips + // fill the gap; they stop the INSTANT speech starts, the mic re-arms, or the + // conversation ends. Gated by voice.thinking_sound + the shared sound mute. + useEffect(() => { + if (enabled && !muted && status === 'thinking') { + startThinkingSound() + + return stopThinkingSound + } + + stopThinkingSound() + + return undefined + }, [enabled, muted, status]) + // Drive the loop: when a voice-submitted reply appears, open a live speech // session (which feeds itself from then on). Otherwise start listening when // idle between turns. diff --git a/apps/desktop/src/app/session/hooks/use-hermes-config.ts b/apps/desktop/src/app/session/hooks/use-hermes-config.ts index 06f5a1393671..22d6ae552461 100644 --- a/apps/desktop/src/app/session/hooks/use-hermes-config.ts +++ b/apps/desktop/src/app/session/hooks/use-hermes-config.ts @@ -14,7 +14,7 @@ import { setDefaultReasoningEffort, setIntroPersonality } from '@/store/session' -import { applyAutoSpeakFromConfig } from '@/store/voice-prefs' +import { applyAutoSpeakFromConfig, applyThinkingSoundFromConfig, applyVoiceStopPhraseFromConfig } from '@/store/voice-prefs' const DEFAULT_VOICE_SECONDS = 120 const FAST_TIERS = new Set(['fast', 'priority', 'on']) @@ -105,6 +105,8 @@ export function useHermesConfig({ activeSessionIdRef }: HermesConfigOptions) { setVoiceMaxRecordingSeconds(recordingLimit(config.voice?.max_recording_seconds)) setSttEnabled(config.stt?.enabled !== false) applyAutoSpeakFromConfig(config) + applyVoiceStopPhraseFromConfig(config) + applyThinkingSoundFromConfig(config) } catch { // Config is nice-to-have; chat still works without it. } diff --git a/apps/desktop/src/i18n/ar.ts b/apps/desktop/src/i18n/ar.ts index a0c2fe06c05d..c72898a9cea5 100644 --- a/apps/desktop/src/i18n/ar.ts +++ b/apps/desktop/src/i18n/ar.ts @@ -137,6 +137,7 @@ export const ar = defineLocale({ noSpeechDetected: 'لم يتم اكتشاف كلام', playbackFailed: 'فشل تشغيل الصوت', recordingFailed: 'فشل التسجيل', + sayStopToEnd: phrase => `قل "${phrase}" لإنهاء المحادثة الصوتية.`, transcriptionFailed: 'فشل التفريغ النصي', transcriptionUnavailable: 'التفريغ النصي غير متاح.', tryRecordingAgain: 'حاول التسجيل مرة أخرى.', diff --git a/apps/desktop/src/i18n/en.ts b/apps/desktop/src/i18n/en.ts index 29fa06c439fe..d55c494e0704 100644 --- a/apps/desktop/src/i18n/en.ts +++ b/apps/desktop/src/i18n/en.ts @@ -154,6 +154,7 @@ export const en: Translations = { noSpeechDetected: 'No speech detected', playbackFailed: 'Voice playback failed', recordingFailed: 'Voice recording failed', + sayStopToEnd: phrase => `Say "${phrase}" to end the voice chat.`, transcriptionFailed: 'Voice transcription failed', transcriptionUnavailable: 'Voice transcription is not available yet.', tryRecordingAgain: 'Try recording again.', diff --git a/apps/desktop/src/i18n/ja.ts b/apps/desktop/src/i18n/ja.ts index 22a11179e084..ad8f206abe2e 100644 --- a/apps/desktop/src/i18n/ja.ts +++ b/apps/desktop/src/i18n/ja.ts @@ -155,6 +155,7 @@ export const ja = defineLocale({ noSpeechDetected: '音声が検出されませんでした', playbackFailed: '音声再生に失敗しました', recordingFailed: '音声録音に失敗しました', + sayStopToEnd: phrase => `「${phrase}」と言うと音声チャットを終了できます。`, transcriptionFailed: '音声文字起こしに失敗しました', transcriptionUnavailable: '音声文字起こしはまだ利用できません。', tryRecordingAgain: 'もう一度録音してください。', diff --git a/apps/desktop/src/i18n/types.ts b/apps/desktop/src/i18n/types.ts index 9b986fd90a92..8ccc8a034ca3 100644 --- a/apps/desktop/src/i18n/types.ts +++ b/apps/desktop/src/i18n/types.ts @@ -195,6 +195,7 @@ export interface Translations { noSpeechDetected: string playbackFailed: string recordingFailed: string + sayStopToEnd: (phrase: string) => string transcriptionFailed: string transcriptionUnavailable: string tryRecordingAgain: string diff --git a/apps/desktop/src/i18n/zh-hant.ts b/apps/desktop/src/i18n/zh-hant.ts index 9f1c2474e40a..5343d3957244 100644 --- a/apps/desktop/src/i18n/zh-hant.ts +++ b/apps/desktop/src/i18n/zh-hant.ts @@ -150,6 +150,7 @@ export const zhHant = defineLocale({ noSpeechDetected: '未偵測到語音', playbackFailed: '語音播放失敗', recordingFailed: '語音錄製失敗', + sayStopToEnd: phrase => `說「${phrase}」即可結束語音對話。`, transcriptionFailed: '語音轉寫失敗', transcriptionUnavailable: '語音轉寫暫不可用。', tryRecordingAgain: '請再錄製一次。', diff --git a/apps/desktop/src/i18n/zh.ts b/apps/desktop/src/i18n/zh.ts index 7a881a763197..86049412b4ca 100644 --- a/apps/desktop/src/i18n/zh.ts +++ b/apps/desktop/src/i18n/zh.ts @@ -150,6 +150,7 @@ export const zh: Translations = { noSpeechDetected: '没有检测到语音', playbackFailed: '语音播放失败', recordingFailed: '语音录制失败', + sayStopToEnd: phrase => `说“${phrase}”即可结束语音对话。`, transcriptionFailed: '语音转写失败', transcriptionUnavailable: '语音转写暂不可用。', tryRecordingAgain: '请再录一次。', diff --git a/apps/desktop/src/lib/thinking-sound.test.ts b/apps/desktop/src/lib/thinking-sound.test.ts new file mode 100644 index 000000000000..536180cfa2d7 --- /dev/null +++ b/apps/desktop/src/lib/thinking-sound.test.ts @@ -0,0 +1,118 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/store/haptics', () => ({ $hapticsMuted: { get: vi.fn(() => false) } })) +vi.mock('@/hermes', () => ({ + getHermesConfigRecord: vi.fn(async () => ({})), + saveHermesConfig: vi.fn(async () => undefined) +})) + +import { $hapticsMuted } from '@/store/haptics' +import { $thinkingSoundEnabled } from '@/store/voice-prefs' + +import { isThinkingSoundActive, startThinkingSound, stopThinkingSound } from './thinking-sound' + +class FakeOscillator { + type = 'sine' + frequency = { exponentialRampToValueAtTime: vi.fn(), setValueAtTime: vi.fn() } + connect = vi.fn() + start = vi.fn() + stop = vi.fn() +} + +class FakeGain { + gain = { exponentialRampToValueAtTime: vi.fn(), setValueAtTime: vi.fn() } + connect = vi.fn() +} + +const started: FakeOscillator[] = [] + +class FakeAudioContext { + currentTime = 0 + destination = {} + state = 'running' + + createOscillator() { + const osc = new FakeOscillator() + + started.push(osc) + + return osc + } + + createGain() { + return new FakeGain() + } + + resume() { + return Promise.resolve() + } +} + +describe('thinking-sound', () => { + beforeEach(() => { + vi.useFakeTimers() + started.length = 0 + $thinkingSoundEnabled.set(true) + vi.mocked($hapticsMuted.get).mockReturnValue(false) + vi.stubGlobal('AudioContext', FakeAudioContext) + }) + + afterEach(() => { + stopThinkingSound() + vi.unstubAllGlobals() + vi.useRealTimers() + }) + + it('plays repeating blips while active and stops instantly', () => { + startThinkingSound() + expect(isThinkingSoundActive()).toBe(true) + + vi.advanceTimersByTime(5_000) + expect(started.length).toBeGreaterThanOrEqual(3) + + const count = started.length + + stopThinkingSound() + expect(isThinkingSoundActive()).toBe(false) + vi.advanceTimersByTime(5_000) + expect(started.length).toBe(count) // nothing after stop + }) + + it('respects the voice.thinking_sound config gate', () => { + $thinkingSoundEnabled.set(false) + startThinkingSound() + expect(isThinkingSoundActive()).toBe(false) + vi.advanceTimersByTime(3_000) + expect(started.length).toBe(0) + }) + + it('stays silent while sounds are muted but keeps the loop alive', () => { + vi.mocked($hapticsMuted.get).mockReturnValue(true) + startThinkingSound() + vi.advanceTimersByTime(3_000) + expect(started.length).toBe(0) + + // Unmute mid-loop → blips resume without a restart. + vi.mocked($hapticsMuted.get).mockReturnValue(false) + vi.advanceTimersByTime(3_000) + expect(started.length).toBeGreaterThan(0) + }) + + it('start is idempotent', () => { + startThinkingSound() + startThinkingSound() + vi.advanceTimersByTime(1_300) + + // One loop: at most ~2 blips in 1.3s (first at 400ms, next ≥800ms later). + expect(started.length).toBeLessThanOrEqual(2) + }) + + it('never throws when WebAudio is unavailable', () => { + vi.stubGlobal('AudioContext', undefined) + expect(() => { + startThinkingSound() + vi.advanceTimersByTime(2_000) + }).not.toThrow() + stopThinkingSound() + }) +}) diff --git a/apps/desktop/src/lib/thinking-sound.ts b/apps/desktop/src/lib/thinking-sound.ts new file mode 100644 index 000000000000..6a4d2bd6da8e --- /dev/null +++ b/apps/desktop/src/lib/thinking-sound.ts @@ -0,0 +1,108 @@ +// Ambient "thinking" sound for the desktop voice conversation. While the agent +// works (status === 'thinking') no audio flows, which reads as "it died" during +// long thinking/tool stretches. A calm, quiet, repeating pair of soft bubble +// blips fills the gap — same WebAudio oscillator synthesis approach as +// wake-sound.ts / completion-sound.ts (no asset to ship), mirroring the +// backend's numpy-synthesized blips in tools/voice_mode.py so CLI and desktop +// sound alike. +// +// Honours the shared sound-mute toggle ($hapticsMuted) and the +// voice.thinking_sound config gate ($thinkingSoundEnabled). Stops instantly on +// stopThinkingSound() — callers fire it the moment TTS starts, the mic re-arms, +// or the conversation ends. + +import { $hapticsMuted } from '@/store/haptics' +import { $thinkingSoundEnabled } from '@/store/voice-prefs' + +let ctx: AudioContext | null = null +let timer: number | null = null +let blipIndex = 0 + +function getCtx(): AudioContext | null { + if (typeof window === 'undefined') { + return null + } + + try { + if (!ctx) { + const Ctor = + window.AudioContext || (window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext + + if (!Ctor) { + return null + } + + ctx = new Ctor() + } + + if (ctx.state === 'suspended') { + void ctx.resume().catch(() => undefined) + } + + return ctx + } catch { + return null + } +} + +// One soft "blub": short sine with a gentle downward pitch glide and a smooth +// attack into an exponential decay — no clicks, deliberately quiet. +function blub(ac: AudioContext, freq: number) { + const t0 = ac.currentTime + 0.01 + const dur = 0.16 + const osc = ac.createOscillator() + const env = ac.createGain() + + osc.type = 'sine' + osc.frequency.setValueAtTime(freq, t0) + osc.frequency.exponentialRampToValueAtTime(freq * 0.72, t0 + dur) + + env.gain.setValueAtTime(0.0001, t0) + env.gain.exponentialRampToValueAtTime(0.08, t0 + 0.02) + env.gain.exponentialRampToValueAtTime(0.0001, t0 + dur) + + osc.connect(env) + env.connect(ac.destination) + osc.start(t0) + osc.stop(t0 + dur + 0.02) +} + +export function isThinkingSoundActive(): boolean { + return timer !== null +} + +/** Start the repeating thinking blips (idempotent). Best-effort, never throws. */ +export function startThinkingSound(): void { + if (timer !== null || !$thinkingSoundEnabled.get()) { + return + } + + const tick = () => { + if ($hapticsMuted.get() === false) { + const ac = getCtx() + + if (ac) { + try { + // Alternate two calm pitches (G4 / E4), matching the backend blips. + blub(ac, blipIndex % 2 === 0 ? 392 : 329.6) + } catch { + // Audio backend unavailable — stay silent, keep the loop harmless. + } + } + } + + blipIndex += 1 + // ~0.8-1.2s spacing with slight randomization so it reads organic. + timer = window.setTimeout(tick, 800 + Math.random() * 400) + } + + timer = window.setTimeout(tick, 400) +} + +/** Stop the thinking blips instantly (idempotent). */ +export function stopThinkingSound(): void { + if (timer !== null) { + window.clearTimeout(timer) + timer = null + } +} diff --git a/apps/desktop/src/store/voice-prefs.test.ts b/apps/desktop/src/store/voice-prefs.test.ts new file mode 100644 index 000000000000..e648aa744bec --- /dev/null +++ b/apps/desktop/src/store/voice-prefs.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('@/hermes', () => ({ + getHermesConfigRecord: vi.fn(async () => ({})), + saveHermesConfig: vi.fn(async () => undefined) +})) + +import { $voiceStopPhrase, applyVoiceStopPhraseFromConfig } from './voice-prefs' + +describe('applyVoiceStopPhraseFromConfig', () => { + it('defaults to "stop" when the key is absent (backend default applies)', () => { + applyVoiceStopPhraseFromConfig({ voice: {} }) + expect($voiceStopPhrase.get()).toBe('stop') + + applyVoiceStopPhraseFromConfig(null) + expect($voiceStopPhrase.get()).toBe('stop') + }) + + it('uses the first configured phrase so a custom phrase renders correctly', () => { + applyVoiceStopPhraseFromConfig({ voice: { stop_phrases: ['goodbye hermes', 'stop'] } }) + expect($voiceStopPhrase.get()).toBe('goodbye hermes') + }) + + it('coerces a bare string like the backend does', () => { + applyVoiceStopPhraseFromConfig({ voice: { stop_phrases: 'halt' } }) + expect($voiceStopPhrase.get()).toBe('halt') + }) + + it('null phrase when stop phrases are disabled — no notice is shown', () => { + applyVoiceStopPhraseFromConfig({ voice: { stop_phrases: [] } }) + expect($voiceStopPhrase.get()).toBeNull() + }) + + it('malformed entries are skipped; all-blank list disables', () => { + applyVoiceStopPhraseFromConfig({ voice: { stop_phrases: [' ', ''] } }) + expect($voiceStopPhrase.get()).toBeNull() + }) +}) diff --git a/apps/desktop/src/store/voice-prefs.ts b/apps/desktop/src/store/voice-prefs.ts index f7e414e2556c..51a073c23b36 100644 --- a/apps/desktop/src/store/voice-prefs.ts +++ b/apps/desktop/src/store/voice-prefs.ts @@ -12,6 +12,42 @@ export function applyAutoSpeakFromConfig(config: { voice?: { auto_tts?: unknown $autoSpeakReplies.set(Boolean(config?.voice?.auto_tts)) } +// First configured `voice.stop_phrases` entry — drives the "Say "stop" to end +// the voice chat" notice shown when a voice conversation starts. `null` means +// the user disabled stop phrases (`stop_phrases: []`), so no notice is shown. +// Defaults to "stop" (the backend default) before config loads. +export const $voiceStopPhrase = atom('stop') + +/** Seed the stop-phrase atom from a loaded config payload (mount / refresh). */ +export function applyVoiceStopPhraseFromConfig( + config: { voice?: { stop_phrases?: unknown } | null } | null | undefined +) { + const raw = config?.voice?.stop_phrases + + if (raw === undefined) { + // Key absent — backend default applies. + $voiceStopPhrase.set('stop') + + return + } + + const list = Array.isArray(raw) ? raw : typeof raw === 'string' ? [raw] : [] + const first = list.map(entry => String(entry).trim()).find(entry => entry.length > 0) + + $voiceStopPhrase.set(first ?? null) +} + +// `voice.thinking_sound` — ambient bubble blips while the agent works during a +// voice conversation (default on, matching the backend default). +export const $thinkingSoundEnabled = atom(true) + +/** Seed the thinking-sound gate from a loaded config payload. */ +export function applyThinkingSoundFromConfig( + config: { voice?: { thinking_sound?: unknown } | null } | null | undefined +) { + $thinkingSoundEnabled.set(config?.voice?.thinking_sound !== false) +} + /** * Flip the preference and persist it. Optimistic — the atom updates instantly and * reverts if the config write fails. Read-modify-writes the whole record (the diff --git a/apps/desktop/src/types/hermes.ts b/apps/desktop/src/types/hermes.ts index 7f4da0706a8b..65979f5e20e7 100644 --- a/apps/desktop/src/types/hermes.ts +++ b/apps/desktop/src/types/hermes.ts @@ -331,6 +331,8 @@ export interface HermesConfig { voice?: { max_recording_seconds?: number auto_tts?: boolean + stop_phrases?: unknown + thinking_sound?: unknown } } diff --git a/cli.py b/cli.py index 86463a9b89cc..2017e6b9f269 100644 --- a/cli.py +++ b/cli.py @@ -12000,14 +12000,27 @@ def _voice_stop_and_transcribe(self): pass # Track consecutive no-speech cycles to avoid infinite restart loops. + # While the agent is mid-turn or TTS is speaking, the user is + # CORRECTLY silent (waiting/listening) — those cycles must not + # count, or a multi-minute tool run ends the voice chat under + # the user. The stop phrase and barge-in still work during the + # hold (they run on their own paths above). stop_continuous_restart = False + _tts_done = getattr(self, "_voice_tts_done", None) + _activity_hold = bool( + getattr(self, "_agent_running", False) + or (_tts_done is not None and not _tts_done.is_set()) + ) if not submitted: - self._no_speech_count = getattr(self, '_no_speech_count', 0) + 1 - if self._no_speech_count >= 3: - self._voice_continuous = False - self._no_speech_count = 0 - _cprint(f"{_DIM}No speech detected 3 times, continuous mode stopped.{_RST}") - stop_continuous_restart = True + if _activity_hold: + pass # held: keep listening without counting the cycle + else: + self._no_speech_count = getattr(self, '_no_speech_count', 0) + 1 + if self._no_speech_count >= 3: + self._voice_continuous = False + self._no_speech_count = 0 + _cprint(f"{_DIM}No speech detected 3 times, continuous mode stopped.{_RST}") + stop_continuous_restart = True else: self._no_speech_count = 0 @@ -12033,6 +12046,19 @@ def _voice_speak_response_async(self, text: str) -> None: args=(text,), daemon=True, ).start() + # Spoken barge-in must work on the whole-file fallback path too — + # previously only the streaming pipeline armed the monitor, so when + # streaming TTS couldn't start (missing sounddevice, failed probe) + # talking over the reply did nothing. The monitor's _cut_playback + # uses stop_playback(), which kills the file player, so the same + # machinery covers this path; the stop event it receives is only + # used to signal the (nonexistent) streaming pipeline. + if self._voice_continuous: + threading.Thread( + target=self._voice_barge_in_monitor, + args=(threading.Event(),), + daemon=True, + ).start() def _voice_speak_response(self, text: str): """Speak the agent's response aloud using TTS (runs in background thread).""" @@ -12117,6 +12143,11 @@ def _voice_barge_in_monitor(self, stop_event: threading.Event) -> None: — restarting the recorder after detection would lose the opening words). ``_voice_barge_capture`` suppresses process_loop's auto- restart until the captured utterance has been submitted. + + A short startup grace period delays VAD activation so TTS playback + has time to establish before the mic starts listening. Without + this, speaker bleed during the first few hundred milliseconds can + falsely trigger barge-in and cut the response short. """ try: from hermes_cli.config import load_config @@ -12125,8 +12156,25 @@ def _voice_barge_in_monitor(self, stop_event: threading.Event) -> None: return from tools.voice_mode import listen_for_speech, stop_playback + # Grace period: wait briefly before opening the mic so the + # first TTS sentence is already playing and the VAD calibration + # samples the actual playback level (not silence). This + # prevents speaker bleed from falsely triggering barge-in + # at the start of playback. + _grace_s = float(voice_cfg.get("barge_in_grace_seconds", 2.0)) + if _grace_s > 0: + stop_event.wait(timeout=_grace_s) + if stop_event.is_set() or self._voice_tts_done.is_set(): + return + def _cut_playback(): if not self._voice_tts_done.is_set(): + import traceback as _tb + logger.debug( + "TTS CUT: barge-in _cut_playback fired (VAD trip) — " + "stop_event.set() + stop_playback()\n%s", + "".join(_tb.format_stack()), + ) from tools.tts_streaming import mark_speech_interrupted mark_speech_interrupted() self._voice_barge_capture.set() @@ -12137,6 +12185,8 @@ def _cut_playback(): lambda: stop_event.is_set() or self._voice_tts_done.is_set(), capture=True, on_trigger=_cut_playback, + sustained_ms=1000, + calibration_ms=800, ) if wav_path and self._voice_barge_capture.is_set(): self._voice_submit_barge_utterance(wav_path) @@ -12247,6 +12297,15 @@ def _enable_voice_mode(self): _ptt_display = self._voice_record_key_label() _cprint(f"\n{_ACCENT}Voice mode enabled{tts_status}{_RST}") _cprint(f" {_DIM}{_ptt_display} to start/stop recording{_RST}") + # Spoken-stop hint sourced from voice.stop_phrases (first entry); the + # helper returns "" when stop phrases are disabled — show no hint then. + try: + from tools.voice_mode import voice_stop_hint + _stop_hint = voice_stop_hint() + except Exception: + _stop_hint = "" + if _stop_hint: + _cprint(f" {_DIM}{_stop_hint}{_RST}") _cprint(f" {_DIM}/voice tts to toggle speech output{_RST}") _cprint(f" {_DIM}/voice off to disable voice mode{_RST}") @@ -12302,6 +12361,7 @@ def _bg_shutdown(rec=recorder): # Stop any active TTS playback (file player + streaming pipeline) try: if self._voice_tts_stop is not None: + logger.info("TTS CUT: _disable_voice_mode setting stop event") self._voice_tts_stop.set() from tools.voice_mode import stop_playback stop_playback() @@ -13304,10 +13364,12 @@ def _stage_user_message() -> None: # chunks as they arrive, everything else synthesizes per sentence. use_streaming_tts = False _streaming_box_opened = False + _thinking_started = False text_queue = None tts_thread = None stream_callback = None stop_event = None + _tts_normal_exit = False if self._voice_tts: try: @@ -13325,23 +13387,32 @@ def _stage_user_message() -> None: text_queue = queue.Queue() stop_event = threading.Event() - def display_callback(sentence: str): - """Called by TTS consumer when a sentence is ready to display + speak.""" - nonlocal _streaming_box_opened - if not _streaming_box_opened: - _streaming_box_opened = True - w = self._scrollback_box_width(getattr(self.console, "width", 80)) - label = " ⚕ Hermes " - if self.show_timestamps: - label = f"{label}{datetime.now().strftime(getattr(self, 'timestamp_format', '%H:%M'))} " - fill = w - 2 - HermesCLI._status_bar_display_width(label) - _cprint(f"\n{_ACCENT}╭─{label}{'─' * max(fill - 1, 0)}╮{_RST}") - _cprint(f"{_STREAM_PAD}{sentence.rstrip()}") + # When token streaming is enabled (the common case), the + # CLI's _stream_delta already renders text token-by-token as + # the model generates it. Passing a display_callback here too + # would render every sentence a second time. Only attach the + # callback when streaming is disabled, so the TTS consumer + # becomes the sole display path. + _tts_display_cb = None + if not self.streaming_enabled: + def display_callback(sentence: str): + """Called by TTS consumer when a sentence is ready to display + speak.""" + nonlocal _streaming_box_opened + if not _streaming_box_opened: + _streaming_box_opened = True + w = self._scrollback_box_width(getattr(self.console, "width", 80)) + label = " ⚕ Hermes " + if self.show_timestamps: + label = f"{label}{datetime.now().strftime(getattr(self, 'timestamp_format', '%H:%M'))} " + fill = w - 2 - HermesCLI._status_bar_display_width(label) + _cprint(f"\n{_ACCENT}╭─{label}{'─' * max(fill - 1, 0)}╮{_RST}") + _cprint(f"{_STREAM_PAD}{sentence.rstrip()}") + _tts_display_cb = display_callback tts_thread = threading.Thread( target=stream_tts_to_speaker, args=(text_queue, stop_event, self._voice_tts_done), - kwargs={"display_callback": display_callback}, + kwargs={"display_callback": _tts_display_cb}, daemon=True, ) tts_thread.start() @@ -13498,6 +13569,27 @@ def run_agent(): agent_thread = threading.Thread(target=run_agent, daemon=True) agent_thread.start() + # Ambient "thinking" sound: calm bubble blips while the agent + # works in voice mode with no audio flowing, so the user knows + # it's alive during long thinking/tool stretches. Skipped per-blip + # while TTS speaks, the mic records, or a barge capture is live; + # stopped outright as soon as the turn ends. voice.thinking_sound + # gates it (default on); macOS is handled inside (TCC-safe skip). + _thinking_started = False + if self._voice_mode: + try: + from tools.voice_mode import start_thinking_sound + + _thinking_started = start_thinking_sound( + should_play=lambda: ( + self._voice_tts_done.is_set() + and not self._voice_recording + and not self._voice_barge_capture.is_set() + ) + ) + except Exception: + _thinking_started = False + # Monitor the dedicated interrupt queue while the agent runs. # _interrupt_queue is separate from _pending_input, so process_loop # and chat() never compete for the same queue. @@ -13611,6 +13703,12 @@ def run_agent(): text_queue.put(None) # sentinel if tts_thread is not None: tts_thread.join(timeout=120) + # Mark normal completion only if the thread actually + # finished. If join() timed out and the thread is still + # alive, leave _tts_normal_exit False so the finally block + # sets stop_event to kill the runaway worker. + if tts_thread is not None and not tts_thread.is_alive(): + _tts_normal_exit = True # Drain any remaining agent output still in the StdoutProxy # buffer so tool/status lines render ABOVE our response box. @@ -13898,16 +13996,30 @@ def run_agent(): print(f"Error: {e}") return None finally: + # Stop the ambient thinking sound the moment the turn ends — + # every exit path (normal, error, interrupt) lands here. + if _thinking_started: + try: + from tools.voice_mode import stop_thinking_sound + stop_thinking_sound() + except Exception: + pass # Ensure streaming TTS resources are cleaned up even on error. # Normal path sends the sentinel at line ~3568; this is a safety # net for exception paths that skip it. Duplicate sentinels are # harmless — stream_tts_to_speaker exits on the first None. + # + # Only set stop_event on the exception path. On normal exit + # (_tts_normal_exit is True) the pipeline has already drained — + # setting stop_event here would race the playback worker and + # could cut the final sentence mid-audio. if text_queue is not None: try: text_queue.put_nowait(None) except Exception: pass - if stop_event is not None: + if stop_event is not None and not _tts_normal_exit: + logger.info("TTS CUT: exception finally block setting stop_event") stop_event.set() if tts_thread is not None and tts_thread.is_alive(): tts_thread.join(timeout=5) @@ -15407,6 +15519,7 @@ def handle_voice_record(event): # the stop event drains the streaming pipeline if one is live. if not cli_ref._voice_tts_done.is_set(): try: + logger.info("TTS CUT: record key handler cutting TTS") from tools.tts_streaming import mark_speech_interrupted mark_speech_interrupted() if cli_ref._voice_tts_stop is not None: diff --git a/contributors/emails/randy@heroictek.com b/contributors/emails/randy@heroictek.com new file mode 100644 index 000000000000..6e6ba5fdf234 --- /dev/null +++ b/contributors/emails/randy@heroictek.com @@ -0,0 +1 @@ +beardedeagle diff --git a/hermes_cli/config.py b/hermes_cli/config.py index b51b4cea4d54..8307b3a338d3 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -2372,9 +2372,11 @@ def _ensure_hermes_home_managed(home: Path): "auto_tts": False, "beep_enabled": True, # Play record start/stop beeps in CLI voice mode "beep_volume": 0.3, # Beep amplitude multiplier (0.0-1.0, default keeps prior hardcoded value) + "thinking_sound": True, # Calm ambient bubble sound while the agent works in voice chat (volume follows beep_volume) "silence_threshold": 200, # RMS below this = silence (0-32767) "silence_duration": 3.0, # Seconds of silence before auto-stop "barge_in": True, # Stop TTS playback when the user starts talking + "barge_in_grace_seconds": 2.0, # Delay before the barge mic opens so VAD calibrates against live TTS playback (0 disables) # Saying EXACTLY one of these phrases (and nothing else) ends the # voice chat instead of being sent to the agent. Case-insensitive, # surrounding punctuation ignored. Set [] to disable. diff --git a/hermes_cli/voice.py b/hermes_cli/voice.py index 83854a015fb0..24ee29736cc4 100644 --- a/hermes_cli/voice.py +++ b/hermes_cli/voice.py @@ -301,6 +301,47 @@ def _play_beep(frequency: int, count: int = 1) -> None: # leak into the mic. _tts_playing = threading.Event() _tts_playing.set() # initially "not playing" + +# ── Silence-count hold (agent busy) ────────────────────────────────── +# While the agent is mid-turn (thinking / tool-calling, possibly for +# minutes) or TTS is playing, the user is CORRECTLY silent — those cycles +# must not count toward the no-speech limit or a long tool run ends the +# voice chat under the user (#silence-must-not-end-the-chat). The host +# surface (tui_gateway) registers a probe that reports "agent busy"; +# TTS-playing is already tracked via _tts_playing above. +_voice_busy_probe: Optional[Callable[[], bool]] = None + + +def set_voice_busy_probe(probe: Optional[Callable[[], bool]]) -> None: + """Register a callable that returns True while the agent is mid-turn. + + Called by the hosting surface (tui_gateway registers one that checks + every session's ``running`` flag). ``None`` clears it. The probe must + be cheap and thread-safe — it runs on the silence-callback thread. + """ + global _voice_busy_probe + _voice_busy_probe = probe + + +def _voice_activity_held() -> bool: + """True while silent cycles must NOT count toward the no-speech limit. + + Held when TTS is playing (the user is listening) or when the + registered busy probe reports the agent mid-turn (the user is + waiting). Fail-open to "not held" so a broken probe can never make + the voice chat immortal. + """ + if not _tts_playing.is_set(): + return True + probe = _voice_busy_probe + if probe is None: + return False + try: + return bool(probe()) + except Exception: + return False + + _continuous_on_transcript: Optional[Callable[[str], None]] = None _continuous_on_status: Optional[Callable[[str], None]] = None _continuous_on_silent_limit: Optional[Callable[[], None]] = None @@ -574,9 +615,17 @@ def _transcribe_and_cleanup(): logger.warning("on_transcript callback raised: %s", e) if track_no_speech: + held = _voice_activity_held() with _continuous_lock: if transcript or stop_phrase: _continuous_no_speech_count = 0 + elif held: + # Agent busy / TTS playing — the user is + # correctly silent; don't count the cycle. + _debug( + "stop_continuous: silent cycle ignored " + "(agent busy or TTS playing)" + ) else: _continuous_no_speech_count += 1 should_halt = ( @@ -712,6 +761,14 @@ def _continuous_on_silence() -> None: _debug(f"_continuous_on_silence: stop phrase {transcript!r} — ending loop") transcript = None + # Silent cycle while the agent is mid-turn or TTS is playing: the user + # is CORRECTLY quiet (waiting/listening), so the cycle must not count + # toward the no-speech limit — a multi-minute tool run would otherwise + # end the voice chat under the user. Checked outside the lock (probe + # may call into the host surface). + _silence_held = (transcript is None and not stop_phrase + and _voice_activity_held()) + with _continuous_lock: if not _continuous_active: # User stopped us while we were transcribing — discard. @@ -719,6 +776,11 @@ def _continuous_on_silence() -> None: return if transcript: _continuous_no_speech_count = 0 + elif _silence_held: + _debug( + "_continuous_on_silence: silent cycle ignored " + "(agent busy or TTS playing)" + ) elif not stop_phrase: _continuous_no_speech_count += 1 should_halt = stop_phrase or ( @@ -819,7 +881,7 @@ def _continuous_on_silence() -> None: # ── TTS API ────────────────────────────────────────────────────────── -def _speak_text_streaming(text: str) -> bool: +def _speak_text_streaming(text: str, stop_event: Optional[threading.Event] = None) -> bool: """Speak ``text`` via the generic streaming dispatcher; True on success. Bridges the one-shot ``speak_text`` contract onto the shared @@ -829,6 +891,11 @@ def _speak_text_streaming(text: str) -> bool: has, so callers (and the mic re-arm logic in ``speak_text``) see no behavioral difference beyond earlier first audio. + ``stop_event`` (optional) is wired straight into the pipeline so + external barge-in / stop paths can cut streaming playback — without + it the pipeline's stop event was private and speech over this path + was uninterruptible (the desktop/TUI fallback-speak hole). + Returns False when playback produced nothing (caller falls back to the whole-file sync path). """ @@ -840,13 +907,14 @@ def _speak_text_streaming(text: str) -> bool: text_queue: "_queue.Queue" = _queue.Queue() text_queue.put(text) text_queue.put(None) # end-of-text sentinel - stop_event = _threading.Event() + if stop_event is None: + stop_event = _threading.Event() done_event = _threading.Event() stream_tts_to_speaker(text_queue, stop_event, done_event) return done_event.is_set() -def speak_text(text: str) -> None: +def speak_text(text: str, stop_event: Optional[threading.Event] = None) -> None: """Synthesize ``text`` with the configured TTS provider and play it. Mirrors cli.py:_voice_speak_response exactly — same markdown strip @@ -901,7 +969,7 @@ def speak_text(text: str) -> None: from tools.tts_tool import _load_tts_config if resolve_streaming_provider(_load_tts_config()) is not None: - if _speak_text_streaming(text): + if _speak_text_streaming(text, stop_event): return except Exception as e: _debug(f"speak_text: streaming dispatch unavailable ({e}); using sync path") diff --git a/tests/hermes_cli/test_voice_wrapper.py b/tests/hermes_cli/test_voice_wrapper.py index 2d996969d45d..a403d3c28921 100644 --- a/tests/hermes_cli/test_voice_wrapper.py +++ b/tests/hermes_cli/test_voice_wrapper.py @@ -424,6 +424,7 @@ def fake_recorder(self, monkeypatch): monkeypatch.setattr(voice, "_continuous_on_status", None) monkeypatch.setattr(voice, "_continuous_on_silent_limit", None) monkeypatch.setattr(voice, "_continuous_auto_restart", True, raising=False) + monkeypatch.setattr(voice, "_voice_busy_probe", None, raising=False) monkeypatch.setattr(voice, "_play_beep", lambda *_, **__: None) class FakeRecorder: @@ -724,6 +725,162 @@ def test_silent_limit_halts_loop_after_three_strikes(self, fake_recorder, monkey assert voice.is_continuous_active() is False assert fake_recorder.cancelled >= 1 + def test_silent_cycles_do_not_count_while_agent_busy(self, fake_recorder, monkeypatch): + """Agent mid-turn: silent cycles must NOT count toward the no-speech + limit — a multi-minute tool run would otherwise end the voice chat + while the user is correctly waiting quietly.""" + import hermes_cli.voice as voice + + monkeypatch.setattr( + voice, + "transcribe_recording", + lambda _p: {"success": True, "transcript": ""}, + ) + monkeypatch.setattr(voice, "is_whisper_hallucination", lambda _t: False) + monkeypatch.setattr(voice, "_voice_busy_probe", lambda: True) + + silent_limit_fired = [] + + voice.start_continuous( + on_transcript=lambda _t: None, + on_silent_limit=lambda: silent_limit_fired.append(True), + ) + + # Way past the 3-strike limit while the agent is busy. + for _ in range(6): + fake_recorder.last_callback() + + assert silent_limit_fired == [] + assert voice.is_continuous_active() is True + assert voice._continuous_no_speech_count == 0 + + # Agent finishes → strikes count again, limit fires as before. + monkeypatch.setattr(voice, "_voice_busy_probe", lambda: False) + for _ in range(3): + fake_recorder.last_callback() + assert silent_limit_fired == [True] + assert voice.is_continuous_active() is False + + def test_silent_cycles_do_not_count_while_tts_playing(self, fake_recorder, monkeypatch): + """TTS speaking: the user is listening, not ignoring the mic.""" + import hermes_cli.voice as voice + + monkeypatch.setattr( + voice, + "transcribe_recording", + lambda _p: {"success": True, "transcript": ""}, + ) + monkeypatch.setattr(voice, "is_whisper_hallucination", lambda _t: False) + # Keep the TTS-wait re-arm path from blocking: _tts_playing cleared + # means "playing"; use a tiny wait timeout via a fake event-like shim. + monkeypatch.setattr(voice, "_voice_busy_probe", None) + + class _FakePlaying: + def is_set(self): + return False + + def wait(self, timeout=None): + return True + + monkeypatch.setattr(voice, "_tts_playing", _FakePlaying()) + + silent_limit_fired = [] + voice.start_continuous( + on_transcript=lambda _t: None, + on_silent_limit=lambda: silent_limit_fired.append(True), + ) + for _ in range(4): + fake_recorder.last_callback() + + assert silent_limit_fired == [] + assert voice._continuous_no_speech_count == 0 + voice.stop_continuous() + + def test_stop_phrase_still_ends_chat_during_busy_hold(self, fake_recorder, monkeypatch): + """The hold suppresses the silence counter only — a spoken stop + phrase must still end the voice chat instantly.""" + import hermes_cli.voice as voice + + monkeypatch.setattr( + voice, + "transcribe_recording", + lambda _p: {"success": True, "transcript": "stop"}, + ) + monkeypatch.setattr(voice, "is_whisper_hallucination", lambda _t: False) + monkeypatch.setattr(voice, "is_voice_stop_phrase", lambda _t: True) + monkeypatch.setattr(voice, "_voice_busy_probe", lambda: True) + + stop_fired = [] + voice.start_continuous( + on_transcript=lambda _t: None, + on_stop_phrase=lambda t: stop_fired.append(t), + ) + fake_recorder.last_callback() + + assert stop_fired == ["stop"] + assert voice.is_continuous_active() is False + + def test_broken_busy_probe_fails_open(self, fake_recorder, monkeypatch): + """A raising probe must not make the voice chat immortal — silent + cycles count as if no probe were registered.""" + import hermes_cli.voice as voice + + monkeypatch.setattr( + voice, + "transcribe_recording", + lambda _p: {"success": True, "transcript": ""}, + ) + monkeypatch.setattr(voice, "is_whisper_hallucination", lambda _t: False) + + def _boom(): + raise RuntimeError("probe broken") + + monkeypatch.setattr(voice, "_voice_busy_probe", _boom) + + silent_limit_fired = [] + voice.start_continuous( + on_transcript=lambda _t: None, + on_silent_limit=lambda: silent_limit_fired.append(True), + ) + for _ in range(3): + fake_recorder.last_callback() + + assert silent_limit_fired == [True] + assert voice.is_continuous_active() is False + + def test_force_transcribe_silent_cycle_held_while_busy(self, fake_recorder, monkeypatch): + """The single-shot (auto_restart=False, force_transcribe) strike path + honors the busy hold too — desktop/TUI clients drive that loop.""" + import hermes_cli.voice as voice + + class ImmediateThread: + def __init__(self, target, daemon=False): + self.target = target + + def start(self): + self.target() + + monkeypatch.setattr(voice.threading, "Thread", ImmediateThread) + monkeypatch.setattr( + voice, + "transcribe_recording", + lambda _p: {"success": True, "transcript": ""}, + ) + monkeypatch.setattr(voice, "is_whisper_hallucination", lambda _t: False) + monkeypatch.setattr(voice, "_voice_busy_probe", lambda: True) + + silent_limit_fired = [] + for _ in range(4): + voice.start_continuous( + on_transcript=lambda _t: None, + on_silent_limit=lambda: silent_limit_fired.append(True), + auto_restart=False, + ) + voice.stop_continuous(force_transcribe=True) + + assert silent_limit_fired == [] + assert voice._continuous_no_speech_count == 0 + def test_stop_during_transcription_discards_restart(self, fake_recorder, monkeypatch): """User hits Ctrl+B mid-transcription: the in-flight transcript must still fire (it's a real utterance), but the loop must NOT restart.""" diff --git a/tests/test_tui_gateway_server.py b/tests/test_tui_gateway_server.py index d4eea3039900..133808fee8c2 100644 --- a/tests/test_tui_gateway_server.py +++ b/tests/test_tui_gateway_server.py @@ -1188,6 +1188,47 @@ def test_voice_toggle_returns_configured_record_key(monkeypatch): assert status_resp["result"]["record_key"] == "ctrl+o" +def test_voice_toggle_on_carries_stop_hint(monkeypatch): + """voice.toggle action=on returns the spoken-stop hint for clients to + render — sourced from voice.stop_phrases so a custom phrase shows + correctly, and empty when the feature is disabled (stop_phrases: []).""" + monkeypatch.setattr(server, "_load_cfg", lambda: {"voice": {}}) + monkeypatch.setitem( + sys.modules, + "tools.voice_mode", + types.SimpleNamespace( + check_voice_requirements=lambda: {"available": True, "details": ""}, + voice_stop_hint=lambda: 'Say "halt" to end the voice chat.', + ), + ) + monkeypatch.setenv("HERMES_VOICE", "0") + + on_resp = server.dispatch( + {"id": "voice-on", "method": "voice.toggle", "params": {"action": "on"}} + ) + assert on_resp["result"]["stop_hint"] == 'Say "halt" to end the voice chat.' + + # Disabled stop phrases → empty hint, clients show nothing. + monkeypatch.setitem( + sys.modules, + "tools.voice_mode", + types.SimpleNamespace( + check_voice_requirements=lambda: {"available": True, "details": ""}, + voice_stop_hint=lambda: "", + ), + ) + on_resp = server.dispatch( + {"id": "voice-on2", "method": "voice.toggle", "params": {"action": "on"}} + ) + assert on_resp["result"]["stop_hint"] == "" + + # off carries no hint text (mode is ending). + off_resp = server.dispatch( + {"id": "voice-off", "method": "voice.toggle", "params": {"action": "off"}} + ) + assert off_resp["result"]["stop_hint"] == "" + + def test_voice_toggle_handles_non_dict_voice_cfg(monkeypatch): """Round-3 Copilot review regression on #19835. @@ -14520,6 +14561,9 @@ def default_listen(should_stop, capture=False, on_trigger=None, **_kw): types.SimpleNamespace( check_tts_requirements=lambda: requirements, stream_tts_to_speaker=fake_stream, + _get_provider=lambda cfg: "edge", + _load_tts_config=lambda: {}, + get_env_value=lambda key, default="": default, ), ) monkeypatch.setitem( @@ -14651,7 +14695,7 @@ def fake_listen(should_stop, capture=False, on_trigger=None, **_kw): with server._tts_stream_lock: state = server._tts_stream_state assert state is not None - assert state["stop"].wait(2.0) + assert state["stop"].wait(5.0) # grace period (2s) + fake_listen + margin deadline = time.monotonic() + 2.0 while time.monotonic() < deadline and wav.exists(): time.sleep(0.01) # unlink (finally) runs after the transcript emit @@ -14662,6 +14706,103 @@ def fake_listen(should_stop, capture=False, on_trigger=None, **_kw): server._tts_stream_stop() +def test_speak_text_with_barge_arms_monitor_and_cuts_playback(monkeypatch, tmp_path): + """The fallback whole-reply speak path (streaming pipeline couldn't + start) and the voice.tts RPC must be barge-able too: speaking over the + reply cuts playback and the captured interruption is emitted as + voice.transcript — previously these paths called speak_text bare and + were uninterruptible by voice.""" + import tools.tts_streaming as ts + + ts._interrupted_at = None + monkeypatch.setenv("HERMES_VOICE", "1") + monkeypatch.setenv("HERMES_VOICE_TTS", "1") + monkeypatch.setattr( + server, + "_load_cfg", + lambda: {"voice": {"barge_in": True, "barge_in_grace_seconds": 0}}, + ) + events: list = [] + monkeypatch.setattr( + server, "_voice_emit", lambda event, payload=None: events.append((event, payload)) + ) + + wav = tmp_path / "barge.wav" + wav.write_bytes(b"RIFF") + + speak_calls = {} + speak_started = threading.Event() + release_speak = threading.Event() + + def fake_speak_text(text, stop_event=None): + speak_calls["text"] = text + speak_calls["stop_event"] = stop_event + speak_started.set() + release_speak.wait(5) + + monkeypatch.setitem( + sys.modules, + "hermes_cli.voice", + types.SimpleNamespace(speak_text=fake_speak_text), + ) + + def fake_listen(should_stop, capture=False, on_trigger=None, **_kw): + assert capture is True + speak_started.wait(5) + on_trigger() # user talks over the reply → cut now + return str(wav) + + _fake_tts_modules( + monkeypatch, + listen=fake_listen, + transcribe=lambda path, model=None: {"success": True, "transcript": "hang on"}, + ) + + server._speak_text_with_barge("a long spoken reply") + + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline and ("voice.transcript", {"text": "hang on"}) not in events: + time.sleep(0.01) + release_speak.set() + + assert speak_calls["text"] == "a long spoken reply" + # The pipeline stop event is shared with speak_text so a streaming + # dispatch inside it is cut too. + assert speak_calls["stop_event"] is not None + assert speak_calls["stop_event"].is_set() + assert ("voice.interrupted", None) in events + assert ("voice.transcript", {"text": "hang on"}) in events + assert ts.take_speech_interrupted() is True + + +def test_speak_text_with_barge_no_monitor_when_voice_mode_off(monkeypatch): + """Auto-speak with voice mode off (no mic loop) must not open the mic.""" + monkeypatch.setenv("HERMES_VOICE", "0") + monkeypatch.setenv("HERMES_VOICE_TTS", "1") + monkeypatch.setattr(server, "_load_cfg", lambda: {"voice": {"barge_in": True}}) + + listened = threading.Event() + + def fake_listen(should_stop, capture=False, on_trigger=None, **_kw): + listened.set() + return None + + done_speaking = threading.Event() + monkeypatch.setitem( + sys.modules, + "hermes_cli.voice", + types.SimpleNamespace( + speak_text=lambda text, stop_event=None: done_speaking.set() + ), + ) + _fake_tts_modules(monkeypatch, listen=fake_listen) + + server._speak_text_with_barge("quiet reply") + assert done_speaking.wait(5) + time.sleep(0.1) + assert not listened.is_set() + + def test_clarify_callback_uses_configured_timeout(monkeypatch): """The TUI/desktop clarify bridge honors the canonical clarify timeout (via _clarify_timeout_seconds) instead of the hardcoded _block default.""" diff --git a/tests/tools/test_voice_cli_integration.py b/tests/tools/test_voice_cli_integration.py index 0fa4304b4876..b9fb323502c4 100644 --- a/tests/tools/test_voice_cli_integration.py +++ b/tests/tools/test_voice_cli_integration.py @@ -1502,3 +1502,47 @@ def test_non_string_input_passes_through(self): cli = self._cli(_voice_mode=True) assert cli._typed_voice_stop(("text", ["img.png"])) is False assert cli._disable_calls == [] + + +# ============================================================================ +# Fallback (whole-file) TTS path arms the spoken barge-in monitor +# ============================================================================ + +class TestFallbackSpeakArmsBargeMonitor: + """_voice_speak_response_async must arm _voice_barge_in_monitor in + continuous voice mode. Previously ONLY the streaming pipeline armed the + monitor (chat() gate), so when streaming TTS couldn't start the whole-file + fallback speech was uninterruptible by voice — Teknium's "speaking over + the agent does nothing" report on the non-streaming path.""" + + def _cli(self, **overrides): + cli = _make_voice_cli(**overrides) + cli._monitor_calls = [] + cli._voice_barge_in_monitor = ( + lambda stop_event: cli._monitor_calls.append(stop_event) + ) + cli._voice_speak_response = lambda text: None + return cli + + def _drain_threads(self): + import time + time.sleep(0.15) + + def test_monitor_armed_in_continuous_voice_mode(self): + cli = self._cli(_voice_mode=True, _voice_tts=True, _voice_continuous=True) + cli._voice_speak_response_async("a reply") + self._drain_threads() + assert len(cli._monitor_calls) == 1 + assert isinstance(cli._monitor_calls[0], threading.Event) + + def test_no_monitor_outside_continuous_mode(self): + cli = self._cli(_voice_mode=True, _voice_tts=True, _voice_continuous=False) + cli._voice_speak_response_async("a reply") + self._drain_threads() + assert cli._monitor_calls == [] + + def test_no_monitor_when_tts_disabled(self): + cli = self._cli(_voice_mode=True, _voice_tts=False, _voice_continuous=True) + cli._voice_speak_response_async("a reply") + self._drain_threads() + assert cli._monitor_calls == [] diff --git a/tests/tools/test_voice_mode.py b/tests/tools/test_voice_mode.py index c8e245f1c079..2ae72076814d 100644 --- a/tests/tools/test_voice_mode.py +++ b/tests/tools/test_voice_mode.py @@ -2075,6 +2075,77 @@ def test_loud_floor_raises_trigger(self, mock_sd): heard, _ = self._run(mock_sd, levels) assert heard is False + def test_quiet_then_loud_playback_does_not_trip(self, mock_sd): + """TTS that starts quiet and gets louder must NOT trip barge-in. + + This is the core regression: a one-shot calibration freezes the + floor from the quiet opening, then louder TTS exceeds the stale + floor and false-triggers. The rolling window keeps the floor + current so the louder passage is absorbed into the floor. + """ + levels = [100] * self.CALIB_BLOCKS + [200] * 30 + [500] * 30 + [1000] * 30 + heard, _ = self._run(mock_sd, levels) + assert heard is False + + def test_8x_multiplier_absorbs_tts_volume_spikes(self, mock_sd): + """TTS volume spikes that would exceed a 5x floor must NOT trip. + + At 5x multiplier, a quiet TTS passage (RMS 200) sets a floor of + ~180 and a trigger of 900. A subsequent louder passage at RMS + 1000 exceeds the trigger, is excluded from the floor window, and + after sustained_ms of consecutive above-trigger blocks the VAD + false-trips and cuts playback mid-sentence. The 8x multiplier + raises the trigger to 1440 so the 1000-RMS passage stays below + it and gets absorbed into the rolling floor. + """ + # Calib at 200 RMS → floor ~180 → 8x trigger = 1440 + # Then 1000 RMS TTS: below 3200 (400*8), absorbed into floor, no trip. + # With old 5x: trigger=2000 (400*5), 1000 < 2000, would NOT trip either. + # To actually test the 8x multiplier, use levels where 5x would trip + # but 8x would not: calib at 200 → floor=180 → 5x trigger=900, + # 8x trigger=1440. Feed 1200 RMS: above 900 (5x trips) but below + # 1440 (8x absorbs). With min_floor=400 the trigger is max(400,180*8)=1440, + # so 1200 < 1440 → no trip at 8x, but 1200 > 900 → would trip at 5x. + levels = [200] * self.CALIB_BLOCKS + [1200] * 50 + heard, _ = self._run(mock_sd, levels) + assert heard is False + + def test_trigger_ceiling_lets_genuine_speech_trip(self, mock_sd): + """Even with a loud TTS floor, genuine speech must still trip. + + Loud TTS at 3000 RMS → floor ~2700 → 8x trigger = 21600, but + the ceiling caps it at 4000. Speech at 5000 RMS exceeds the + capped trigger and trips after sustained_ms blocks. + """ + levels = [3000] * self.CALIB_BLOCKS + [5000] * 50 + heard, _ = self._run(mock_sd, levels) + assert heard is True + + def test_silence_calibration_does_not_false_trip_on_tts(self, mock_sd): + """Calibration during an inter-sentence gap must NOT false-trip. + + If the grace period ends during a pause between TTS sentences, the + calibration window samples near-silence. Without the min_floor clamp, + min_floor locks near zero, the trigger drops to 400 RMS (SILENCE_RMS_THRESHOLD + * 2), and the next TTS sentence at 800 RMS exceeds it — those blocks are + excluded from the rolling window (rms >= trigger), the floor freezes, and + after sustained_ms the VAD false-triggers and cuts playback mid-sentence. + + With the clamp, min_floor stays at SILENCE_RMS_THRESHOLD * 2 = 400, the + trigger is max(400, 400 * 8.0) = 3200, and 800-RMS TTS stays below it and + feeds the rolling floor. No false trip. + """ + # calibration_ms=800 → CALIB_BLOCKS = 800/30 ≈ 26 blocks of silence + # Then TTS resumes at 800 RMS — must NOT trip (below 3200 trigger). + calib = 800 // 30 + levels = [0] * calib + [800] * 100 + heard, _ = self._run( + mock_sd, levels, + sustained_ms=1000, + calibration_ms=800, + ) + assert heard is False + class TestListenForSpeechCapture: """capture=True: the barge monitor records the interruption with pre-roll, diff --git a/tests/tools/test_voice_stop_phrase.py b/tests/tools/test_voice_stop_phrase.py index a6f0f48805b0..974530db8d78 100644 --- a/tests/tools/test_voice_stop_phrase.py +++ b/tests/tools/test_voice_stop_phrase.py @@ -17,9 +17,29 @@ DEFAULT_VOICE_STOP_PHRASES, _load_voice_stop_phrases, is_voice_stop_phrase, + voice_stop_hint, ) +class TestVoiceStopHint: + """The 'Say "stop" to end the voice chat.' hint shown on voice-mode start.""" + + def test_default_phrase(self): + with patch("tools.voice_mode._load_voice_stop_phrases", return_value=("stop",)): + assert voice_stop_hint() == 'Say "stop" to end the voice chat.' + + def test_custom_phrase_uses_first_entry(self): + with patch( + "tools.voice_mode._load_voice_stop_phrases", + return_value=("goodbye hermes", "stop"), + ): + assert voice_stop_hint() == 'Say "goodbye hermes" to end the voice chat.' + + def test_disabled_phrases_show_no_hint(self): + with patch("tools.voice_mode._load_voice_stop_phrases", return_value=()): + assert voice_stop_hint() == "" + + class TestIsVoiceStopPhrase: @pytest.mark.parametrize("utterance", [ "stop", "Stop", "STOP", "stop.", "Stop!", " stop ", '"Stop."', "stop?", diff --git a/tests/tools/test_voice_thinking_sound.py b/tests/tools/test_voice_thinking_sound.py new file mode 100644 index 000000000000..5a21b3f89aa7 --- /dev/null +++ b/tests/tools/test_voice_thinking_sound.py @@ -0,0 +1,189 @@ +"""Tests for the ambient voice-chat "thinking" sound (tools/voice_mode.py). + +Contract: + - `voice.thinking_sound` config gates it (default True). + - `start_thinking_sound()` is idempotent, returns False when disabled. + - The loop synthesizes blips with numpy (no assets), scales volume by + `voice.beep_volume`, and NEVER plays through sounddevice on macOS + (_sounddevice_output_allowed → TCC-safe silent skip). + - `stop_thinking_sound()` stops the loop instantly and is idempotent. + - The should_play callback gates each blip (no blips while TTS speaks + or the mic captures). + - mark_audio_output_active / is_audio_output_active ref-count playback. +""" + +import threading +import time +from unittest.mock import patch + +import pytest + +np = pytest.importorskip( + "numpy", reason="numpy is a lazy voice dependency, absent in hermetic CI" +) + +import tools.voice_mode as vm + + +class _FakeSD: + def __init__(self): + self.played = [] + + def play(self, audio, samplerate=None): + self.played.append((audio, samplerate)) + + def stop(self): + pass + + +def _reset(): + vm.stop_thinking_sound() + # Drain any residual output ref-counts from prior tests. + with vm._audio_output_lock: + vm._audio_output_active_count = 0 + + +class TestConfigGate: + def test_default_enabled(self): + with patch("hermes_cli.config.load_config", return_value={"voice": {}}): + assert vm.thinking_sound_enabled() is True + + def test_disabled_via_config(self): + with patch( + "hermes_cli.config.load_config", + return_value={"voice": {"thinking_sound": False}}, + ): + assert vm.thinking_sound_enabled() is False + + def test_quoted_false_string(self): + with patch( + "hermes_cli.config.load_config", + return_value={"voice": {"thinking_sound": "false"}}, + ): + assert vm.thinking_sound_enabled() is False + + def test_start_refuses_when_disabled(self): + _reset() + with patch.object(vm, "thinking_sound_enabled", return_value=False): + assert vm.start_thinking_sound() is False + assert vm._thinking_stop is None + + +class TestBlipSynthesis: + def test_blip_is_int16_low_volume(self): + with patch.object(vm, "_get_beep_volume", return_value=0.3): + blip = vm._synth_thinking_blip(np, 392.0) + assert blip.dtype == np.int16 + assert len(blip) == int(vm.SAMPLE_RATE * 0.16) + # Quieter than the beeps: 0.3 * 0.5 * 32767 ≈ 4915 peak ceiling. + assert int(np.abs(blip).max()) <= int(0.3 * 0.5 * 32767) + 1 + assert int(np.abs(blip).max()) > 0 + + def test_blip_volume_follows_beep_volume(self): + with patch.object(vm, "_get_beep_volume", return_value=1.0): + loud = vm._synth_thinking_blip(np, 392.0) + with patch.object(vm, "_get_beep_volume", return_value=0.1): + quiet = vm._synth_thinking_blip(np, 392.0) + assert int(np.abs(loud).max()) > int(np.abs(quiet).max()) * 5 + + def test_no_click_smooth_attack(self): + blip = vm._synth_thinking_blip(np, 392.0) + # First sample near zero (enveloped attack, no click). + assert abs(int(blip[0])) < 200 + + +class TestLoopLifecycle: + def test_loop_plays_blips_and_stops_instantly(self): + _reset() + fake = _FakeSD() + stop = threading.Event() + with patch.object(vm, "_sounddevice_output_allowed", return_value=True), \ + patch.object(vm, "_import_audio", return_value=(fake, np)), \ + patch.object(vm, "_get_beep_volume", return_value=0.3): + t = threading.Thread( + target=vm._thinking_sound_loop, args=(stop, None), daemon=True + ) + t.start() + deadline = time.monotonic() + 3.0 + while not fake.played and time.monotonic() < deadline: + time.sleep(0.01) + stop.set() + t.join(timeout=3.0) + assert fake.played, "loop never played a blip" + assert not t.is_alive() + + def test_should_play_false_suppresses_blips(self): + _reset() + fake = _FakeSD() + stop = threading.Event() + with patch.object(vm, "_sounddevice_output_allowed", return_value=True), \ + patch.object(vm, "_import_audio", return_value=(fake, np)), \ + patch.object(vm, "_get_beep_volume", return_value=0.3): + t = threading.Thread( + target=vm._thinking_sound_loop, + args=(stop, lambda: False), + daemon=True, + ) + t.start() + time.sleep(0.3) + stop.set() + t.join(timeout=3.0) + assert fake.played == [] + + def test_macos_tcc_gate_skips_silently(self): + """sounddevice output is gated on macOS — the loop must exit without + importing/playing anything (per-second afplay churn is worse than + silence).""" + _reset() + stop = threading.Event() + + def _boom(): + raise AssertionError("must not import audio when output is gated") + + with patch.object(vm, "_sounddevice_output_allowed", return_value=False), \ + patch.object(vm, "_import_audio", _boom): + vm._thinking_sound_loop(stop, None) # returns immediately + + def test_start_is_idempotent_and_stop_clears(self): + _reset() + with patch.object(vm, "thinking_sound_enabled", return_value=True), \ + patch.object(vm, "_sounddevice_output_allowed", return_value=False): + assert vm.start_thinking_sound() is True + first_stop = vm._thinking_stop + assert vm.start_thinking_sound() is True + assert vm._thinking_stop is first_stop # no second loop + vm.stop_thinking_sound() + assert vm._thinking_stop is None + assert first_stop.is_set() + vm.stop_thinking_sound() # idempotent + + +class TestAudioOutputRefcount: + def test_refcount_tracks_nested_playback(self): + _reset() + assert vm.is_audio_output_active() is False + vm.mark_audio_output_active(True) + vm.mark_audio_output_active(True) + assert vm.is_audio_output_active() is True + vm.mark_audio_output_active(False) + assert vm.is_audio_output_active() is True + vm.mark_audio_output_active(False) + assert vm.is_audio_output_active() is False + # Never goes negative. + vm.mark_audio_output_active(False) + assert vm.is_audio_output_active() is False + + def test_play_audio_file_brackets_refcount(self, tmp_path): + """play_audio_file flags real speaker output for its whole duration, + so the thinking loop knows audio is flowing.""" + _reset() + seen = [] + + def fake_impl(path): + seen.append(vm.is_audio_output_active()) + return True + + with patch.object(vm, "_play_audio_file_impl", fake_impl): + vm.play_audio_file(str(tmp_path / "x.wav")) + assert seen == [True] + assert vm.is_audio_output_active() is False diff --git a/tools/tts_tool.py b/tools/tts_tool.py index c1febc274db2..04abda135d3c 100644 --- a/tools/tts_tool.py +++ b/tools/tts_tool.py @@ -3445,10 +3445,24 @@ def _speak_sentence(sentence: str): audio_iter = streamer.stream(cleaned) if output_stream is not None: import numpy as _np - for chunk in audio_iter: - if stop_event.is_set(): - break - output_stream.write(_np.frombuffer(chunk, dtype=_np.int16).reshape(-1, 1)) + + # Flag real speaker output for the duration of this + # sentence so ambient cues (thinking sound) stay quiet. + # Fail-open: stubbed/partial voice_mode modules (tests) + # must never break sentence playback. + try: + from tools.voice_mode import mark_audio_output_active + except Exception: + def mark_audio_output_active(_active): + return None + mark_audio_output_active(True) + try: + for chunk in audio_iter: + if stop_event.is_set(): + break + output_stream.write(_np.frombuffer(chunk, dtype=_np.int16).reshape(-1, 1)) + finally: + mark_audio_output_active(False) else: # No audio device: buffer chunks to a temp WAV and play it. _play_via_tempfile(audio_iter, stop_event, streamer.sample_rate) diff --git a/tools/voice_mode.py b/tools/voice_mode.py index 3016605c0bd6..81f358834231 100644 --- a/tools/voice_mode.py +++ b/tools/voice_mode.py @@ -528,6 +528,155 @@ def play_beep(frequency: int = 880, duration: float = 0.12, count: int = 1) -> N logger.debug("Beep playback failed: %s", e) +# ============================================================================ +# Thinking sound — calm ambient "blub blub" while the agent works +# ============================================================================ +# During a voice conversation the agent can think / run tools for minutes with +# zero audio, which reads as "it died". A quiet, repeating pair of soft water- +# bubble blips fills that gap. Fully synthesized with numpy (no binary asset), +# volume-scaled by voice.beep_volume, gated by voice.thinking_sound (default +# on), and macOS-TCC-safe: sounddevice OUTPUT is gated there +# (_sounddevice_output_allowed), and spawning afplay every second would churn +# subprocesses, so on macOS the thinking sound is skipped silently. + +# The host's *should_play* callback decides when blips are allowed; the +# module-level output ref-count below tracks when real audio (TTS sentences, +# file playback) is actually flowing so hosts have an accurate signal. + +_audio_output_active_count = 0 +_audio_output_lock = threading.Lock() + + +def mark_audio_output_active(active: bool) -> None: + """Reference-count real audio output (TTS/file playback). + + Playback paths bracket their work with ``mark_audio_output_active(True)`` + / ``(False)`` so ``is_audio_output_active()`` reflects whether speech + audio is leaving the speakers RIGHT NOW — unlike the per-turn TTS-done + events, which stay 'busy' for a whole turn even while the pipeline is + silently waiting for text. + """ + global _audio_output_active_count + with _audio_output_lock: + _audio_output_active_count = max( + 0, _audio_output_active_count + (1 if active else -1) + ) + + +def is_audio_output_active() -> bool: + """True while TTS/file audio is actually playing on the speakers.""" + with _audio_output_lock: + return _audio_output_active_count > 0 + + +_thinking_lock = threading.Lock() +_thinking_stop: Optional[threading.Event] = None + + +def thinking_sound_enabled() -> bool: + """Config gate: ``voice.thinking_sound`` (default True).""" + try: + from hermes_cli.config import load_config + from utils import is_truthy_value + + voice_cfg = load_config().get("voice", {}) + if isinstance(voice_cfg, dict): + return is_truthy_value( + voice_cfg.get("thinking_sound", True), default=True + ) + except Exception: + pass + return True + + +def _synth_thinking_blip(np, frequency: float) -> "Any": + """One soft 'blub': short sine with a gentle downward pitch glide and a + smooth attack/decay envelope (no clicks), low-volume.""" + duration = 0.16 + n = int(SAMPLE_RATE * duration) + t = np.linspace(0, duration, n, endpoint=False) + # Downward glide (water-drop feel): freq → 0.72*freq over the blip. + glide = np.linspace(1.0, 0.72, n) + phase = 2 * np.pi * np.cumsum(frequency * glide) / SAMPLE_RATE + tone = np.sin(phase) + # Soften harmonics (cheap low-pass feel): add a quieter octave-down sine. + tone = 0.8 * tone + 0.2 * np.sin(phase / 2.0) + # Envelope: quick-but-smooth attack, long exponential-ish decay. + attack = int(0.02 * SAMPLE_RATE) + env = np.ones(n) + env[:attack] = np.linspace(0.0, 1.0, attack) + env *= np.exp(-t * 14.0) + volume = _get_beep_volume() * 0.5 # deliberately quieter than the beeps + return (tone * env * volume * 32767).astype(np.int16) + + +def _thinking_sound_loop(stop: threading.Event, should_play) -> None: + """Daemon loop: play alternating-pitch blips every ~0.8-1.2s until *stop*. + + Skips a blip (without stopping) whenever *should_play* returns False — + e.g. TTS audio started flowing or the mic re-armed. macOS: sounddevice + output is TCC-gated, and per-second afplay subprocess churn is worse + than silence, so the loop exits immediately there. + """ + if not _sounddevice_output_allowed(): + return + try: + sd, np = _import_audio() + except (ImportError, OSError): + return + + import random + + pitches = (392.0, 329.6) # G4 / E4 — calm, low, alternating + blips = [_synth_thinking_blip(np, p) for p in pitches] + i = 0 + while not stop.is_set(): + try: + if should_play is None or should_play(): + blip = blips[i % len(blips)] + sd.play(blip, samplerate=SAMPLE_RATE) + stop.wait(len(blip) / SAMPLE_RATE + 0.02) + sd.stop() + i += 1 + except Exception as e: + logger.debug("Thinking sound blip failed: %s", e) + return + stop.wait(0.8 + random.random() * 0.4) + + +def start_thinking_sound(should_play=None) -> bool: + """Start the ambient thinking sound (idempotent). + + *should_play* is polled before each blip; return False to skip while + speech audio flows or the mic is capturing. Returns True when the loop + was started (or already running), False when disabled/unavailable. + """ + global _thinking_stop + if not thinking_sound_enabled(): + return False + with _thinking_lock: + if _thinking_stop is not None and not _thinking_stop.is_set(): + return True # already running + stop = threading.Event() + _thinking_stop = stop + threading.Thread( + target=_thinking_sound_loop, + args=(stop, should_play), + daemon=True, + name="voice-thinking-sound", + ).start() + return True + + +def stop_thinking_sound() -> None: + """Stop the ambient thinking sound instantly (idempotent).""" + global _thinking_stop + with _thinking_lock: + stop, _thinking_stop = _thinking_stop, None + if stop is not None: + stop.set() + + # ============================================================================ # Termux Audio Recorder # ============================================================================ @@ -1161,6 +1310,21 @@ def is_voice_stop_phrase(transcript: str, stop_phrases: Optional[tuple] = None) return cleaned in stop_phrases +def voice_stop_hint() -> str: + """One-line 'Say "stop" to end the voice chat.' hint for voice-mode start. + + Sources the phrase from ``voice.stop_phrases`` (first entry) so a custom + phrase renders correctly; returns "" when stop phrases are disabled + (``stop_phrases: []``) so surfaces show no hint at all. Every surface + that announces voice-mode start (CLI /voice on, TUI, desktop) uses this + one owner instead of hardcoding the wording. + """ + phrases = _load_voice_stop_phrases() + if not phrases: + return "" + return f'Say "{phrases[0]}" to end the voice chat.' + + # ============================================================================ # STT dispatch # ============================================================================ @@ -1398,6 +1562,16 @@ def play_audio_file(file_path: str) -> bool: Returns: ``True`` if playback succeeded, ``False`` otherwise. """ + # Ref-count real speaker output for the whole call so the thinking-sound + # loop (and any other ambient cue) knows audio is flowing right now. + mark_audio_output_active(True) + try: + return _play_audio_file_impl(file_path) + finally: + mark_audio_output_active(False) + + +def _play_audio_file_impl(file_path: str) -> bool: global _active_playback if not os.path.isfile(file_path): @@ -1599,9 +1773,18 @@ def listen_for_speech( trip_blocks = max(1, sustained_ms // 30) endpoint_blocks = max(1, endpoint_silence_ms // 30) max_blocks = max(1, max_utterance_ms // 30) - floor_samples: List[float] = [] + + # Rolling floor window: continuously tracks TTS speaker-bleed volume + # throughout playback, not just the first calibration_ms. This is the + # key fix for false barge-in — a one-shot calibration freezes a floor + # from the opening TTS passage, but later louder passages exceed the + # stale floor and false-trigger. The rolling window keeps the floor + # current so only genuinely louder-than-playback speech trips the VAD. + floor_window: "deque[float]" = deque(maxlen=max(calib_blocks, 100)) # ~3s rolling pre_roll: deque = deque(maxlen=max(1, pre_roll_ms // 30)) consecutive = 0 + min_floor = 0.0 # baseline from initial calibration; floor never drops below this + block_idx = 0 # block counter for diagnostic logging try: with sd.InputStream(samplerate=SAMPLE_RATE, channels=1, dtype="int16", blocksize=block) as stream: @@ -1610,15 +1793,79 @@ def listen_for_speech( rms = float(np.sqrt(np.mean(data.astype(np.float64) ** 2))) if capture: pre_roll.append(data.copy()) - if len(floor_samples) < calib_blocks: - floor_samples.append(rms) + block_idx += 1 + + # Wait for at least calib_blocks before evaluating. During + # the initial warmup we always feed the window so calibration + # has data to work with. + if len(floor_window) < calib_blocks: + floor_window.append(rms) continue - trigger = max(float(threshold or SILENCE_RMS_THRESHOLD * 2), float(np.median(floor_samples)) * 3.5) + + # Lock a minimum floor from the initial calibration samples. + # During inter-sentence pauses the rolling window can flush + # with near-silence, collapsing the 90th-percentile floor + # toward zero and false-triggering on the next rising + # sentence. min_floor keeps the trigger from ever dropping + # below the baseline TTS playback level established during + # the initial calibration_ms window. + # + # If the grace period ended during an inter-sentence gap the + # calibration samples near-silence. Locking a near-zero + # floor sets the trigger so low that TTS blocks exceed it, + # are excluded from the rolling window (rms >= trigger), and + # the floor freezes — guaranteeing a false trigger the moment + # TTS resumes. Clamp min_floor to SILENCE_RMS_THRESHOLD * 2 + # (400 RMS) so the 8x multiplier yields a trigger of at least + # (500-2000 RMS) stays below it and feeds the rolling window, + # while genuine speech (3000-8000 RMS) can still trip it. + if min_floor == 0.0 and len(floor_window) >= calib_blocks: + _pct90 = float(np.percentile(list(floor_window), 90)) + min_floor = max(_pct90, SILENCE_RMS_THRESHOLD * 2) + else: + _pct90 = float(np.percentile(list(floor_window), 90)) + + # Use the 90th percentile of the ROLLING window for the + # noise floor so the trigger reflects the loudest parts of + # recent playback — not a frozen snapshot from TTS onset. + _floor = max(_pct90, min_floor) + # 8.0x multiplier: TTS speaker bleed has wide + # volume variation between sentences and within sentences. + # At 5x, louder TTS passages exceed the trigger, get + # excluded from the floor window, and create a low-stale + # floor that false-triggers on the next loud passage. + # 8x gives enough headroom for TTS dynamics to stay below + # the trigger and get absorbed into the rolling floor. + trigger = max(float(threshold or SILENCE_RMS_THRESHOLD * 2), _floor * 8.0) + # Ceiling: never let the trigger exceed 4000 RMS, otherwise + # a very loud TTS passage would push the trigger so high + # that genuine speech (which is typically 3000–8000 RMS) + # couldn't trip it. + trigger = min(trigger, 4000.0) + + # Only feed the floor window with blocks that are NOT above + # the current trigger — speech blocks would inflate the floor + # and make the trigger unreachable. + if rms < trigger: + floor_window.append(rms) + consecutive = consecutive + 1 if rms >= trigger else 0 + if consecutive > 0: + logger.debug( + "VAD above-trigger: block=%d rms=%.0f floor=%.0f trigger=%.0f " + "consec=%d/%d min_floor=%.0f window_len=%d", + block_idx, rms, _floor, trigger, consecutive, + trip_blocks, min_floor, len(floor_window), + ) if consecutive < trip_blocks: continue # Tripped — the user is talking over playback. + logger.info( + "VAD TRIPPED: block=%d rms=%.0f floor=%.0f trigger=%.0f " + "consec=%d min_floor=%.0f — cutting TTS playback", + block_idx, rms, _floor, trigger, consecutive, min_floor, + ) if on_trigger: try: on_trigger() diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 18cb5f35e302..2e0f065bc5d0 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -11995,6 +11995,7 @@ def run(): goal_followup = None # set by the post-turn goal hook below result = None # turn outcome; read after the finally for leftover /steer tts_queue = None # streaming-TTS feed for this turn (voice mode) + thinking_started = False # ambient thinking sound armed for this turn one_turn_restore = session.pop("one_turn_model_restore", None) # True once a failed turn's snapshot was retained for resume replay — # tells the finally below to skip the normal inflight clear. @@ -12143,6 +12144,36 @@ def run(): # consume the latch below. tts_queue = _tts_stream_begin() + # Ambient "thinking" sound (voice mode only): calm bubble blips + # while the agent works with no audio flowing, so long + # thinking/tool stretches don't read as a dead session. Per-blip + # gate skips while real TTS audio flows or the mic is capturing; + # stopped in the finally the instant the turn ends. + # voice.thinking_sound config-gates it; macOS TCC handled inside. + thinking_started = False + if _voice_mode_enabled(): + try: + from tools.voice_mode import ( + is_audio_output_active, + start_thinking_sound, + ) + + def _thinking_should_play() -> bool: + if is_audio_output_active(): + return False + try: + from hermes_cli.voice import is_continuous_active + + return not is_continuous_active() + except Exception: + return True + + thinking_started = start_thinking_sound( + should_play=_thinking_should_play + ) + except Exception: + thinking_started = False + # Barged mid-speech? Tell the model (API-message note, same # enrichment channel as attached images) so it can react # ("rude!") instead of being oblivious to its own interruption. @@ -12509,11 +12540,11 @@ def _interim_assistant_cb(text: str, *, already_streamed: bool = False) -> None: and _voice_tts_enabled() ): try: - from hermes_cli.voice import speak_text - spoken = raw + # Barge-aware: spoken interruptions must cut this + # fallback playback too, not just the streaming path. threading.Thread( - target=speak_text, args=(spoken,), daemon=True + target=_speak_text_with_barge, args=(spoken,), daemon=True ).start() except ImportError: logger.warning("voice TTS skipped: hermes_cli.voice unavailable") @@ -12551,6 +12582,15 @@ def _interim_assistant_cb(text: str, *, already_streamed: bool = False) -> None: ) _emit("error", sid, {"message": str(e)}) finally: + if thinking_started: + # Kill the ambient thinking sound the moment the turn ends — + # error and success paths both land here. + try: + from tools.voice_mode import stop_thinking_sound + + stop_thinking_sound() + except Exception: + pass if tts_queue is not None: tts_queue.put(None) # end-of-text sentinel — flush + finish speaking if one_turn_restore: @@ -17701,6 +17741,21 @@ def _voice_tts_enabled() -> bool: return os.environ.get("HERMES_VOICE_TTS", "").strip() == "1" +def _any_session_running() -> bool: + """True while any session's agent turn is in flight. + + Registered as the voice busy-probe (``hermes_cli.voice.set_voice_busy_probe``) + so silent capture cycles during a long agent turn don't count toward the + no-speech limit — the user is correctly quiet while the agent works. + Voice is process-global (one microphone), so any running session holds. + """ + try: + with _sessions_lock: + return any(s.get("running") for s in _sessions.values()) + except Exception: + return False + + # ── Streaming TTS (one active pipeline per process — one speaker) ────────── # Token deltas from the running turn feed a sentence-buffering consumer # (tools.tts_tool.stream_tts_to_speaker) so speech starts on the first @@ -17757,6 +17812,12 @@ def _tts_stream_stop(user_barge: bool = True) -> None: if state is None: return if user_barge and not state["done"].is_set(): + import traceback as _tb + logger.debug( + "TTS CUT: _tts_stream_stop(user_barge=True) — new turn or " + "interrupt cutting in-flight TTS\n%s", + "".join(_tb.format_stack()), + ) from tools.tts_streaming import mark_speech_interrupted mark_speech_interrupted() @@ -17782,10 +17843,28 @@ def _tts_stream_barge_in_monitor(stop: threading.Event, done: threading.Event) - from tools.tts_streaming import mark_speech_interrupted from tools.voice_mode import listen_for_speech, stop_playback, transcribe_recording + # Grace period: wait briefly before opening the mic so the + # first TTS sentence is already playing and the VAD calibration + # samples the actual playback level (not silence). This + # prevents speaker bleed from falsely triggering barge-in + # at the start of playback. Mirrors the CLI path in cli.py + # _voice_barge_in_monitor. + _grace_s = float(_voice_cfg_dict().get("barge_in_grace_seconds", 2.0)) + if _grace_s > 0: + stop.wait(timeout=_grace_s) + if stop.is_set() or done.is_set(): + return + barged = threading.Event() def _cut_playback(): if not done.is_set(): + import traceback as _tb + logger.debug( + "TTS CUT: gateway barge-in _cut_playback fired (VAD trip) — " + "stop.set() + stop_playback()\n%s", + "".join(_tb.format_stack()), + ) barged.set() mark_speech_interrupted() stop.set() @@ -17796,6 +17875,8 @@ def _cut_playback(): lambda: stop.is_set() or done.is_set(), capture=True, on_trigger=_cut_playback, + sustained_ms=1000, + calibration_ms=800, ) if not (wav_path and barged.is_set()): return @@ -17837,6 +17918,39 @@ def _cut_playback(): logger.debug("TTS barge-in monitor failed: %s", e) +def _speak_text_with_barge(text: str) -> None: + """Speak *text* via hermes_cli.voice.speak_text with spoken barge-in. + + The streaming-TTS turn pipeline arms ``_tts_stream_barge_in_monitor``; + the fallback whole-reply path (streaming couldn't start) and the + ``voice.tts`` RPC previously called ``speak_text`` bare — speech over + those paths was UNINTERRUPTIBLE by voice. Run the same monitor beside + the speak thread: it cuts playback (``stop_playback`` kills the file + player; the stop event drains a streaming dispatch inside speak_text), + captures the interruption, and emits ``voice.transcript`` / + the stop-phrase signal exactly like the streaming path. + """ + from hermes_cli.voice import speak_text + + stop = threading.Event() + done = threading.Event() + + def _speak(): + try: + speak_text(text, stop) + except TypeError: + # Older wrapper without the stop_event parameter. + speak_text(text) + finally: + done.set() + + threading.Thread(target=_speak, daemon=True).start() + if _voice_mode_enabled() and _voice_cfg_dict().get("barge_in", True): + threading.Thread( + target=_tts_stream_barge_in_monitor, args=(stop, done), daemon=True + ).start() + + def _voice_cfg_dict() -> dict: """Shape-safe accessor for the ``voice:`` block in config.yaml. @@ -18246,6 +18360,18 @@ def _(rid, params: dict) -> dict: # persisted stale toggle. os.environ["HERMES_VOICE"] = "1" if enabled else "0" + stop_hint = "" + if enabled: + # Spoken-stop hint for the client to render on voice-mode start. + # Sourced from voice.stop_phrases (custom phrases render + # correctly); empty when the feature is disabled. + try: + from tools.voice_mode import voice_stop_hint + + stop_hint = voice_stop_hint() + except Exception: + stop_hint = "" + if not enabled: # Disabling the mode must tear the continuous loop down; the # loop holds the microphone and would otherwise keep running. @@ -18269,6 +18395,7 @@ def _(rid, params: dict) -> dict: "enabled": enabled, "record_key": _voice_record_key(), "tts": _voice_tts_enabled(), + "stop_hint": stop_hint, }, ) @@ -18328,6 +18455,18 @@ def _(rid, params: dict) -> dict: from hermes_cli.voice import start_continuous + # Register the agent-busy probe so the shared voice wrapper can + # hold the no-speech counter during long agent turns (item: + # silence must not end the chat while the agent works). Safe to + # re-register on every start; older wrappers without the setter + # are tolerated. + try: + from hermes_cli.voice import set_voice_busy_probe + + set_voice_busy_probe(_any_session_running) + except Exception: + pass + # Shape-safe lookups: malformed ``voice:`` YAML (bool/scalar/list) # must not crash /voice with a 5025 — fall back to VAD defaults. # @@ -18444,9 +18583,13 @@ def _(rid, params: dict) -> dict: if not text: return _err(rid, 4020, "text required") try: - from hermes_cli.voice import speak_text + # Import check up front so a missing voice module still returns the + # documented 5026 instead of failing silently in the thread. + import hermes_cli.voice # noqa: F401 - threading.Thread(target=speak_text, args=(text,), daemon=True).start() + threading.Thread( + target=_speak_text_with_barge, args=(text,), daemon=True + ).start() return _ok(rid, {"status": "speaking"}) except ImportError: return _err(rid, 5026, "voice module not available") diff --git a/ui-tui/src/app/slash/commands/session.ts b/ui-tui/src/app/slash/commands/session.ts index e2a841c6fc20..3fc60e8564af 100644 --- a/ui-tui/src/app/slash/commands/session.ts +++ b/ui-tui/src/app/slash/commands/session.ts @@ -380,6 +380,14 @@ export const sessionCommands: SlashCommand[] = [ const tts = r.tts ? ' (TTS enabled)' : '' ctx.transcript.sys(`Voice mode enabled${tts}`) ctx.transcript.sys(` ${recordKeyLabel} to start/stop recording`) + + // Spoken-stop hint — backend-sourced from voice.stop_phrases so a + // custom phrase renders correctly; absent/empty means the feature + // is disabled (stop_phrases: []) and no hint is shown. + if (r.stop_hint) { + ctx.transcript.sys(` ${r.stop_hint}`) + } + ctx.transcript.sys(' /voice tts to toggle speech output') ctx.transcript.sys(' /voice off to disable voice mode') } else { diff --git a/ui-tui/src/gatewayTypes.ts b/ui-tui/src/gatewayTypes.ts index 607da742594f..3066b83848fb 100644 --- a/ui-tui/src/gatewayTypes.ts +++ b/ui-tui/src/gatewayTypes.ts @@ -394,6 +394,7 @@ export interface VoiceToggleResponse { details?: string enabled?: boolean record_key?: string + stop_hint?: string stt_available?: boolean tts?: boolean }