Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 78 additions & 4 deletions apps/desktop/src/app/chat/composer/controls.tsx
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
import { Button } from '@/components/ui/button'
import { Codicon } from '@/components/ui/codicon'
import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from '@/components/ui/dropdown-menu'
import { Tip } from '@/components/ui/tooltip'
import { useI18n } from '@/i18n'
import { triggerHaptic } from '@/lib/haptics'
import { AudioLines, Layers3, Loader2, Square, SteeringWheel } from '@/lib/icons'
import { AudioLines, Layers3, Loader2, Mic, Square, SteeringWheel } from '@/lib/icons'
import { cn } from '@/lib/utils'
import { useMemo } from 'react'

import type { MicrophoneDevice } from './hooks/use-mic-device'
import type { ConversationStatus } from './hooks/use-voice-conversation'
import type { ChatBarState, VoiceStatus } from './types'

Expand Down Expand Up @@ -43,10 +46,13 @@ export function ComposerControls({
conversation,
disabled,
hasComposerPayload,
selectedVoiceDeviceId,
state,
voiceDevices,
voiceStatus,
onDictate,
onSteer
onSteer,
onChangeVoiceDevice
}: {
busy: boolean
busyAction: 'queue' | 'stop'
Expand All @@ -55,10 +61,13 @@ export function ComposerControls({
conversation: ConversationProps
disabled: boolean
hasComposerPayload: boolean
selectedVoiceDeviceId?: string | null
state: ChatBarState
voiceDevices?: MicrophoneDevice[]
voiceStatus: VoiceStatus
onDictate: () => void
onSteer: () => void
onChangeVoiceDevice?: (deviceId: string | null) => void
}) {
const { t } = useI18n()
const c = t.composer
Expand All @@ -69,9 +78,24 @@ export function ComposerControls({

const showVoicePrimary = !busy && !hasComposerPayload

const selectedLabel = useMemo(() => {
const match = voiceDevices?.find(device => device.deviceId === selectedVoiceDeviceId)

return match?.label ?? ''
}, [selectedVoiceDeviceId, voiceDevices])

return (
<div className="ml-auto flex shrink-0 items-center gap-(--composer-control-gap)">
<DictationButton disabled={disabled} onToggle={onDictate} state={state.voice} status={voiceStatus} />
{(selectedLabel || voiceDevices?.length) && (
<MicDeviceMenu
devices={voiceDevices ?? []}
disabled={disabled}
label={selectedLabel}
onChange={onChangeVoiceDevice}
selectedDeviceId={selectedVoiceDeviceId}
/>
)}
{canSteer && (
<Tip label={c.steer}>
<Button
Expand Down Expand Up @@ -210,9 +234,9 @@ function ConversationPill({
}

function ConversationIndicator({
level,
listening,
speaking
speaking,
level
}: {
level: number
listening: boolean
Expand All @@ -236,6 +260,56 @@ function ConversationIndicator({
)
}

function MicDeviceMenu({
devices,
disabled,
label,
onChange,
selectedDeviceId
}: {
devices: MicrophoneDevice[]
disabled: boolean
label: string
onChange?: (deviceId: string | null) => void
selectedDeviceId?: string | null
}) {
const value = selectedDeviceId ?? ''

return (
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button className={cn(GHOST_ICON_BTN, 'gap-1 px-1.5 text-[0.8rem]')} disabled={disabled} size="icon" type="button" variant="ghost">
<Mic size={14} />
<MicLabel label={label} />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end" className="max-h-64">
{devices.map(device => (
<DropdownMenuItem
key={device.deviceId}
onSelect={() => onChange?.(device.deviceId === value ? null : device.deviceId)}
>
<span className="truncate">{device.label}</span>
{device.deviceId === value && <span className="ml-auto text-xs text-(--ui-accent)">✓</span>}
</DropdownMenuItem>
))}
</DropdownMenuContent>
</DropdownMenu>
)
}

function MicLabel({ label }: { label: string }) {
if (!label) {
return <Mic size={14} />
}

return (
<span className="truncate text-xs">
{label}
</span>
)
}

function DictationButton({
disabled,
state,
Expand Down
122 changes: 122 additions & 0 deletions apps/desktop/src/app/chat/composer/hooks/use-mic-device.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
import { useEffect, useRef, useState } from 'react'

const STORAGE_KEY = 'hermes-voice-selected-device-id'

export interface MicrophoneDevice {
deviceId: string
label: string
}

export function useMicDevice() {
const [selectedDeviceId, setSelectedDeviceId] = useState<string | null>(() => {
if (typeof window === 'undefined') {
return null
}

return window.localStorage.getItem(STORAGE_KEY)
})
const [devices, setDevices] = useState<MicrophoneDevice[]>([])
const [pendingDeviceId, setPendingDeviceId] = useState<string | null>(null)
const initializedRef = useRef(false)

const refreshDevices = async () => {
if (typeof navigator === 'undefined' || !navigator.mediaDevices?.enumerateDevices) {
return
}

try {
const all = await navigator.mediaDevices.enumerateDevices()
const inputs = all
.filter((device): device is MediaDeviceInfo & { deviceId: string } => device.kind === 'audioinput')
.map(device => ({
deviceId: device.deviceId,
label: device.label || `Microphone ${device.deviceId.slice(0, 6)}`
}))

setDevices(inputs)
} catch {
// enumeration is best-effort
}
}

const chooseDevice = async (deviceId: string | null) => {
if (!deviceId) {
setPendingDeviceId(null)
setSelectedDeviceId(null)
if (typeof window !== 'undefined') {
window.localStorage.removeItem(STORAGE_KEY)
}
return
}

try {
await navigator.mediaDevices.getUserMedia({
audio: { deviceId: { exact: deviceId } }
})
} catch {
// keep previous choice if probing fails
}

setPendingDeviceId(deviceId)
setSelectedDeviceId(deviceId)
if (typeof window !== 'undefined') {
window.localStorage.setItem(STORAGE_KEY, deviceId)
}
}

useEffect(() => {
if (typeof navigator === 'undefined' || !navigator.mediaDevices?.getUserMedia) {
return
}

let cancelled = false

async function prime() {
try {
await navigator.mediaDevices.getUserMedia({ audio: true })
} catch {
// non-fatal: labels may stay empty until permission is granted
}

if (cancelled) {
return
}

await refreshDevices()

if (cancelled) {
return
}

const current = selectedDeviceId
if (current && !devices.some(device => device.deviceId === current)) {
await chooseDevice(devices[0]?.deviceId ?? null)
}

initializedRef.current = true
}

prime()

return () => {
cancelled = true
}
}, [])

const clear = () => {
setPendingDeviceId(null)
setSelectedDeviceId(null)
if (typeof window !== 'undefined') {
window.localStorage.removeItem(STORAGE_KEY)
}
}

return {
clear,
chooseDevice,
devices,
pendingDeviceId,
refreshDevices,
selectedDeviceId
}
}
12 changes: 9 additions & 3 deletions apps/desktop/src/app/chat/composer/hooks/use-mic-recorder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { useEffect, useRef, useState } from 'react'
type BrowserAudioContext = typeof AudioContext

export interface MicRecorderOptions {
deviceId?: string
onLevel?: (level: number) => void
onError?: (error: Error) => void
onSilence?: () => void
Expand Down Expand Up @@ -180,9 +181,14 @@ export function useMicRecorder(copy: MicRecorderErrorCopy): { handle: MicRecorde
let stream: MediaStream

try {
stream = await navigator.mediaDevices.getUserMedia({
audio: { echoCancellation: true, noiseSuppression: true }
})
const constraints: MediaStreamConstraints = {
audio: {
deviceId: options.deviceId ? { exact: options.deviceId } : undefined,
echoCancellation: true,
noiseSuppression: true
}
}
stream = await navigator.mediaDevices.getUserMedia(constraints)
} catch (error) {
throw micError(error, copy)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ interface PendingVoiceResponse {

interface VoiceConversationOptions {
busy: boolean
deviceId?: string
enabled: boolean
onFatalError?: () => void
onSubmit: (text: string) => Promise<void> | void
Expand All @@ -26,6 +27,7 @@ interface VoiceConversationOptions {

export function useVoiceConversation({
busy,
deviceId,
enabled,
onFatalError,
onSubmit,
Expand Down Expand Up @@ -198,8 +200,8 @@ export function useVoiceConversation({
}

try {
// VAD tuning mirrors `tools.voice_mode` defaults so the browser loop matches the CLI.
await handle.start({
deviceId,
silenceLevel: 0.075,
silenceMs: 1_250,
idleSilenceMs: 12_000,
Expand All @@ -218,7 +220,7 @@ export function useVoiceConversation({
setStatus('idle')
onFatalError?.()
}
}, [handle, handleTurn, onFatalError, voiceCopy.couldNotStartSession, voiceCopy.microphoneFailed])
}, [handle, deviceId, handleTurn, onFatalError, voiceCopy.couldNotStartSession, voiceCopy.microphoneFailed])

const speak = useCallback(async (text: string) => {
setStatus('speaking')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,15 @@ import type { VoiceActivityState, VoiceStatus } from '../types'
import { useMicRecorder } from './use-mic-recorder'

interface VoiceRecorderOptions {
deviceId?: string
maxRecordingSeconds: number
onTranscribeAudio?: (audio: Blob) => Promise<string>
focusInput: () => void
onTranscript: (text: string) => void
}

export function useVoiceRecorder({
deviceId,
maxRecordingSeconds,
onTranscribeAudio,
focusInput,
Expand Down
5 changes: 5 additions & 0 deletions apps/desktop/src/app/chat/composer/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -1361,9 +1361,14 @@ export function ChatBar({
}}
disabled={disabled}
hasComposerPayload={hasComposerPayload}
selectedVoiceDeviceId={state.voice.deviceId}
onDictate={dictate}
onSteer={steerDraft}
onChangeVoiceDevice={deviceId => {
setState(prev => prev satisfies ChatBarState ? { ...prev, voice: { ...prev.voice, deviceId: deviceId ?? '' } } as ChatBarState : prev)
}}
state={state}
voiceDevices={[]}
voiceStatus={voiceStatus}
/>
)
Expand Down
2 changes: 1 addition & 1 deletion apps/desktop/src/app/chat/composer/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ export interface ChatBarState {
quickModels?: QuickModelOption[]
}
tools: { enabled: boolean; label: string; suggestions?: ContextSuggestion[] }
voice: { enabled: boolean; active: boolean }
voice: { deviceId?: string; enabled: boolean; active: boolean }
}

export interface ChatBarProps {
Expand Down
14 changes: 12 additions & 2 deletions gateway/platforms/wecom.py
Original file line number Diff line number Diff line change
Expand Up @@ -270,13 +270,23 @@ async def disconnect(self) -> None:

async def _cleanup_ws(self) -> None:
"""Close the live websocket/session, if any."""
pending_tasks = getattr(self, "_pending_text_batch_tasks", None)
if pending_tasks is not None:
for task in pending_tasks.values():
if not task.done():
task.cancel()
pending_tasks.clear()
pending_events = getattr(self, "_pending_text_batches", None)
if pending_events is not None:
pending_events.clear()
if self._ws and not self._ws.closed:
await self._ws.close()
self._ws = None

if self._session and not self._session.closed:
session = getattr(self, "_session", None)
if session is not None and not session.closed:
await self._session.close()
self._session = None
self._session = None

async def _open_connection(self) -> None:
"""Open and authenticate a websocket connection."""
Expand Down
Loading