Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 6 additions & 8 deletions apps/desktop/src/app/chat/right-rail/preview-pane.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -179,10 +179,9 @@ describe('PreviewPane console state', () => {
)
})

await waitFor(
() => expect(rendered.container.querySelector('iframe')).not.toBeNull(),
{ container: rendered.container }
)
await waitFor(() => expect(rendered.container.querySelector('iframe')).not.toBeNull(), {
container: rendered.container
})
expect(rendered.container.querySelector('iframe')?.getAttribute('src')).toBe('blob:pdf-preview-1')
expect(readFileDataUrl).toHaveBeenCalledWith('/tmp/spec.pdf')
const blob = createObjectURL.mock.calls[0]?.[0]
Expand Down Expand Up @@ -328,10 +327,9 @@ describe('PreviewPane console state', () => {
$connection.set({ baseUrl: 'http://macmini', mode: 'remote', profile: 'macmini' } as never)
})

await waitFor(
() => expect(rendered.container.querySelector('iframe')).not.toBeNull(),
{ container: rendered.container }
)
await waitFor(() => expect(rendered.container.querySelector('iframe')).not.toBeNull(), {
container: rendered.container
})
expect(api).toHaveBeenCalledWith({
path: `/api/fs/read-data-url?path=${encodeURIComponent(filePath)}`,
profile: 'macmini'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -663,6 +663,7 @@ export function useSlashCommand(deps: SlashCommandDeps) {
client_capture: true,
surface: 'gui'
})

applyWakeStatus(current)

return current
Expand Down
35 changes: 28 additions & 7 deletions apps/desktop/src/lib/wake-client-capture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,7 @@
const TARGET_RATE = 16_000
const DEFAULT_FRAME = 1280 // 80 ms @ 16 kHz — matches tools/wake_word.py

export type WakeFeedRequester = (
method: string,
params?: Record<string, unknown>
) => Promise<unknown>
export type WakeFeedRequester = (method: string, params?: Record<string, unknown>) => Promise<unknown>

export interface ClientWakeCaptureOptions {
/** Samples per frame at 16 kHz (from wake.start response). */
Expand All @@ -31,59 +28,69 @@ function downsampleTo16k(input: Float32Array, inputRate: number): Float32Array {
if (inputRate === TARGET_RATE) {
return input
}

if (inputRate <= 0) {
return new Float32Array(0)
}

const ratio = inputRate / TARGET_RATE
const outLen = Math.max(1, Math.floor(input.length / ratio))
const out = new Float32Array(outLen)

for (let i = 0; i < outLen; i++) {
const start = Math.floor(i * ratio)
const end = Math.min(input.length, Math.floor((i + 1) * ratio))
let sum = 0
let count = 0

for (let j = start; j < end; j++) {
sum += input[j] ?? 0
count++
}

out[i] = count > 0 ? sum / count : 0
}

return out
}

function floatToInt16LE(input: Float32Array): ArrayBuffer {
const buf = new ArrayBuffer(input.length * 2)
const view = new DataView(buf)

for (let i = 0; i < input.length; i++) {
const s = Math.max(-1, Math.min(1, input[i] ?? 0))
view.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true)
}

return buf
}

function bytesToBase64(buf: ArrayBuffer): string {
const bytes = new Uint8Array(buf)
let binary = ''
const chunk = 0x8000

for (let i = 0; i < bytes.length; i += chunk) {
binary += String.fromCharCode(...bytes.subarray(i, i + chunk))
}

return btoa(binary)
}

/**
* Start streaming the default microphone to `wake.feed`.
* Returns a handle whose `stop()` ends tracks + audio graph.
*/
export async function startClientWakeCapture(
options: ClientWakeCaptureOptions
): Promise<ClientWakeCaptureHandle> {
export async function startClientWakeCapture(options: ClientWakeCaptureOptions): Promise<ClientWakeCaptureHandle> {
const frameLength = Math.max(160, Math.trunc(options.frameLength || DEFAULT_FRAME))
const audioWindow = window as Window & { webkitAudioContext?: typeof AudioContext }
const AudioContextCtor = window.AudioContext || audioWindow.webkitAudioContext

if (!AudioContextCtor) {
throw new Error('AudioContext unavailable for client wake capture')
}

if (!navigator.mediaDevices?.getUserMedia) {
throw new Error('getUserMedia unavailable for client wake capture')
}
Expand Down Expand Up @@ -123,13 +130,17 @@ export async function startClientWakeCapture(
if (draining) {
return
}

draining = true

try {
while (!stopped && queue.length > 0) {
const batch = queue.splice(0, MAX_FRAMES_PER_FEED)

if (batch.length === 0) {
break
}

try {
const merged = new Float32Array(batch.length * frameLength)
batch.forEach((frame, i) => merged.set(frame, i * frameLength))
Expand All @@ -145,6 +156,7 @@ export async function startClientWakeCapture(
}
} finally {
draining = false

if (!stopped && queue.length > 0) {
void drainQueue()
}
Expand All @@ -155,29 +167,35 @@ export async function startClientWakeCapture(
if (stopped) {
return
}

queue.push(frame)

while (queue.length > MAX_QUEUED_FRAMES) {
queue.shift()
}

void drainQueue()
}

processor.onaudioprocess = event => {
if (stopped) {
return
}

const input = event.inputBuffer.getChannelData(0)
const at16k = downsampleTo16k(input, context.sampleRate)
// Append to pending and emit full frames
const merged = new Float32Array(pending.length + at16k.length)
merged.set(pending, 0)
merged.set(at16k, pending.length)
let offset = 0

while (offset + frameLength <= merged.length) {
const frame = merged.subarray(offset, offset + frameLength)
offset += frameLength
enqueueFrame(new Float32Array(frame))
}

pending = merged.subarray(offset)
}

Expand All @@ -197,15 +215,18 @@ export async function startClientWakeCapture(
if (stopped) {
return
}

stopped = true
queue.length = 0

try {
processor.disconnect()
source.disconnect()
mute.disconnect()
} catch {
// ignore
}

void context.close().catch(() => undefined)
stream.getTracks().forEach(t => t.stop())
}
Expand Down
1 change: 1 addition & 0 deletions apps/desktop/src/store/preview.ts
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ function isPdfFileTarget(target: PreviewTarget): boolean {
* obsolete raw-binary path after Desktop itself has been upgraded. */
export function decodePreviewTabs(raw: string): PreviewTab[] {
const parsed = JSON.parse(raw) as unknown

const tabs = (Array.isArray(parsed) ? parsed.filter(isPreviewTab) : []).map(tab =>
isPdfFileTarget(tab.target) && tab.target.previewKind === 'binary'
? { ...tab, target: { ...tab.target, previewKind: 'pdf' as const } }
Expand Down
2 changes: 2 additions & 0 deletions apps/desktop/src/store/updates.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -772,6 +772,7 @@ describe('applyBackendUpdate recovery', () => {

it('restores the fixed action deadline after reconnecting', async () => {
updateHermesSpy.mockResolvedValue({ action_id: 'a'.repeat(32), ok: true, name: 'hermes-update', pid: 1 })

const running = {
exit_code: null,
lines: ['still running'],
Expand All @@ -783,6 +784,7 @@ describe('applyBackendUpdate recovery', () => {
for (let attempt = 0; attempt < 119; attempt += 1) {
getActionStatusSpy.mockResolvedValueOnce(running)
}

getActionStatusSpy.mockRejectedValueOnce(new Error('ECONNRESET')).mockResolvedValue(running)

const promise = applyBackendUpdate()
Expand Down
2 changes: 2 additions & 0 deletions apps/desktop/src/store/updates.ts
Original file line number Diff line number Diff line change
Expand Up @@ -569,9 +569,11 @@ async function runBackendUpdate(): Promise<DesktopUpdateApplyResult> {
try {
const previousStatus = $backendUpdateStatus.get()
const requestedTargetSha = previousStatus?.commits?.at(0)?.sha

const previousVersion = previousStatus?.targetSha?.startsWith('backend:')
? previousStatus.targetSha.slice('backend:'.length)
: undefined

const started = await updateHermes()

if (!started.ok) {
Expand Down
23 changes: 15 additions & 8 deletions apps/desktop/src/store/wake-word.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
import { atom } from 'nanostores'

import {
type ClientWakeCaptureHandle,
startClientWakeCapture
} from '@/lib/wake-client-capture'
import { type ClientWakeCaptureHandle, startClientWakeCapture } from '@/lib/wake-client-capture'
import { $gateway } from '@/store/gateway'

// "Hey Hermes" wake-word listener state for the composer toggle. The gateway is
Expand Down Expand Up @@ -48,13 +45,17 @@ export function stopClientCapture(): void {

async function maybeStartClientCapture(result: WakeStartResponse | null | undefined): Promise<void> {
stopClientCapture()

if (!result?.started) {
return
}

const mode = (result.capture || '').toLowerCase()

if (mode !== 'client' && mode !== 'remote' && mode !== 'external') {
return
}

try {
clientCapture = await startClientWakeCapture({
frameLength: result.frame_length,
Expand All @@ -65,12 +66,10 @@ async function maybeStartClientCapture(result: WakeStartResponse | null | undefi
$wakeWord.set({
...current,
listening: false,
notice:
error instanceof Error
? error.message
: 'Failed to open the client microphone for wake word',
notice: error instanceof Error ? error.message : 'Failed to open the client microphone for wake word',
pending: false
})

// Best-effort: release server lease if client mic failed.
try {
await gatewayRequester('wake.stop', {})
Expand Down Expand Up @@ -255,12 +254,14 @@ export async function armWakeWord(request: WakeRequester = gatewayRequester): Pr
client_capture: true,
surface: 'gui'
})

applyWakeStatus(status)

if (!status?.available || status.listening) {
// Armed already (e.g. another surface/restart) — reattach feeder if client.
if (status?.listening) {
const mode = (status.capture || '').toLowerCase()

if (mode === 'client' || mode === 'remote' || mode === 'external') {
void maybeStartClientCapture({
started: true,
Expand All @@ -269,13 +270,15 @@ export async function armWakeWord(request: WakeRequester = gatewayRequester): Pr
})
}
}

return
}

const result = await request<WakeStartResponse>('wake.start', {
surface: 'gui',
client_capture: true
})

applyWakeStartResult(result)
} catch {
// Older backends / transient failures — keep whatever we last knew.
Expand Down Expand Up @@ -350,6 +353,7 @@ export async function resumeWakeAfterVoice(request: WakeRequester = gatewayReque
client_capture: true,
surface: 'gui'
})

applyWakeStatus(status)

// Config says off (or the feature can't run) — off is the correct rest
Expand All @@ -362,20 +366,23 @@ export async function resumeWakeAfterVoice(request: WakeRequester = gatewayReque
// Server lease is still armed (e.g. wake.resume after voice).
// Client PCM was stopped on wake.detected — reattach if needed.
const mode = (status.capture || '').toLowerCase()

if (mode === 'client' || mode === 'remote' || mode === 'external') {
void maybeStartClientCapture({
started: true,
capture: 'client',
frame_length: status.frame_length ?? 1280
})
}

return
}

const started = await request<WakeStartResponse>('wake.start', {
surface: 'gui',
client_capture: true
})

applyWakeStartResult(started)

if (started?.started) {
Expand Down
Loading