From 9bf1faa41f7ff7c627f55ac0d8717173803956b2 Mon Sep 17 00:00:00 2001 From: lesseradmin Date: Tue, 25 Aug 2026 20:43:47 -0500 Subject: [PATCH 1/7] feat(web): add mobile browser voice turns --- web/src/components/ChatSidebar.tsx | 8 +- web/src/components/ChatVoiceControl.test.tsx | 176 ++++++++++++ web/src/components/ChatVoiceControl.tsx | 266 +++++++++++++++++++ web/src/lib/assistant-voice-output.test.ts | 32 +++ web/src/lib/assistant-voice-output.ts | 41 +++ web/src/lib/native-voice-bridge.test.ts | 41 +++ web/src/lib/native-voice-bridge.ts | 79 ++++++ web/src/lib/pty-voice-submit.test.ts | 35 +++ web/src/lib/pty-voice-submit.ts | 30 +++ web/src/pages/ChatPage.tsx | 38 +++ 10 files changed, 745 insertions(+), 1 deletion(-) create mode 100644 web/src/components/ChatVoiceControl.test.tsx create mode 100644 web/src/components/ChatVoiceControl.tsx create mode 100644 web/src/lib/assistant-voice-output.test.ts create mode 100644 web/src/lib/assistant-voice-output.ts create mode 100644 web/src/lib/native-voice-bridge.test.ts create mode 100644 web/src/lib/native-voice-bridge.ts create mode 100644 web/src/lib/pty-voice-submit.test.ts create mode 100644 web/src/lib/pty-voice-submit.ts diff --git a/web/src/components/ChatSidebar.tsx b/web/src/components/ChatSidebar.tsx index e7669921129c2..a79511e2c8c04 100644 --- a/web/src/components/ChatSidebar.tsx +++ b/web/src/components/ChatSidebar.tsx @@ -34,6 +34,7 @@ import { ModelReloadConfirm } from "@/components/ModelReloadConfirm"; import { ReasoningPicker } from "@/components/ReasoningPicker"; import { GatewayClient, type ConnectionState } from "@/lib/gatewayClient"; import { api, buildWsUrl } from "@/lib/api"; +import { assistantFinalText } from "@/lib/assistant-voice-output"; import { maybeReloadForLoopbackWsAuthFailure } from "@/lib/dashboard-auth-reload"; import { EVENTS_CONNECT_TIMEOUT_MS, @@ -92,6 +93,7 @@ interface ChatSidebarProps { className?: string; onDashboardNewSessionRequest?: () => void; onSessionTitleChange?: (title: string | null) => void; + onAssistantFinal?: (text: string) => void; } /** Build the ``session.create`` params for the sidecar session. @@ -115,6 +117,7 @@ export function ChatSidebar({ className, onDashboardNewSessionRequest, onSessionTitleChange, + onAssistantFinal, }: ChatSidebarProps) { // `version` bumps on reconnect; gw is derived so we never call setState // for it inside an effect (React 19's set-state-in-effect rule). The @@ -413,6 +416,9 @@ export function ChatSidebar({ } } else if (type === "dashboard.new_session_requested") { onDashboardNewSessionRequest?.(); + } else if (type === "message.complete") { + const text = assistantFinalText(type, payload); + if (text) onAssistantFinal?.(text); } }); }; @@ -429,7 +435,7 @@ export function ChatSidebar({ } ws?.close(); }; - }, [channel, onDashboardNewSessionRequest, onSessionTitleChange, version]); + }, [channel, onAssistantFinal, onDashboardNewSessionRequest, onSessionTitleChange, version]); // Seed the badge on mount and re-read it whenever the sockets are rebuilt // (a profile/channel switch bumps `version`). diff --git a/web/src/components/ChatVoiceControl.test.tsx b/web/src/components/ChatVoiceControl.test.tsx new file mode 100644 index 0000000000000..c26a2b5a80620 --- /dev/null +++ b/web/src/components/ChatVoiceControl.test.tsx @@ -0,0 +1,176 @@ +// @vitest-environment jsdom +import { act } from "react"; +import { createRoot } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { ChatVoiceControl } from "./ChatVoiceControl"; + +(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +interface FakeResult { 0: { transcript: string }; isFinal: boolean } + +class FakeRecognition { + static instances: FakeRecognition[] = []; + continuous = false; + interimResults = false; + lang = ""; + onresult: ((event: { resultIndex: number; results: FakeResult[] }) => void) | null = null; + onend: (() => void) | null = null; + onerror: ((event: { error: string }) => void) | null = null; + start = vi.fn(); + stop = vi.fn(); + abort = vi.fn(); + constructor() { FakeRecognition.instances.push(this); } + result(resultIndex: number, ...rows: Array<[string, boolean]>) { + this.onresult?.({ resultIndex, results: rows.map(([transcript, isFinal]) => ({ 0: { transcript }, isFinal })) }); + } +} + +describe("ChatVoiceControl browser speech input", () => { + let host: HTMLDivElement; + let root: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + FakeRecognition.instances = []; + vi.stubGlobal("SpeechRecognition", FakeRecognition); + host = document.createElement("div"); + document.body.append(host); + root = createRoot(host); + }); + + afterEach(async () => { + await act(async () => root.unmount()); + delete (window as Window & { hermesVoiceBridge?: unknown }).hermesVoiceBridge; + host.remove(); + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + async function render(submit = vi.fn(), connected = true) { + await act(async () => root.render()); + return { submit, surface: host.querySelector('[aria-label="Browser voice input"]') as HTMLButtonElement }; + } + + it("accumulates final results while replacing interim text without submitting", async () => { + const { submit, surface } = await render(); + await act(async () => surface.click()); + const recognition = FakeRecognition.instances[0]; + expect(recognition.continuous).toBe(true); + expect(recognition.interimResults).toBe(true); + await act(async () => recognition.result(0, ["Call the ", true], ["cr", false])); + await act(async () => recognition.result(1, ["ignored", true], ["crew", false])); + expect(host.textContent).toContain("Call the crew"); + expect(host.textContent).not.toContain("ignored"); + expect(submit).not.toHaveBeenCalled(); + }); + + it("restarts Android premature onend with bounded exponential backoff and no submit", async () => { + const { submit, surface } = await render(); + await act(async () => surface.click()); + await act(async () => FakeRecognition.instances[0].result(0, ["Keep going", true])); + await act(async () => FakeRecognition.instances[0].onend?.()); + await act(async () => vi.advanceTimersByTime(249)); + expect(FakeRecognition.instances).toHaveLength(1); + await act(async () => vi.advanceTimersByTime(1)); + expect(FakeRecognition.instances).toHaveLength(2); + await act(async () => FakeRecognition.instances[1].onend?.()); + await act(async () => vi.advanceTimersByTime(499)); + expect(FakeRecognition.instances).toHaveLength(2); + await act(async () => vi.advanceTimersByTime(1)); + expect(FakeRecognition.instances).toHaveLength(3); + expect(host.textContent).toContain("Keep going"); + expect(submit).not.toHaveBeenCalled(); + }); + + it("stops retrying after the bounded restart budget", async () => { + const { surface } = await render(); + await act(async () => surface.click()); + for (let index = 0; index < 6; index += 1) { + await act(async () => FakeRecognition.instances.at(-1)?.onend?.()); + await act(async () => vi.runOnlyPendingTimers()); + } + await act(async () => FakeRecognition.instances.at(-1)?.onend?.()); + await act(async () => vi.runAllTimers()); + expect(FakeRecognition.instances).toHaveLength(7); + expect(host.textContent).toContain("kept stopping"); + }); + + it("does not restart after a fatal permission error", async () => { + const { surface } = await render(); + await act(async () => surface.click()); + const recognition = FakeRecognition.instances[0]; + await act(async () => recognition.onerror?.({ error: "not-allowed" })); + await act(async () => recognition.onend?.()); + await act(async () => vi.runAllTimers()); + expect(FakeRecognition.instances).toHaveLength(1); + expect(host.textContent).toContain("permission was denied"); + }); + + it("submits one normalized transcript and aborts recognition", async () => { + const { submit, surface } = await render(); + await act(async () => surface.click()); + await act(async () => FakeRecognition.instances[0].result(0, ["Call the ", true], [" crew", false])); + await act(async () => surface.click()); + await act(async () => surface.click()); + expect(submit).toHaveBeenCalledTimes(1); + expect(submit).toHaveBeenCalledWith("Call the crew"); + expect(FakeRecognition.instances[0].abort).toHaveBeenCalledOnce(); + }); + + it("does not submit empty audio and remains listening", async () => { + const { submit, surface } = await render(); + await act(async () => surface.click()); + await act(async () => surface.click()); + expect(submit).not.toHaveBeenCalled(); + expect(host.textContent).toContain("No speech heard"); + expect(host.textContent).toContain("TAP ANYWHERE TO SEND"); + }); + + it("cancel aborts capture, clears text, and prevents restart", async () => { + const { submit, surface } = await render(); + await act(async () => surface.click()); + const recognition = FakeRecognition.instances[0]; + await act(async () => recognition.result(0, ["do not send", true])); + const cancel = host.querySelector('[aria-label="Cancel browser voice input"]') as HTMLButtonElement; + await act(async () => cancel.click()); + await act(async () => recognition.onend?.()); + await act(async () => vi.runAllTimers()); + expect(recognition.abort).toHaveBeenCalledOnce(); + expect(submit).not.toHaveBeenCalled(); + expect(host.textContent).not.toContain("do not send"); + }); + + it("keeps the terminal text-entry fallback available", async () => { + const { surface } = await render(); + expect(surface.textContent).toContain("Gboard typed fallback remains below"); + }); + + it("refuses to start while chat is disconnected", async () => { + const { surface } = await render(vi.fn(), false); + await act(async () => surface.click()); + expect(FakeRecognition.instances).toHaveLength(0); + expect(host.textContent).toContain("not connected"); + }); + + it("uses the optional native bridge and accepts partial and final events", async () => { + let listener: ((event: MessageEvent) => void) | undefined; + const bridge = { + postMessage: vi.fn(), + addEventListener: vi.fn((_name: string, cb: (event: MessageEvent) => void) => { listener = cb; }), + removeEventListener: vi.fn(), + }; + Object.defineProperty(window, "hermesVoiceBridge", { configurable: true, value: bridge }); + const { submit, surface } = await render(); + await act(async () => surface.click()); + expect(FakeRecognition.instances).toHaveLength(0); + expect(bridge.postMessage).toHaveBeenCalledWith(JSON.stringify({ version: 1, command: "start" })); + await act(async () => listener?.({ data: JSON.stringify({ version: 1, event: "partial", transcript: "Call" }) } as MessageEvent)); + await act(async () => listener?.({ data: JSON.stringify({ version: 1, event: "final", transcript: "Call the crew" }) } as MessageEvent)); + expect(host.textContent).toContain("Call the crew"); + await act(async () => surface.click()); + expect(submit).toHaveBeenCalledOnce(); + expect(submit).toHaveBeenCalledWith("Call the crew"); + expect(bridge.postMessage).toHaveBeenCalledWith(JSON.stringify({ version: 1, command: "stop" })); + }); +}); diff --git a/web/src/components/ChatVoiceControl.tsx b/web/src/components/ChatVoiceControl.tsx new file mode 100644 index 0000000000000..5be2941579cdd --- /dev/null +++ b/web/src/components/ChatVoiceControl.tsx @@ -0,0 +1,266 @@ +import { Mic, X } from "lucide-react"; +import { useCallback, useEffect, useRef, useState } from "react"; + +import { Button } from "@nous-research/ui/ui/components/button"; +import { + connectNativeVoiceBridge, + type NativeVoiceBridgeConnection, + type NativeVoiceMessageTarget, +} from "@/lib/native-voice-bridge"; + +const RESTART_DELAY_MS = 250; +const MAX_RESTARTS = 6; + +type VoiceState = "idle" | "listening" | "sent" | "error"; + +interface SpeechRecognitionResultLike { + readonly isFinal: boolean; + readonly 0: { readonly transcript: string }; +} + +interface SpeechRecognitionEventLike { + readonly resultIndex: number; + readonly results: ArrayLike; +} + +interface SpeechRecognitionLike { + continuous: boolean; + interimResults: boolean; + lang: string; + onresult: ((event: SpeechRecognitionEventLike) => void) | null; + onend: (() => void) | null; + onerror: ((event: { error: string }) => void) | null; + start(): void; + abort(): void; +} + +type SpeechRecognitionConstructor = new () => SpeechRecognitionLike; + +declare global { + interface Window { + SpeechRecognition?: SpeechRecognitionConstructor; + webkitSpeechRecognition?: SpeechRecognitionConstructor; + hermesVoiceBridge?: NativeVoiceMessageTarget; + } +} + +interface ChatVoiceControlProps { + connected: boolean; + submit: (transcript: string) => void; + onBargeIn?: () => void; +} + +function recognitionConstructor(): SpeechRecognitionConstructor | undefined { + return window.SpeechRecognition ?? window.webkitSpeechRecognition; +} + +export function ChatVoiceControl({ connected, submit, onBargeIn }: ChatVoiceControlProps) { + const [state, setState] = useState("idle"); + const [finalTranscript, setFinalTranscript] = useState(""); + const [interimTranscript, setInterimTranscript] = useState(""); + const [error, setError] = useState(""); + const recognitionRef = useRef(null); + const listeningRef = useRef(false); + const fatalRef = useRef(false); + const restartCountRef = useRef(0); + const restartTimerRef = useRef(null); + const generationRef = useRef(0); + const nativeBridgeRef = useRef(null); + const finalRef = useRef(""); + const interimRef = useRef(""); + + const clearRestart = useCallback(() => { + if (restartTimerRef.current !== null) window.clearTimeout(restartTimerRef.current); + restartTimerRef.current = null; + }, []); + + const updateFinal = (value: string) => { + finalRef.current = value; + setFinalTranscript(value); + }; + + const updateInterim = (value: string) => { + interimRef.current = value; + setInterimTranscript(value); + }; + + function startRecognition(generation: number) { + if (!listeningRef.current || generation !== generationRef.current) return; + const Constructor = recognitionConstructor(); + if (!Constructor) { + listeningRef.current = false; + setError("Chrome speech recognition is unavailable. Use Gboard typed input below."); + setState("error"); + return; + } + + const recognition = new Constructor(); + recognition.continuous = true; + recognition.interimResults = true; + recognition.lang = "en-US"; + recognition.onresult = (event) => { + if (generation !== generationRef.current || !listeningRef.current) return; + let appendedFinal = ""; + let nextInterim = ""; + for (let index = event.resultIndex; index < event.results.length; index += 1) { + const result = event.results[index]; + const text = result?.[0]?.transcript ?? ""; + if (result?.isFinal) appendedFinal += text; + else nextInterim += text; + } + if (appendedFinal) updateFinal(`${finalRef.current}${appendedFinal}`); + updateInterim(nextInterim); + setError(""); + }; + recognition.onerror = (event) => { + if (generation !== generationRef.current) return; + const fatal = event.error === "not-allowed" || event.error === "service-not-allowed"; + if (fatal) { + fatalRef.current = true; + listeningRef.current = false; + setError(event.error === "not-allowed" + ? "Microphone permission was denied. Allow mic access in Chrome, then tap again." + : "Chrome speech recognition is not allowed on this device."); + setState("error"); + } else { + setError(`Speech recognition error: ${event.error}. Retrying…`); + } + }; + recognition.onend = () => { + if (generation !== generationRef.current || !listeningRef.current || fatalRef.current) return; + if (restartCountRef.current >= MAX_RESTARTS) { + listeningRef.current = false; + setError("Speech recognition kept stopping. Tap to resume or use Gboard typed input below."); + setState("error"); + return; + } + const delay = RESTART_DELAY_MS * 2 ** Math.min(restartCountRef.current, 3); + restartCountRef.current += 1; + restartTimerRef.current = window.setTimeout(() => startRecognition(generation), delay); + }; + recognitionRef.current = recognition; + try { + recognition.start(); + } catch { + if (generation !== generationRef.current) return; + listeningRef.current = false; + setError("Could not start Chrome speech recognition. Tap to try again."); + setState("error"); + } + } + + const begin = () => { + if (!connected) { + setError("Chat is not connected yet."); + setState("error"); + return; + } + onBargeIn?.(); + clearRestart(); + fatalRef.current = false; + restartCountRef.current = 0; + listeningRef.current = true; + const generation = generationRef.current + 1; + generationRef.current = generation; + setError(""); + setState("listening"); + if (nativeBridgeRef.current) nativeBridgeRef.current.command("start"); + else startRecognition(generation); + }; + + const commit = () => { + const transcript = `${finalRef.current} ${interimRef.current}`.replace(/\s+/g, " ").trim(); + if (!transcript) { + setError("No speech heard yet — keep talking, then tap anywhere to send."); + return; + } + listeningRef.current = false; + generationRef.current += 1; + clearRestart(); + recognitionRef.current?.abort(); + recognitionRef.current = null; + nativeBridgeRef.current?.command("stop"); + submit(transcript); + updateFinal(""); + updateInterim(""); + setError(""); + setState("sent"); + }; + + const cancel = useCallback(() => { + listeningRef.current = false; + generationRef.current += 1; + clearRestart(); + recognitionRef.current?.abort(); + recognitionRef.current = null; + nativeBridgeRef.current?.command("cancel"); + updateFinal(""); + updateInterim(""); + setError(""); + setState("idle"); + }, [clearRestart]); + + useEffect(() => { + const connection = connectNativeVoiceBridge(window.hermesVoiceBridge, (event) => { + if (event.event === "partial") { + updateInterim(event.transcript ?? ""); + } else if (event.event === "final") { + updateFinal(event.transcript ?? ""); + updateInterim(""); + } else if (event.event === "error") { + setError(event.message ?? "Native speech recognition failed."); + if (event.fatal) { + listeningRef.current = false; + setState("error"); + } + } else if (event.event === "listening") { + setState("listening"); + } + }); + nativeBridgeRef.current = connection; + connection?.command("check"); + return () => { + connection?.command("cancel"); + connection?.disconnect(); + if (nativeBridgeRef.current === connection) nativeBridgeRef.current = null; + }; + }, []); + + useEffect(() => cancel, [cancel]); + + const transcript = `${finalTranscript}${interimTranscript}`.trim(); + const headline = state === "listening" + ? (transcript || "LISTENING…") + : state === "sent" + ? "SENT" + : "TAP TO TALK"; + + return ( +
+ + + {error &&
{error}
} +
+ ); +} diff --git a/web/src/lib/assistant-voice-output.test.ts b/web/src/lib/assistant-voice-output.test.ts new file mode 100644 index 0000000000000..d984c9bbda25a --- /dev/null +++ b/web/src/lib/assistant-voice-output.test.ts @@ -0,0 +1,32 @@ +// @vitest-environment jsdom +import { describe, expect, it, vi } from "vitest"; + +import { assistantFinalText, speakAssistantFinal } from "./assistant-voice-output"; + +describe("assistant voice output", () => { + it("accepts only successful structured message.complete payloads", () => { + expect(assistantFinalText("message.complete", { status: "complete", text: "Done" })).toBe("Done"); + expect(assistantFinalText("message.complete", { status: "error", text: "secret failure" })).toBeNull(); + expect(assistantFinalText("message.delta", { text: "partial" })).toBeNull(); + expect(assistantFinalText("message.complete", { status: "complete", text: " " })).toBeNull(); + }); + + it("speaks through browser synthesis and settles once on end or cancellation", () => { + class FakeUtterance { + onend: (() => void) | null = null; + onerror: (() => void) | null = null; + readonly text: string; + constructor(text: string) { this.text = text; } + } + vi.stubGlobal("SpeechSynthesisUtterance", FakeUtterance); + const synth = { speak: vi.fn(), cancel: vi.fn() }; + const settled = vi.fn(); + const playback = speakAssistantFinal(synth, "Done", settled); + const utterance = synth.speak.mock.calls[0][0] as SpeechSynthesisUtterance; + expect(utterance.text).toBe("Done"); + utterance.onend?.({} as SpeechSynthesisEvent); + playback.cancel(); + expect(settled).toHaveBeenCalledOnce(); + expect(synth.cancel).toHaveBeenCalledOnce(); + }); +}); diff --git a/web/src/lib/assistant-voice-output.ts b/web/src/lib/assistant-voice-output.ts new file mode 100644 index 0000000000000..cb23fe69f1c7f --- /dev/null +++ b/web/src/lib/assistant-voice-output.ts @@ -0,0 +1,41 @@ +interface AssistantCompletePayload { + status?: unknown; + text?: unknown; +} + +export function assistantFinalText(type: string, payload: unknown): string | null { + if (type !== "message.complete" || !payload || typeof payload !== "object") return null; + const complete = payload as AssistantCompletePayload; + if (complete.status !== undefined && complete.status !== "complete") return null; + if (typeof complete.text !== "string") return null; + const text = complete.text.trim(); + return text || null; +} + +interface SpeechSynthesisLike { + speak(utterance: SpeechSynthesisUtterance): void; + cancel(): void; +} + +export function speakAssistantFinal( + synthesis: SpeechSynthesisLike, + text: string, + onSettled: () => void, +): { cancel(): void } { + const utterance = new SpeechSynthesisUtterance(text); + let settled = false; + const settle = () => { + if (settled) return; + settled = true; + onSettled(); + }; + utterance.onend = settle; + utterance.onerror = settle; + synthesis.speak(utterance); + return { + cancel() { + synthesis.cancel(); + settle(); + }, + }; +} diff --git a/web/src/lib/native-voice-bridge.test.ts b/web/src/lib/native-voice-bridge.test.ts new file mode 100644 index 0000000000000..dde405e3eef8f --- /dev/null +++ b/web/src/lib/native-voice-bridge.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it, vi } from "vitest"; + +import { connectNativeVoiceBridge } from "./native-voice-bridge"; + +describe("origin-scoped native voice bridge", () => { + it("sends only the declared command envelope and accepts declared transcript events", () => { + let listener: ((event: MessageEvent) => void) | undefined; + const target = { + postMessage: vi.fn(), + addEventListener: vi.fn((_name: string, cb: (event: MessageEvent) => void) => { listener = cb; }), + removeEventListener: vi.fn(), + }; + const events = vi.fn(); + const bridge = connectNativeVoiceBridge(target, events); + expect(bridge).not.toBeNull(); + bridge?.command("check"); + bridge?.command("start"); + expect(target.postMessage).toHaveBeenNthCalledWith(1, JSON.stringify({ version: 1, command: "check" })); + expect(target.postMessage).toHaveBeenNthCalledWith(2, JSON.stringify({ version: 1, command: "start" })); + + listener?.({ data: JSON.stringify({ version: 1, event: "partial", transcript: "hello" }) } as MessageEvent); + listener?.({ data: JSON.stringify({ version: 1, event: "unknown", transcript: "ignored" }) } as MessageEvent); + expect(events).toHaveBeenCalledOnce(); + expect(events).toHaveBeenCalledWith({ version: 1, event: "partial", transcript: "hello" }); + }); + + it("fails closed for missing or malformed bridge objects and events", () => { + expect(connectNativeVoiceBridge(undefined, vi.fn())).toBeNull(); + const events = vi.fn(); + let listener: ((event: MessageEvent) => void) | undefined; + const target = { + postMessage: vi.fn(), + addEventListener: vi.fn((_name: string, cb: (event: MessageEvent) => void) => { listener = cb; }), + removeEventListener: vi.fn(), + }; + connectNativeVoiceBridge(target, events); + listener?.({ data: "not json" } as MessageEvent); + listener?.({ data: JSON.stringify({ version: 2, event: "final", transcript: "no" }) } as MessageEvent); + expect(events).not.toHaveBeenCalled(); + }); +}); diff --git a/web/src/lib/native-voice-bridge.ts b/web/src/lib/native-voice-bridge.ts new file mode 100644 index 0000000000000..2e86efebe09a9 --- /dev/null +++ b/web/src/lib/native-voice-bridge.ts @@ -0,0 +1,79 @@ +export type NativeVoiceCommand = "check" | "start" | "stop" | "cancel"; +export type NativeVoiceEventName = + | "availability" + | "ready" + | "listening" + | "partial" + | "final" + | "error" + | "ended"; + +export interface NativeVoiceEvent { + version: 1; + event: NativeVoiceEventName; + transcript?: string; + available?: boolean; + message?: string; + fatal?: boolean; +} + +export interface NativeVoiceMessageTarget { + postMessage(message: string): void; + addEventListener(type: "message", listener: (event: MessageEvent) => void): void; + removeEventListener(type: "message", listener: (event: MessageEvent) => void): void; +} + +const EVENT_NAMES = new Set([ + "availability", "ready", "listening", "partial", "final", "error", "ended", +]); + +function parseNativeVoiceEvent(data: unknown): NativeVoiceEvent | null { + if (typeof data !== "string") return null; + try { + const value = JSON.parse(data) as Record; + if (value.version !== 1 || typeof value.event !== "string" || !EVENT_NAMES.has(value.event as NativeVoiceEventName)) { + return null; + } + if (value.transcript !== undefined && typeof value.transcript !== "string") return null; + if (value.available !== undefined && typeof value.available !== "boolean") return null; + if (value.message !== undefined && typeof value.message !== "string") return null; + if (value.fatal !== undefined && typeof value.fatal !== "boolean") return null; + return value as unknown as NativeVoiceEvent; + } catch { + return null; + } +} + +export interface NativeVoiceBridgeConnection { + command(command: NativeVoiceCommand): void; + disconnect(): void; +} + +/** + * Bind the optional AndroidX WebKit WebMessage listener object. + * + * The native host owns origin allow-listing when it injects this target. The + * web app neither assumes a specific Android recognizer nor receives provider + * credentials. Missing/malformed targets fail closed to ordinary Web Speech. + */ +export function connectNativeVoiceBridge( + target: NativeVoiceMessageTarget | undefined, + onEvent: (event: NativeVoiceEvent) => void, +): NativeVoiceBridgeConnection | null { + if (!target || typeof target.postMessage !== "function" || typeof target.addEventListener !== "function" || typeof target.removeEventListener !== "function") { + return null; + } + const listener = (message: MessageEvent) => { + const event = parseNativeVoiceEvent(message.data); + if (event) onEvent(event); + }; + target.addEventListener("message", listener); + return { + command(command) { + target.postMessage(JSON.stringify({ version: 1, command })); + }, + disconnect() { + target.removeEventListener("message", listener); + }, + }; +} diff --git a/web/src/lib/pty-voice-submit.test.ts b/web/src/lib/pty-voice-submit.test.ts new file mode 100644 index 0000000000000..e3cceeb4839c0 --- /dev/null +++ b/web/src/lib/pty-voice-submit.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it, vi } from "vitest"; + +import { submitVoiceTranscriptToPty } from "./pty-voice-submit"; + +describe("submitVoiceTranscriptToPty", () => { + it("sends transcript then exactly one Return in a separate PTY frame", () => { + vi.useFakeTimers(); + const socket = { readyState: WebSocket.OPEN, send: vi.fn() }; + const current = () => socket; + + expect(submitVoiceTranscriptToPty(current, "Call the crew")).toBe(true); + expect(socket.send).toHaveBeenCalledTimes(1); + expect(socket.send).toHaveBeenNthCalledWith(1, "Call the crew"); + vi.advanceTimersByTime(100); + expect(socket.send).toHaveBeenCalledTimes(2); + expect(socket.send).toHaveBeenNthCalledWith(2, "\r"); + vi.useRealTimers(); + }); + + it("does not send Return through a replacement or closed socket", () => { + vi.useFakeTimers(); + const first = { readyState: WebSocket.OPEN, send: vi.fn<(data: string) => void>() }; + const second = { readyState: WebSocket.OPEN, send: vi.fn<(data: string) => void>() }; + let active: { readyState: number; send(data: string): void } = first; + expect(submitVoiceTranscriptToPty(() => active, "Hello")).toBe(true); + active = second; + vi.advanceTimersByTime(100); + expect(first.send).toHaveBeenCalledOnce(); + expect(second.send).not.toHaveBeenCalled(); + + active = { readyState: WebSocket.CLOSED, send: vi.fn<(data: string) => void>() }; + expect(submitVoiceTranscriptToPty(() => active, "Nope")).toBe(false); + vi.useRealTimers(); + }); +}); diff --git a/web/src/lib/pty-voice-submit.ts b/web/src/lib/pty-voice-submit.ts new file mode 100644 index 0000000000000..25c13916c8983 --- /dev/null +++ b/web/src/lib/pty-voice-submit.ts @@ -0,0 +1,30 @@ +const PTY_TRANSCRIPT_RETURN_DELAY_MS = 100; + +interface PtySocket { + readonly readyState: number; + send(data: string): void; +} + +/** + * Submit one finalized browser transcript to Ink's composer. + * + * Return is deliberately a separate frame: a combined mobile WebSocket frame + * can be classified as a paste and remain in the composer. The captured socket + * must still be current before Return is sent, so a reconnect cannot submit the + * transcript into a replacement PTY session. + */ +export function submitVoiceTranscriptToPty( + currentSocket: () => PtySocket | null, + transcript: string, +): boolean { + const socket = currentSocket(); + if (!socket || socket.readyState !== WebSocket.OPEN) return false; + + socket.send(transcript); + globalThis.setTimeout(() => { + if (currentSocket() === socket && socket.readyState === WebSocket.OPEN) { + socket.send("\r"); + } + }, PTY_TRANSCRIPT_RETURN_DELAY_MS); + return true; +} diff --git a/web/src/pages/ChatPage.tsx b/web/src/pages/ChatPage.tsx index 244fa1680751a..c69875714cdff 100644 --- a/web/src/pages/ChatPage.tsx +++ b/web/src/pages/ChatPage.tsx @@ -32,9 +32,11 @@ import { useSearchParams } from "react-router"; import { ChatSidebar } from "@/components/ChatSidebar"; import { ChatSessionList } from "@/components/ChatSessionList"; +import { ChatVoiceControl } from "@/components/ChatVoiceControl"; import { usePageHeader } from "@/contexts/usePageHeader"; import { useI18n } from "@/i18n"; import { api } from "@/lib/api"; +import { speakAssistantFinal } from "@/lib/assistant-voice-output"; import { latchChatActivation } from "@/lib/chat-activation"; import { copyTextToClipboard } from "@/lib/clipboard"; import { normalizeSessionTitle } from "@/lib/chat-title"; @@ -66,6 +68,7 @@ import { resolvePtyKeyboardShortcut, sendPtyShortcutSequence, } from "@/lib/pty-keyboard-shortcuts"; +import { submitVoiceTranscriptToPty } from "@/lib/pty-voice-submit"; import { isViewportPinnedToBottom, parseResumeControlMessage, @@ -180,6 +183,8 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) { const termRef = useRef(null); const fitRef = useRef(null); const wsRef = useRef(null); + const voiceTurnRef = useRef(false); + const voicePlaybackRef = useRef<{ cancel(): void } | null>(null); const stickToBottomRef = useRef(true); // Exposed to the main metrics-sync effect so it can refit the terminal // the moment `isActive` flips back to true (display:none → display:flex @@ -506,6 +511,32 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) { termRef.current?.focus(); }; + const submitVoiceTranscript = useCallback((transcript: string) => { + if (!submitVoiceTranscriptToPty(() => wsRef.current, transcript)) { + setBanner("Chat disconnected before the voice transcript could be sent."); + return; + } + voiceTurnRef.current = true; + termRef.current?.focus(); + }, []); + + const cancelVoicePlayback = useCallback(() => { + voicePlaybackRef.current?.cancel(); + voicePlaybackRef.current = null; + }, []); + + const speakVoiceResponse = useCallback((text: string) => { + if (!voiceTurnRef.current || !("speechSynthesis" in window)) return; + voiceTurnRef.current = false; + cancelVoicePlayback(); + const playback = speakAssistantFinal(window.speechSynthesis, text, () => { + if (voicePlaybackRef.current === playback) voicePlaybackRef.current = null; + }); + voicePlaybackRef.current = playback; + }, [cancelVoicePlayback]); + + useEffect(() => cancelVoicePlayback, [cancelVoicePlayback]); + useEffect(() => { // Don't spawn the chat PTY (and the TUI/agent bootstrap it triggers) // until the chat tab has been activated. Prevents the persistently @@ -1789,6 +1820,7 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) { profile={scopedProfile} onDashboardNewSessionRequest={startFreshDashboardChat} onSessionTitleChange={handleSessionTitleChange} + onAssistantFinal={speakVoiceResponse} /> +
From 73476fccd2e12b415cab9fe2d63ceae5ae227ae0 Mon Sep 17 00:00:00 2001 From: lesseradmin Date: Tue, 25 Aug 2026 20:55:10 -0500 Subject: [PATCH 2/7] fix(web): keep browser voice on Web Speech --- docs/review-artifacts/791ca2b9.patch.gz | Bin 0 -> 30291 bytes docs/review-artifacts/s25-voice-lineage.md | 33 ++++++++ web/src/components/ChatVoiceControl.test.tsx | 21 ----- web/src/components/ChatVoiceControl.tsx | 38 +-------- web/src/lib/native-voice-bridge.test.ts | 41 ---------- web/src/lib/native-voice-bridge.ts | 79 ------------------- 6 files changed, 34 insertions(+), 178 deletions(-) create mode 100644 docs/review-artifacts/791ca2b9.patch.gz create mode 100644 docs/review-artifacts/s25-voice-lineage.md delete mode 100644 web/src/lib/native-voice-bridge.test.ts delete mode 100644 web/src/lib/native-voice-bridge.ts diff --git a/docs/review-artifacts/791ca2b9.patch.gz b/docs/review-artifacts/791ca2b9.patch.gz new file mode 100644 index 0000000000000000000000000000000000000000..5c933e98d0d4f6ad070213d65f99785044fd97e0 GIT binary patch literal 30291 zcmV(oK=HpHiwFP!000021Ks`in(9avFADyjPmyfQm7ug6Zwes^BfLl0nwCKZ1Tr9t zB;Y#_F;6gm+y^^PGC{hX+f`SeIWzZuv%4#e6tNVs%x{@kBAbS;J{L+Uo=a~_QZZjB zj05`9E{H zXC#?RCZA^Z_8{`spU;1dt5m)H?`KKAK(4rj|9rMh@mY}_@!33yBENr@bmLjox3VHW zJG#t^&$4#1t*0jhL7tvD!;&SAx6FU-MMD)$&U_x$$n`dkC+ zPCg?TSWhOMeBOGNXg)oE{rdIi`Cv7#EBKMpXDZoZvhYp?O!${+@c#mAIO&?GfkoB# zoV9pRtm~R(=*m|x(E4hbmj{(puJD|A`G>sha%zkgC>3mA>^s&r32uZqKg8K`l%!W_gg$U>%UTg%r#9rr5OsQc;H5UhyDRi6bwn|lgEUo9O zV8a|Zszjl+}gAcHujF(+zLyQXjF(l{MEI~*OEqQNQIv*pNzLr$@DT#Qru9izde zIGoCjJCO+LwwMJLN1l8=n{J#EW1}gW;~k+34Z}Gj#=^dZW;V($v+T0#C!3fc&Wb4; zcB5-Yl5DQz5?C%o(zO6=1e&gU$mZY(Jm&6i^Xafr+S-+L|H_a~#hRtsXlLHSh_hQCr=YWFtOzd+H)gqZ0H?)qmQU=DS@>yBqDVM z>w-;nzqQ)f%Cyj262!U0;oDvP$=^t~b)$3YwU-z*PmOwo?t0_Ts93>XcemSZsho(N zHj^wbUegf@Jvy1XHy&JNbJ1y+$%2L@E=6Ws4;l`7nNIJHNtMV*Kt`jqL1L?c1 zqLhnWxl)8va%ooJDrm#Xty9zDIv_d;D6_M;d z1Rq`K^tBADxxu^?YmRDdXi$t)Vn~?Y%xpZ}z;_NMLi5HV5$k*XC!dO13e8c3vutKh zB0L+)h3rMU?T^b+m_D$vjzbTzBr0AmM7v38jr5)CM-yX>neGpWm`MY6MhB))W2kL5 zjtfL0vuht(-N`IJoMYQbn=A`)KXkFw9Y@cV((1Gt76fsB3S_vbXC2o*VxuR&mGCfH zkxSZnwA|dL{vtEOeJ5ofg-WG7rZH3;3&s*>KMX1u(XpB_YF_DCO1H|G!Y$-aoe3WF_N}NDI{4=b zkO=Iq8--f_3Wvk7SvHa5re{>6mz!Db$`Mqr+UWtem(@#QAb)#@rOwSk)A6s>68fw%mTXm>Tg5~2sc7(NiS+Ia z$L<{EuJlew`pK_zX8f?47st zraG*4lT`!(yIEtYC6#^hcTo(0@}BC@I&(brks*0HB#eMhRY;+xi>Im^(3tM3mv{^7 z`l{W+mznX5tjDi6z%Fx(J^2KFZ_vfs0v@b_M4~~D1*np!-+DbH*tQxyfk`;duGLz0 zZ_ozgP|B_~i|jIgEjY-X?oJvRfEU6zyzee6<`pY8PxHLJa70F#weV_2xk2!Nh{bGb z+!ivmRCyL%POvFjo{8~@HD(qgG{4_G`4t{Lo{MV3iZ9K(BBp|pjx5poeHJh1lZ=$E zkLrAQf0+r0S2dAw>yC%hYN@xVZ4^UY_-&wCv-`OWcY3Mw06CXKV! z=r`s{rCk+Ssp&uzHgoQy;yTra*ge*s^~HPgB9gf#>hYBv*hrkR$hCN7x*^w$N7^!Q zA~tdgDk)-bDwp&AIxeRo@k5xzp?2NMEiJQ>9Za*h*RPbD{d8~erFs-lpIDl0%DVy{bh zqT@3g4JmRf#Auw7$)KpZ`-Qh}UrKB|*Ii+5-n9*@>|_zD*Sh42m(7uKvtvo~W57lr z?uMO|v4HbU_|7Z&*lH^F+s%2rYUbhY;LgwJlbVk2X0c|Lj>>AMtHm^lo`$RL$^h5wMY;;e+0TS(Q4vGp7fK9Pu7Lo(3?y zHpECw>25Qq(9oqNGJ%ZHlaJTu!nNtUHN9OS2h*5fhB$Eu;}Ya@5xpV|8>7q8*kw1+ zW-S^Cy(7cB9)h%wzK%#$ZpS|PA>wj?YRAzQ_s8g5#u|fxaa3lzPOlTH@&hy8*Y@Hm zRT~J3FGhzhBcr5w(Q@{fA>q())Oqsz#WCFH25l5K%l>d>k>mCRKGy>X=eFI7C?xVV zW+t*1zLFnK5|_lmjiP6*T-lw}Ia@}K~PP40LqedRNIdkij??tI& zfcUsLWBZ5G%J+6cI+yQW;=A4{7M-@a%_e?z0g0-~6ejUSW5m|vgJc)>WhNf)Df@PR zj}jL<1xrp?i8`(@3Dr*71ILa>1iuBZr3tE$h=|LbSCO3zb(wf$+C= zFE2#%Ubxor9jVJGun+EX`#0jXHWjwG!Ia>P`&yN6y$X;T02A; z7aA_Gp_+*2nQ6RAGGeB%Au0jpE*+BG=GvW{(;^x>B>|TrPrkpp8$x%}ZP!>K$&r^( zS>2Mic9@G%y>jMEFOr!}zsFwL$qgG{J5C3Lplj$xsqN|Z*cy63FT{{NG*m4d-?CKG zYKMp-&iS@J8jo)I)xqBOuX{4(X=Y$J>G?n?juMlYP==4aZD6kaBMvsw)-(N$Vdd}g zINRR?W6fh3dWsgJSd&6idcLvQh=Q1!SMr6_AhAsFcK})9xx(z+I^$)x9LNN+7s2gM zLq~qqsLZp%@wQ)}0VPcVEFZmVFG+>3bxxHQ#Z7QOsD%+W7Anh28^)a`MJY}Iw6mSo z&!a-yUgt$Qw4cjR15wI8EqP;c-;BGFS=sF}L_5(?Z21rt?a;K?5g6J}9~R{eZk0hY zK~TqoP~7tubyXdP7^+!InMbXFB^nFJS132G*0=TLSfeGbQpuY$1kvuEux5pOb#&Pc zsvw!Tv08&UtcBwSLTa*@-0V7+t!~{Bak|<2!Vc@GPLN^fpjD|HD!cqBUXsW3YK0CZ zF}ig@dY3{6A=6%t3@y z{vIC;&>X;vja|$wbvjVBSz+yxMuIm^A2)h-g8LPg+H`JLoi>1mD2&?A-~(R6k$h%}e!gc@$U_vJMWo3k@pEltj?%g9~=pNe*forpK)bKiF4Je?EE!cMt0$cqzg6TY1% zc@bS%-wtPEplrl2bJv#0r z>iIN;C+u>+alZSB$+;_5bys7RINnABB)YFCPksynR=?4cbs-qr+bdm~@kNzf$M*1| zz?_5$oEpwtO@{Sl;L(}3=Rg+?n&D&DSQHYPi~!HxFbgx-y~14=$+L$thL%ZNjHXGb z+2&oof1lJWdNU`};-M`nXTFlkE=tt^f>z~YR!?T#AxNy&PARhQ#aG=xjk1?hKP?-j zBY(jK7ol#h2)irI3dwN$LU)8~*2#S@KLZf&qF66(;+t74=VWe4;1DS$v($!t zg75Mf_`azqkl9aq>M|Crb{BT%TK(HRdeHl9Ga>6m{c81!1bws>7V~0r^U8!l06i46 zUVg8Jty;~-FMHh~fX5o-$~~#x5{YwzNX9OSg*TTSUttpsF9%QAYiu4iZk#aJ()^Pz zNL)N}R)Tr1;S;-XoNds%HX%a0poJz&2cpz(#zu4Jq{5d5 z@S+qf8U4_wl3j$86EO@%4(hcTzmSP#L5`YTkOxE&YK+nW(vT}#XU+I_B9X^R&gmGZ zoD(21pB}cw?T|ml6AMh?Qj z7Z|r)Gjbe>Vftu+&{gaf=9>)`GMEJnb7DQV=<`Y|#N3LNp4V?)&=IV~oH|yvqFVvK zU1xH>P@5h1x2@0|sO2ay_IOZDxAVIxboAnh$riq1O>{4KR@^9bw85sw+h%-Y_t*Y~ ziVm!jkl|r-<*5sY3=0%I=t#r;tXGzEqp@E*1Ovl^g093G-ZXv}>bgKeYiqD( zXp-;Ju|)f@sM~|f{XAPIcEM>LoWVwV5ktb6WpTFU_4o+Kg2HI(l^0@59w!UY7CJex zWvtgCazTG6w9W9!>Wr(KSmP4*S#yzVJoy$mlegxaE@E7axm;Lig&s4`k?E=AI6u$d zrrP@2x~xM08+$jAJSmcgD3vMS>be}$^*x{$8_XqZ_0|R9f_0L{shWy5x4X1Hx;D=| z?#8=`k!}aYZf_kL8nji;R6}R4%FhnRNIQUPmm{wzrLG=djBR1}g!# zL?)XljW(rv#;Y97Sv$BNy#_Lj;7n~2#GH69@DA9`0!|=cL=e91&m}UkP9`j*qQ-0D zIVK*>uwJY%_NrchLw0&fuaRY8x`dQv)PznoR(H>ap1jNA!y$^sw3=)M-7bEfAynn8 zL-_JX<%;-ffjI zL_boP#%uY+y-;r{(b(F#8%{VfKF4?4=CIVJRtx2DN=g*eZRhtLxBQ4Z<1=wRpWY_R zYuB0FSU6Qr63s@m-0a99HrmaLHv?S<2QR6JQ`Q!>JH#M$cspP11}3nbJV^T+tx2i1 z9Ir~g6Y$wJyV}^CUt-R5Dw)Er`GtJzZT+bWS4UP3YMVJvs5Wcqjp0lRkrWF9&%SFH z)5ns##$l0;`+nUXo)=6jyglSd^q2-2+wtDK@i`Jx?x%RZlD*BEjDu? zL`hhE1L@9c%=tEL&bHY=k}iquvPv`*sl1u2l^k>J;mLNoyOXhe{95A5u_bdN@0%IR zRtwE1UqOq;LRP1dzBs6BKzXo=h7>AId6x4}(< zv21UCKR2&wGt#ftsYSg#shc4HeEXRdwcvDV){eK`{-#{SJNt2L<{0JOX$V0r1Px(p zadovw?%D-V8X$DxFx2}--nWVnbObqjJrAXbHqo2YoJGxFr;rK5w)$0TyE2D2oTt~H{)(hV(hC{3Cv zFNj8JA;Ky(K|b#R@49l?YNxG*L>3OV5lTopO1+~_WZKH1=Rk-W z!Qz^qwJpbJW>7s8DJlM_aHWY}|Lke`n!B_!+cStep~LuaZdSr^$v;f9G;J&nI~{Tq z#6kQ+vXjf{ZFkf5*YU*8@pkC6fj9ddF;vPo2H7-yFM~XFDkWkLcdATFLs5nN)g%~% zo8BcV%CASik=aJ>+ zI2hJjhx^^-)-87c(>7j2t?p$VMl=lRd!ZBDz?!#JOI(a-GFK8uAl1bdajh#1Z|TVV zP|lP(2^uqdN-B|RwoWBGIHSj|cc~v8L9JW)X6k&u#Cvxj?D00|R`yyA_z^#wOON|a z+@H+d`0Ui8v&wSh#s}f?GGe8fi$+YD+}b`emt(jqoYv)xl`%+~tRZ2(t>T#*SOuxw z>Md<_+RytswkT`oW??lPay}9ZHpNj=oXJx1HFM%8MZQqv{Pz*4}#)I#lu~wQwr<}jMla0 z`9iBP5Tb-`ohWS(x^&c&oxVa$j0eMl{Un9HfFq@wG3oq@f3*Nm&Afcr> zk?3|}%e{?Npq5P%KSK;b4-f+>64nf4MONxK-RgWMbzEgu@NPqo>rB&M4y0ynSP0_+ zR=F+lF0pb-ZLiqHsOrhy8eA?o?x>Zfg#+%%XKv?O)$;YZb(%`;fW65r6|Wz2iG(rC z*BtgBobyyJJ4K*m$MVGaQoP){tD3sV$p_zFw<2I8`!mzPkYd7G6C*#hqeJ6Z;)Yrl z3j$cBMXbbrtUM(s_LdvRFfm!&FUd$T#o|D@g*7_?$%H#CB$60Z%WPXs3|Wov@?hbf z{7!St81zYo+O4jxs^uZKiI%O}_U?~X-LN)jUq_oYzBCJSV5zf2E>}LF= z&ZFC@cZ7}J%I6~eay(e0g~9SPwy{G%>@{PGvc?+gkZA2_*%9EDm-p$dAEB!eG9|TA zz*K6}0?jtCTxA!_cVqB+4b@u}t#2ICAxSLv534d`E$kp>MBRSdCjp7Qv~3)UTlY4b zXS4M&(+-qz<^nlWLDsh6umYRo2sIkr6KTdiZ}aH=9?z9*ZgYc2UaVfj{U={dL=LQf z_KDa$mbN?(fno@bD+}8#UbMY)+KUqfjWGFryjUp(RFqFLTQz5F5(@_CZ*vShyIM*Y zD6il)GElx1G116%@AtX|vY9EYoyaNgo*J_V(=rPj0btuI-Jv*n!si+J%$Elhp{N3% zImAByZ&K7H6LTBM)U1xta?l20orb9(Lx?W& zAapIY_pM2674y2MDqLInUc-=yLX_1TBO5J3T0&$Kd~4PAv%SHQy{8i*%E!y$-KY#; zG`SoDZ`S56WDjdZjg$cmavo8jiK|p1RswTD>87w=hVA(LWU8^)AeifDUcJ?g;I1K* zSW7Cm#5xWvwLL!a&^(!O%GKjOR&4H0DcK&b^FFFyS6&eOA?oxgSMd>P<{zk{R3#<-q&&#LT+S#g_Y$0KL?Ul0{vEf47OK7&4 ztJhU_63*8{tkMMOx+K@zwH0zB0yP{SJo#{fMwCQG@o2@%gMXb4bl+u(x=rE-d)@Jq z^KpL{mbdt}mRg4JNW;Muf#2}s8rxc}Q3Hf)^=Rg@YwqquFdxLOtzx<(8W=hp_m#b! zt8Z38@(2w(p;`H5|RUeD1pXcG=kqL=*0WzQf_bv$faMZ#_N+;UUVV`4s*=k2iQ zvy2wGi~=KVN@Zc5V3ZXWy~d?wxxotS9lFKm@&XqG2M5?Wubv2ok50vP3UTmQtr53} zh1hCVi&QFos(WjfrnN!f=*8k~KJ3@6;-(xjf_ZX19#!)7HS^?m^JL4+Tp=scn>Q;H zi0sU&<;-44;&(@=(vxDfaoLF318bjn5Cq;%2PAibqOCYDhU1XV0aw&A{VR+e#D1g1 zR0eJgYF0MQY_Cc!ebq0Cfzt`YYi?5RAXdCkgL@$qIaq~MY)6+tQU{4k5QF!utUs!q z3|Tx9CummC4x<>nNl0MZAVDG?r~|O2^&A-VN&IW31yPID$QP)f56k6h=J7 zo84G_9_eIS)n#YTM664MuBTYrIoLXmhMTQN88OVY!q*D-A0lunKf*Os~)d>so zRO3QNwN-?YY36#Zv(}VYw1tB|AO`n+FX-<|E#}hAMrQZ`-e7QH(yZ?_fjN^)Moq6{ z=QOa9{(LYMb}Nty9?_!hVQS5bV?XHbT(}s9MG;c_h;i_RMF;~I?Y;k6c$<^HI zW)>^*fP~T9_nnhkspHpuXwkAF1ALuLUoqCMn$xY8o=8pFmto)RL2PVV)|!Do3~T|8 z<|?XHD-cgU?XR4NH!E7mM>SacRq4xUk1>$3XUDbs8>sCEf-`Gj46!kHsn?> z$5||8Wy&JOkLfGW&KVN%Ff=rYRM@_*?R)c1?ZS82qH3hfZ0gzy;hEFO7m12Ajf~31 zz7}6w6I7>xHP?xV(*SNcI4mmYCdnL?JXfo*V^>>@c!E(!+-AZk8NYv`O)d{d;tPH5 z2`ApDvQ!UPdoONebPGz=pZr8()*V;sdjAfxe?3R?b|is$g~OehAyCH?pjKg9M6q>5 zR`I44HP);Z<8_fR7z1|pc4GSE$LcHIhM3_tQb!{R9oaw;XSq_YhCJ(U$GF0nEAKkm z8*8*_hDP~fg%aRFwm+GiW=g{q9wpP$_1qdb)Fo2LFR}2gB2x#%z8ONjb`bVtXp$V) zy121!LA5gG)q-d#(v&sn(yL)9s)BUVz^7URg;sZ-EnE_)eIeW}x+pS6g{YtPB%D|( zI~-p3LW$*`Bxh@1zw8ks>M3bNOgq~~YTX3+h?u;N+S_KRC#@YXbj6HqQfrl2=RSfB zB{f7hjY45qMk3|y&9=ybA2p&NSiWgSnhuCcQUeIi7^irlp14idtsOd=`C6KfpYTI1 zcAB?@-5lhW(7au0_lCE9qJW%MjRZ1Fjr!s?J0DshhXdMaSKHxGxRYJ%F+1K~k1lHe z;%CnLf`3Xgb19`dJvPr3ZVQJ*l#U-UqA5!qRaJcYU6&miAaa zxA(7;V>80Vgj#Pc=q^+~TwPWUbr`Q|o-51tksg-^nR}mcaYj%9J131M)@)^gZN$u- z2S*4$bd^HO>-}1&N)mMr@3Zl=Scin&4nf#~ST!Xgu2Qw4an0k&mZX6^^?V}6)MjzL zpV}T;wb1)a^P8o~ZnrN^v~)|kXN++;%}nlQ1Z(qJyimcVD4)$9ydrlgTmj^;d2?G! z>p6|=w$Ruxd0mcfP@x;mRJw*MWJ94HUX-Xrw0*P?3eLvOVF9j`qK$ny)=oRP4A@B= z;58F6jLZlkq<$RPN@{3h%+@CeZ;MrIa*wC{lk1i$vcfx=Od;h>xH!uX-9PGm~ONDM^G$T}0KlRq{ZcwMlh!Lefcw^IG{6vN#s6^k%ypGKVn4)rX1s zb^;d-z=Oqh4i)w?Z2gMFMA28{KovAeXWwqeUqV{Eom&|cFmp(YW!{$3` zt&9+hbYXIyV)m0?jogjqXeD!vZE?D}BiiUVZH22w;yfA#jg>*Jdsvl`32478d+b{7 zMTCEjE3Ke_4IL*LZtTij*HTHgs?vM<; z-DR%xC|c!HW$5H)5UsmOKKZ@2ta5V`A_ThHv(Fv=U@|+;Nm^%VP*%4$xHrIkjXB0C`>VXc>SBZXB5#q-fqk}m_`letam+%K}8HloMEe7MiG77KZNQ6*W{gUoh* zck`S4wPu9y?rAXU6)I9aQtvbdLuzR9=Q~Iydi5i1vVLqAo2#(~O{pt;&bOmd4i3%J z!Ig`RNmYqtyqRm4+@!a6*LF6dxK-nT_sC2H#21~$wHsS7MNf>aw}lucugP2#eiX?$ z`j}_tFsd@S>SgEneX34p-PmE%6(jlHL92$J{7n6JnqGU{Eu6mj>S}sqAx1C)7cD;q~P-V=t-20M7PV%XWc8K+-{7XT42UYMgE{ImZdN zRF~t{)d4OOgKAlbfP0RV?4Tb*?Th1$cALR6#aNaIA#wQP%u6cMiKg&eU z?X;qvt`g--Y1`DHj(ft`ws*iFqC{*`Gj``*QOCZCMAc|QPku7uOef_)EIQW!dw$^}^*&NB*B57tZIBrcLyt98tIWSX#( znu@l}_7D#soW>nTbxoxcPS%iANeKomSwM@yOtXbWxLPDC^Duj#wcEf9m2<#TW4ha2 zl(#!6K3hwZ0eoX4(fGKN7BQ@K;;KFAa=8<klb zc6eQp2C?c`8m*4i477`2`eo_110LQt)CF=JPvVnQCv=5t!>Nf8mvhG%+$SzQ#fhw& z6CVXUb=j*zPQq4C7+N+ZgbbDEaHN$j`SoRZK0l-U2F_mK1h%KqvUCns_wKA{b=CG+ zzPmwhtUmdve*gPB0?sGHx2@yxsUS*f^Q?pdI3}EO(|QI7Do4K zvqp1OXQxISxGJ{0P2_sx+m;I0c)gvt+2!0Nh9%TuDZD*E<=P2d7mtlobKI~HQmO&s^2%QetTUtjq^!q7Q9SpQ<#__*`VXvp(REmELjTkYlKllsXcq#h59j> z?Uw=W*|dP%?~?_Wz|d%IRvfK$e0y|rhrmQ{!j1%xvmaAp%snLU_8{NO%~SF1fRRr+ z79#9?fNJPKq!`xiH#lzyfOU1A1 zFtxqaj4hM4S0(!xukWWTt31q`bgt^HF*PBI>7YYN%iCp$MLB>M?R{CS%!8^;*tIIl z`Aaotgu9)nfKt@|3Zq&0GD#g*EkA+`)b5jy+0CLiy4`O? z4D#IbB@^^rg4mS%$TX0>eAO+(VoaAa{7%XhDtmN84u%`O&IqpKK^nRnfh4qJnk^M? zPxTSHsrF(uoXdOWfYC37#3ogyPm9EAGVoA*3^x@JG$Gy&e zz*&WOPwX2N`V!Rj`i1Rfo$;}7PNoB!c5}lTi;tJ5P%;VMjI0&r`&~RPj_{FUIsyd( z>`-gHHQU13RIJ!2T#**=TAe^;;o?a*^zBuY3#5L*-ag_BU z@M5=cRD{QYE8<}>5kgXD>%QuF)BH6xVik20nN->btvROhJ6&aI-qYNHP84x9di0b- zMxJKZ$H9rH_O2_C(|4=wi2^M&S;d%46=Eq~uwk26yPLMN7|9$`h8Tba0vUrRI4Y%R z_=+ObY?`f){mgb2dGd`7mhGj|2QOx+#c4PkQ_x;wNp_djjN6z`4XQ45bH0j}`va)i zko>e_oLo^9!WzEG@YJC@d8!940)M{|i@k$w9 zT6@&Mvge2lR&U4T{0K!3w*m%$4~e02uhSK-Q*K$o)+zcBHHqRpKJUAgJqEG+4Y2`$ z(>~9w1x>W$xoZet!WXg%82*K{69fQXRe21In*ZmhM)$WhBI-R=Y}N~YFwebd~k z;c;TFu7jQ;hY@77#+VXprgezgV#3S~pT*nbm@-=??5rk^T^6z5#Du3~r5!xpaM4KoNFR;V z=zJ&E>UQX|QVF{DQioGnHTU_>ZUpx-k(f2-xqQRRf%FfF(Zz5(*>Nc3URw+{!)mle z3(Hzb-KI`>Z^`G+Iv<|MqL?Q)bI$X8RcdsQH7aDBGP*1r0A3^(6}=SUMuEn5HbQ;Z zt^^5rahM_fc}NV8$xSc9i)-UJ$gw_7P!YNriZ+x=&}HRLlrcbnZHEiA6rUgK(!6)D zPS;KrA7P4R)~P!_A4pNBcRk$c{Melltz-O5t?p4_A0s(Do){qb#0NQjRNe($!Y!3k zXdl4|l*Xb(@-Q>hZdB=K+G=$VO}q>*1zrjvb6q;s9Ce^(Hj?i5l2~J)gZN@T8P+=Z zh_h~UF%unA=h@6hf?Cq=%TyXCgzl51bmBbC>`M8tZk#lWV>U-eR~%+<#v#`rn+kHgn}%0cF2s^ z4UmwB4*sOs#*#DdE zM6b(NM;CB~V1Med@e!yU}1BeqO^>MFdOkp1M zk2wxFuhJIHmaIan2d!MSv|0svxduq=06=-|^^Adg>)rvzn5~A6=2mbE z^A15XyR>A&)$+`n(E(Ab`gbNbi}*819sm;jB!$xU#9u+~VB4?N)M-6H<8y~Sy0K}A zna`H>jnJ~D#5EGCu(jK=a83+Uhs``wBtyh#R0M9h*{zNeOOc!?+rDTKo9J|N+0FE^ zRz-UBwVjJdZ1Jecax_lpHFCJlY0mzT8+VuSK2GndAUHT}d!A!vGdy(DENDQum0PFCnxz2_myPU(Z-xMBDZoVU%T|L5d)>2i zi*~2lrQ00aWth$c8DxsfQOd4X8Fn&~omDP5B4essXwE#PVTG*J1+2meqYKl%t8KD7 zrmStf*wSe8*1kgGxvIvEajJb8h+QWf8AP^aTvXBB@_tFLx7}t8udHfB444@@j2HO{ zt&Xzge)Z6#8ucOFgWZKw?p+y~=vE6Yt(`~aa!Kon#4giQ8)Z8~ugCRt8*(Io7jvOK zt|7;Jo5E!2);lqBp72g?fn1Dt8MD_m#TxBL_r+7*xa{~Yvp#8+-pC9c_YG#<qVG22DoT-x0!KBclG0XH{Q2NM%K}`KbE^5oP(8oJX^P0T`v}p z9q8HsxtTc%?Fj<-QD%A=7`08-K98~8p>~ZKK}{P>QbR)HjzWqK5mU@twy{(rI=$+` ztx#$ip-~wG&?06e?BVFTD9%00Y>pkgNcnEOe5D74vbK&G*S_K{tJ!`D?Rrky^eunD zuC2}0Wk+Go34stLQgSABe$Qp{{zb5p-1Uxn(On$5@Hr{&*mFrbjomC1H0RbaiXGRz z+G&kuM5-lJeC0ffZa^jwpvxhBYcjOCNssEjBhrIPy|nDN16sPEm2gQ&oY6y?xHg<> zfJMffCXvg1rd90ZrsKUd074B-)FGR2D3tFlF1e^1EIWZ6-u3C^aEs^#(Ox;>i$^PT z_lm@zl;R~sWH${KB1`n=i94{=!&!-5{6%_vT4!Q0CXR=ZD?2TgFqV(%V?=Sv!6bS+@`x$`g~%qF6u3YkGR*B7EFwaHWkUpipm`2 z8Wt&6w&H3xoz;Xz)jspdL!`2FrzUYegAk>=J0$vK`PNGr@!fJBTi%7da)XZRQ9BMi zSM`97AUGHFMw3%*C7wr%OE~Mt$I{6rNW#5aV-RbNj>%1|In;^6wzAw-r}rDe9F5I> zmIKb$A6RvnsCsRh3hvt2ic^u4(6{JT?&_NQvA-Pw)zVW`rY^zA(@&JLH^y|A^{b0yIZmTqGg4vtz(?(GoD$=2i>>$TE{_><3EIwK1QnJ7e6d3_X% zTh~b*M?33ouEWS;C==Y+D^XkzZjFMDj|wLS83C(UC+f>7w(Mm}Prj81pBl|2(mP8f zwRjo&?dfT*TFy*%GH`{k3#5UBakr=v#b$(ypEr=nvR02CuPdOPYQM{K z)jSg?2!O9ErwB>Yyp=O{D|3dM@qjp|8IxMg&XYRXRuQ)#jBQ>;3Sz@2;&_Mmfof^p z!>%=e(>tBv&N1dDrYdo}5RzI2QS#~d(w4J}(;OX>9{t zox$;uv-8uWOx&}Mhle|Jd@0uMA{>MCdAot{HC3dS6P-=_1^kqW3gyMswNl|zsaurO zZM%>zw?MeohqhcP!e3*(#Jg}d=QT*2DODt0eS5Bq$v!SCQmr?5(qR?sfj zsW#!&Cn6IHf!s{ho9WwTU+hq5YZDUV^jlzwmaUefFcVhR%@Il)!Z=HC+0LA z25e#VL~ykV*0g^F(QlEc)RHV+52-66&qrqUFOr6V)#;)P3^P#JXBonfp(&;emX((R2>h}TxqSkdt{qA(x#uq?>fZAZyWC_Mmtuk z5-J3AC#kOLv7!J6%jJQU)@48fQRrngzbW=8qk}djdvc`-p%rc5-FX7D3!BVnK)D;b zc`RaXkty+A@hoU)+x?Rk*5ma4YXbs`p zW|z(#=!F}}upPA?igqo5s^rV{W#DF;AdHX6-n^|$8RUwWPsUASZt+HJdYvu_Ul8$n zDb8O?a1JC$tGtszx>91x^W{qg#-i2mwP~I}+!>?BMvrY0gP9Sf^oiKL<#1)Z#_r27 zquf0?k*N9mq7VitN4d{2Y^BVYw&{?~`(;XmDW~w{Uo%BCoQ+Q|RwQ$R_;`K3EijpL zTA(eqd}wi%(x%dqThwVnZ1JNVZQg`i zI@h{pQ>}?N-P8-`%Q|RGk&2csi5}u}*~Ge3QutB3*PhX}eH5tHn6IP~npGQjho;L5 z$*iF8(Q6S=2itUETdhw9O++di_&jkdhS2-nEz9TOu9B>6;mLJ8nLqikGY&Np#m;WE z73O3Hjci**&upX9g_e#{y+FkK>ZRLR7UqqtwAv68B0KTyK2e>k$C<{sz^4lN#<5u$ zFUG5lDP(yP#|GyfI;=8k^T?i|!_-s&XjpV!PuFu%ENF_hEND{1p=EPq4hE0cfw2*n zpBER0acPxQHy19{)>0d09PP|KW!>mX^`gh;10k21^lCAyI+J3ltDK@&LakZ4dhI-r zNP*@ociD#0SRyY$=4vM(t94klD_$DDos;FNa!IWBX(x&g7^L6nx^|`(;(goXJBR3L zv3T-lx!o$f%ZaCyGQ6j}-WH!sHqgB7F~$7K8f(Ks#3Ak70a+xjdWO07j>o>eA5B-x z4B6%uOORMW1EW56puQJtk;T%{?+`eomKT+*aij%yX3tUlT$by8zuD8_#F&aSr}+V+ z@WXpxgvM!2dh#pXyc?guLppZMrWV5eoJvl4ooac}p`i)4`IJF%EZzvo z>@37oRQ;kI8DYZ#>=Y<-cl28Fc~(qQ_!_xV;TaM`j9R6etXv!8#zq%xBu$Au-CXFqddwn+Y`E9ok6CKmgpghd0OLPb$axY}+RE$P@=zQLoSKK{Sy(MROPU4)zfsk!HneNF}eJhpPF8Jddy^ ze%FkLC+U(9y|c2j44jj(!E5KuT7UA#%~`7+p3frb#4M*OP9bbIHaHv&7BM(&57P%M zN!LnN)IHyN((b^ngt?ziC(l7DrFB)?1gN#yEL@iEp^>^qeH42@G#vildygccS^$DTy}9 ztC5OaCoTWHPuA`DszXiHVG-X*Xr{2)Z&nJS-)WGKoW{AP%Zbf?WHlbHB3ip$fY_U# zyNi>d$*twHJ3jW3w?0EAM?0L2Q!62SWa@6$x2o>ibU+NwY91NipP~h6#+J`9v}lD? zqc_dcF(FdhP}E%*b)}mfi$TT+X{1Wm2y3Au%r`pXAY6@*AZQd>eY)mB^rJ<3=k++e zRg3FsNI5FU{`l@N7HX+8dpzNH86%buCT*7O8c4YZ#E~0C*tBLn`Rm|1Q!r;Pmc8UH z(j1ZD=th`ZlK?v^1iG$sb-5eU+nuGuB&WHKs!57 ztA9TAHRa&mgbpYL zu(N`2OWaz7%+qpyOHjd@rUWy!)8%Bb!B%ZDg3l;oL*;9!ZcYp@{pNLe$W1HD`6_9o zw;~8pI)l4fY~QASt1?*d>MVZ=HeRvdn_R7Rr6M6o$+lxj1R`JJhL|m%h&LL=v&EYMZ>pkpV?Ey2)HOm9O`kDe;(f;v7C{XKJ}btP{&3 zMY1yAPDZ*l&okJaT7qzmahECQ+JXlJ-Dy{u!sfy>yO({nQ7OShVcFrZgJ2JmvB0>; zkai58dDIx%!$F_&Z^lA7qrgUlrfxS?KV8-QSV5sWsaRDLswfN}Z z6y9{*!C{QMvCct>t}avKXf>HwAFn;&HytddmpqKOTY zJ073TDpTz4-a^~jFeR$fTPTvis=d0Z#@xva+bq$_WD0oE?Tu>jNG*h$7MntH@S5#i zTk^(?vCS1H#hJv=KQ`xNc3Q3*d^6vlN4sKV7p)f}6~EV@A%LBs<=)JM>@Hv5)vsKH zzucG`GVb#^GEEx?rb>1)(~W$)MsKHGt6s1>=;p{~hP~t@XKu#JE)VqLESy%oCe&}6 ztxGjt#a%RZ$=t+o)-spvN*hiBuNJ)}`N4Tr*J~%d8DB_;v)HfQD?)`~L1I-|Tr$Z5 z4;fd*pSDUp4b_m8>q-EA7xDdTHyQxz1UJnhiZMc1pGfSsF0NDl?+xhLX=|Q}{T>-tg z4%=KLF+-hHg-eecy|oyy>x;l527@N9sL~c9Po;MU-nHke3iW)$w`vzS$k zCWx`hqov3s-kVaBUbSh}z$zZSYa5z&d*N7x^^VzL##$z5NJfTe*Q$iHax+nuidA-y zEUy^p?xhl=wwOh$;-b}~3e!4Jt?44rMsb-zcPXma(Wv$SU!22`WDK?=$`~WkNUa$5 z%8uc&redwvQ79X4s6+{J&k?j}cY#m6!Of#PyV!ChN1J7p5XIKWI+$x{DVY zun%J+THuiSu9a%`^2y;jV`SXc!61uKrIy+}`Exi3oJwvyrbpQ+(j}sPvF|CGDU91e z{^;-MI8kWqGUbKt9>&NOnKqf+Hbj}1RhDDcqC9@`qpe1~sVG-dG=(ZnY84EM#EEWl zJerG}0lOc=1rJK@w{WVcZ6?jL@KJPd63eyg1*sZnf>W%mFuzu+C2UVWgb;Nfj)hW4Y2HT5&RN17_(OPB%MgL9I~vF2Tdhm;8U;j26uil94&0Z%yE3Kn6o13Oq82b>a!zDXQqFBiA zi3`6fs8vnAAomf!iY$FwHRYJ?BiJTonyV9(aD#CpE|R5g&2eJGiw1}41F(vFs2a9c zcYKJfsEk$CI-P+U?D#6#q!s%3bcrF^|y5CxEdg^q_Tyl-jW;Eib(de;DKRrH5 z!Y>A=TWWzi{LvyJQ>{@(YgU<>tm+tr99kjEIVqbBi+T}qm#^irh4m;?tGUzDG8tI{ z&(5QSZaL-ENJmj`hfw>0W$Bwz9G9y3N;{;@cmmA|o`JFp!P(!#+g`f9%%Vv&Z7hk! z3R#DX?eeX^3_baa^n}JpY#B-zVe(Aif)iy2!)v;NV|%W+0veJVH}e9_TFqOKnbCk@F&u8TVax-;T?ZHDvp^HTI$W`5IDof4xNZAy#^LzWu9@tz)3&jcCs)nj4 zF{7i7v7v?|Jh!K+${Gj2H%?4WRkL~Ld&^4faF#M9w(R6`4Upwzt_sr%LASArjOC~T zTvN3Hvzg5xxvdgX^-yxt*PP%@ZZObh~3^LbIvPqV5_O)40;m0cz zWtwExVON?*aIqGnrifE~2*sP|0i-m!Q>4NoW+7Rl_IvYKmowpec;^S4J5h)on-ql@ zz+0oD?@scldRWw&4Je=!gtfR2KR^-`j*pntlV5#e9-KWIybXKSxgjT&<^TK-6@5U!; zdw!EnkB;no1}$7rgc25QtW#P**r_2MV=+2?L~j>hW}{Vdd2&2Tu-o@?LJh=DR}S~) z+%>C82rrP$>~4WX`57;r23YI3N*lpsQ`_-Dr@OrznvZ_*v%bn#*ol7f`b(^0;EN>Y zuL^SGT1~B5C^b_na;G_!&<1rPIiYDX?ukwrq1ee}Gx*?I1n_H+8}E8y#+@~k7M*tc z5-(cXv9~<>db}D^NJ=(K>RQjS&F!YqUN<-X0s%_ssfRLGLOLx6q%@hJL?4?=ElxGc zOQ}-auS5Dws1651!5KPfGZ!|y_X^%D&aho1mg=k{2Q|b5hS1iND7?!&`KWbHCT2TP zo|cZTLNLy#F5p3et<;O1T?(tF?BrTLT`cO@GFEIeH5x8!B{G93)-_k3jWe^y$Iw(* z(j0RbN2jg)9gCz=m^A*w6*=Ds(3E5sQ8LY2EBo^t3T@DI&RU#rqtSEEkm%1Kqxv{t= z084Eu6Fqlr?#K|mSWe^fRD%&VBRCggczu#x&%*t4qKQ%Fad01Uv)s_=^KQI4FF~_I zTq=`32wh7p#Uxj?3$>eQDV`NVn-y6+Pbs!UXKqyiPbv)d-LIlV(>Hadf z39SI7aqV@b29@W?Bms2GyM<=01uvz6cB+vG7ab_exM1lrhD8(d}GR% zgA|K5;fhhkS(pusQUf@PlA_NnR@TI2DQs6|3ZPcrN;A=IKi`0`<}g3d^LMw+Vc0D) zn;5jmu|#N6saOiM6sqYN-eqROF`*(eA{x%;0;7ZHp8Q5Twp;nO?^(_~7eBi$}4xIHRD>$ylyI~2J9Kv%Kp@5oWsSMJ=o3ck?$M9rTh;{NUn038t0fN>-Ayu!> zja#-4Br=%Wb@p70b|cl(3GbZm)exL7b6Z=OcR^lLTezAeq!#1dVYrB#uuOYyZB1b- z5w*qG!)LKaTjXYfu2VtJn#9WF6setusWM?wHLne4%PpsVqv?R%+RLNOH>r6{I7DyA z3^!Nn>=eGw??APPBoRIcBT`A8ikCyuN||Z4%XLnMwXv*XMY@?eWhF$FC3n910lPQ0 z!Yo@_tew;$DTL+6JKiI4af@{FCq@iHve06OPRFEstEw?!X50RCSYO@sN#@oE`RjF1 zv)bX*q)%3}#Z63D3st-iu(OhPj9W9#8t26BRIK3_BwwfuGstn?Y$;M~Qt!5A#Y#Ai zmnCMh!3$n=-q|+?GG=>y;coQ{Pd-asux>M_K0Zsm%LU!Cw=trnNpIZd&iayHFVf{+ z?@s69#ol%WK*VXxv#HK)4VR`GuPJ+VO37`#FAPrDyXrBmXRzV(HpZ=q!>5uv^;Ak zY`ei^$^khNmV{^LN|##{PDF!j7rqYVz+CH1WmaRvTmz<$z>!;*2Eb0VPLWAlUo`je z-jec7BQ#HpF86Ncg5GyRXHxb_Z)RT-2E0o|3r2~;vPoe((96U@!AQKJKKXH^f7J9y z!=4YzeokO^#bSxTa=9X-ZKut6r!7s+E`r&9oh^1J4Ah)wVc@*aI@Ggz?b9lHWq~3IKhEnlSszaFuc8M%Yr5TbvIzGsX z(#mF?ob~EucX1!3?A0urVZxE6U7Hmy$?W#fR;DMZ>y73uzUryQ*|p-CdR3dY%IEfc zk(sSQR+L#VajNcT>cZS0S->?nQ#k00>C9|b z4?ERq6~HQ(vk~N{Sj6_HS??liR^PZRV%Oz;RhuaA>bM*3+ue0FG^n?{{MZv2BVON* zM#by3mI~x1B|rIPS!;nLW*f_Z@SDBB1iaXe*NM(9?Pa2KnzUAZ=CJi5(m1ut*&DZv z!_IXZkHsLjv$T5)06Ep5-Sqj`Sk0K{W&ARYwTY^)HOQnCEtYXH*)9zR8Eg^N2&~t! zB6=c!IAwSoN#5$4C_JeFZ+570iDO4GA;*#0*ZY3oMM9UVGfQZj#tgo86PSP{<>@x+ z4fJ8D7J(A2E;S=%YucVgxxobljVVkDFIi%EX_t`^NxB<8C}Wio%{m#g+l;vSgX`&0kyWkiy4^UQ5H{j3MQBwCb+NFpxTHp8 zA<(sXFKl$`(QKJcrl5#WZqa>gTZQH`+xBR^G+ut=}j<4_Sm&_5?; zIhf+bW1GZA%f|K0tQM3{`oJ0Ko2I%Nt093X#mV`TKUL<%YA-XI4`!+?CH54VF8Ik< zuC02TRN><8BbQ-xC3vfePNz!L)~jrM%t?(e3%%P-E-VmtuBPX_fh0yso`thm!b@KY z<4Am?aHk&E#xBJQUc{hQ1Y@$Zo#~g#{bpq2XJj4$& zh&wrV4qT}n6D6fv)wX<<9qaqzwtxkl6w563*nYwu;h+x76tvzAa~jDO>|{(H3^ZY4 zU!Qy%#KsGiK6b9F%qY!XONW7th5~;(u3y%Jm3mFV18E(a>E#+#i<1QmiDziF)|@Gg zOboFepC8U3v&g))NE^V)R9Rcc%By`HiAFDMKC{1>i}Zw~`?vZWzT0Jnl?_;jpz3gk zPXjIiTTmtYXfW!Z^4*Mb?H%-CUH32jOO!M6rQ7hlz9KYvkF21IiZ-?lcUDtk*KEs5 zc0)aE<3~?Xr(gmEKk#0SnY#jVa=faG=)LYT7h%hZB|)DH6nn`bG}hWiqSfIrycpLr zxtb6q3!TE8O39nnDsBOG!fC5j7)8w0gfS%X@e#*7PRtG~5kcBl3{!G+Y|(>8&{DGz zYt4X=QM8g%HJp{Mo!X(beexYBWY6hoXlv&O(xl)I>ZeXS2CvfiIuXM-cX|k-jretd zTWMru!&ZfGBGUZP;xe91n(;?DC>ke*cjVrKE_B{V%F7T|mC2$2$Iiv&jEhP8;-nf5 zrF6Eu+LRbGdN}gMNf|}&q1l~o89Z1;cw)Eu+h%LiED+lA#vE&~c<38BmadlOBHI;O zRhu%{HXe!cbr*IW-ASIzjKgq4v{J$oAVW;9E5$Byh(*Ola(7_`{}{u9SrgwY9+`^B zd33y<&+7DIyGRQj@mf9%EVVaH?pbu_N;$~JQ@U8F&UmEL3|9o7 zsdrWb$*xuQ5>gfrrPRmG1;fLI1ShhxT9_GkK^p_?Y>$&+q3x}qSn3w4Zq7x1>#oR^ zfrLTAffvGJkxgcFv(5C9@PXm^ZJ)Z{7T4q*jWAnSX+8PdLN^m@@o7HlsXL8}&f9&L z4)x+BKBuHs^#%`c&-96t9-TBVLlQ1wb`uJPTjH3;{R~A>PksiuaE-B5Jaw^~;dI6= zxJJ4)KXnP~OE@}KcQ^K&qZ~E@ajduL7dDxo)E@Cffxd{9JKe*c{5rr;ZJ|K*nJY3{ zuFd7-%CN$M*l-?M+?yU`YvE$LmADRW%Aj++-fvQ9SKPT_Ji1Xfh#PzS9g0Q1d~9eS zc+hdRYgfxV!*1#h6ef2OI~0hP%&*n_TCn}A21kllC6Dy-xRe#o(ZqI)9RTE*nfwZ2 z(HYNL$z_ss45)5IP!`EXhQotmt*~BM@nVT`=sJryZm~0tcxBUy2O;6aowf`M93r7N zZ+e+AFXX|Sh{Ap!IPit&EF--WS4X^kxCWk^2hs>f|9A5XeO&7f*VMeW8Qk=08kMb>EA&t?ZB9T1APVfDLlp6Dd#vu5uqC;-D z*tzV}Pkt>W>lNPaL}wSW-zjk?!P1FPRvBM{frRZY(XGTR2D>#d2pSSVD1D61&9ar4 z2e%y=y}Gr>8$J2fVi=F7RYL`f*LzZ%E)2KDK;-};XLC++_o5+;K~42U@~27?Zs+}Sth24 zSS&XzgrfIY3Sm+R5BIBmx(tMRQMF`o&dem;o$_8~t)~0&#~<;^i>LXLxDHbHflHhu zD4QV({o*=^92N0)179mi6!MYe%OBKm37n%nBfp=>S-tKui}w6RdV^Wwu5TS)Y0k}k zT|_&_Tp{e8BvlpM%MoJ6A+?(F9diJ(<@Ln@QERG;lzpobomjO3O2Aerc4(h3XD(7! zq%#bkg~EN=HHu5TX2&9>>2Q9yGpFMasrH`ydHK}z4FtFHj+u^4Oe%?U?XC_*MI+^e z4Z4;ei!hlDgy{C{L<`CJy_dhbq)ToW8K``S0av6qMq#zI3d#aelcU`HbYCsJt({ts zwRE}Ppba_J#m(?MS1MOt9x|1gYC@WXGhbfHpkMNLh zOd)7mJ$fruUqn*}|2C|grF8Z}cUxDcq74Cj*yy@)EF^aTFA(CGDc{BL+2@YB-z2l% zAvlsCar0yS-Ft0To>2~;$vj30&Tq;CFSd* zc5m6;mI|TO@;Vqzw&Nrn>#+zwnw+v3ZC8wTp>&Tt1u0^Y$g@&tk)|K%E<5{blryAFbQMpAziQ_qRiAE|Wq znIN8?|MjnD!jT_C5K(hvL)TP5<@sO=y884i1FDAg96WQp^?ZAl9)GO#iy?yFzLY=R zef<-YQ(gaI5CVw&3xGbL2*xAds7XK|B**XncosyHH{`AO$Fq1jiM;j4vkd-oc^jJ$54sU?v1VSFpGFZ{tL8`HQRpfyj>d<>j&}tKwtk z;|^yFvi{H{_4S^v37o}o4ks(zRuTVvh9J=aBK#BV>v8AL=Ni~7*udjKjxG!E^YvM? z6$KD`e0w8eTKefR_iu;sk8y_0Te<-;;3tt&|9l>Zg3Qfce);2>(N8ab1{Ht1{?3Nc z>t*;kxV`Puu)wo|XA8*S>&Lc@m*KLu`%->>8>pWi;~qah|1kJ_9=+ZDfgSi)=$#FZ z50QDW0{ZLA%jeB}`S%}>=L5F>`8xH3%Yaum_mHf;r{ z=TqYOIr(6`WpLUQJlvko+ZUL_h4(B>1q6% zZh$>PpMM7i<#GT7$3X36Q~YX(rmi?5SlaJzJa&`#UN?7|{E*1&n%~tEtbx;b(fsqr zVSoJ_tm?0Szia_4)si(^w@q+O;uKSS-Pwa9vii!3?{6s_*gp{1mzUpk&3GB}ouO7zqNXk~XqEUXk__3PDFJK{mqscGSMNRm7MSdM+7`pM@65g)7i|cRazvwTx;K*Mk z?V%NkS5^Jt61UZV8urBEbuTz zw4V362?W}v2K|QYkc#+lk>&TB25@_SK2Jr4;K*XQ3$@|*3v%r-51+j?<8(3jVZe3}Ke^8=lC@{er+N`lVYj~#t| zUGdxgpf7^#e0d-GF=nd^-dFB)0yY6X=D!bkZ~@Tzw&_oIU*z>**N5Z&9)DhO@fB-+ z1g8gG{Hv|0x((I@zk}$5I1T_w)nD-Fbq&9#@Ew7Vz5h7>RSr`T`0)=pJip)29!#z( z$}6AW?>)Bm{8*9<820LG%s)O9cq@WmI8Asq?X4;U4C8=F*}z7GXAl*gfHfJqz1xHD z;`6KLc^*@1Ah2Js=M~Ri34I2DPd6pe;`alcKYrZ*Oy}XP0OWx%(G;QXY&87(YUZ-! zLErnAFLe%p(_>LUobDH}$%kKiK)MYqIh{;CZ1L^MKWN{}62JyNmhzi%-?{Qn8v6M4 zeah(NkI;V=L`l{};Db)0VR?|oIr5GJSog>C7xh731Xp>ZT*&!Z~J*od!UyGdM0R255UdaHQ;*Z~O_w5lcYq!LU^`sv4wypDVckAL6;Fwe*LCRo+$^tU~KgEX+&Hs^>n zQPdtt`QYTYo&M+1OWE2(?|Ae&{0)lVSO|WCzWj?j$^Uvy&Az!4{9XM%X&?N@547|b z-26lj5CGR?;K=@bq&>W1gSh7b^slsD(c$6K%smL)-b3*h znMg0u*z0F%KD%Hqy8Z5#tVf)CdVCls81WYR{p8IO-}E7Phnsh(0LgddO$=`q`1ij3 zZQ3W{exh!CN=hofkEQ-fzDH7TqWd)O>$lY8)pVeTGl73U{`;KepUrtBGjAimh#E4V zy+mDq>l;4JehcCM`sTF1{PE)QKr;H*mlp^A$7|;FZ}^8r{H+y#9|Qa_^Hpr{R}Dbr z&&Kd6=u5muMlWb$zD9u0RZTEJ9`p=EYcPO1p{`i>R?*1L! z`l7Z2sFwbx^ZrWx`|FEJ{0xsHa3A__H1!ipf1!H3uD*oc4^?(#QwAa7=LAy+PkVOv zqV_UCdSvDAY9xb6kC;OEze4+dJ^1gm@E1t%X8r#}&3>_rUn<&<(t2o&38mgMrT;Mc z#r6FRE&n*Qm^~`V|5(QWPhRRJ@b56{A3l^EQeS?N?!BPbFA(eDBmrIB1bI5AJ$?i{ z1uTCl2mLy^`~OLH^sBV!Js)}`r7uC@w>stjU&P^Wxz+X8OS_PjXiK5&mWcW2e8@N zM?w3`w_s1ufyx3`^8Dtn-`q6^;s*;vbU?*a*)$)H`q#Sr11|pGAf)@x@zY!3_NA=< zROxW*TxI|A?f>Rkcey%1DB^1uJv_|1Rn-To$?{bwv5>hGD+ zUgQknv)g!l`R|XM|M_Y2$3Zh++H(1AvM7B0niEs$qQE6n{|g7r|J@n$$MJG1^Eh74 zW*(=@-%fmi>izKyyte?po^8$mo@e!O(yG|JEQrYaxpLzBS-&0k?F`p8MN*Ppj(lGV zxYEn_*~_usPp8(O=DmXU#}GmTc^)8d0@3mNf7$Zm|DSffeHs4!Nwy6%UuZs({cmLc zrO4Yy`u?T-?I3PYT`^O#KAJWv%m-6=Vvs#A(-kp1fV^~oT@|G53Su$Li^TltqY zKaMb-Uj#h+I2#p!t9ZTwDSVjs62lbhn(K-L(p4C z!XMz~BOChpyzF6i;=6d8AYqukT=*#YzTSTi0zcYkUdcg^N#x@O$XRt;5#C~BPI!rT zzO|OXKao>(PWVLawQpkfvK{Dc(5sdGIQ8+ZYQAt~+JE87TZhOa3i))T^eZiRsQuq+ z`Snwt_Xb-pJau0*_v=q{pK<*C-f#5sU8awG7dXU^1^%ciQ1DIT{&-K`{&){6Z-4$; zw_c9O{yWnDfZ~U8{PmJvfjOe};+Y;TDerLu`pscs(5t`t)NS$n*Gmy z_L&^?<@@Pxi2MVA&W~4rCh=|YFB<}_5-Z!(}#7v z*}E(O@Oh*5mA4J!(Z(36ytdfD?`r&*Y&^0q-L`%j{?VcJBbiSPYtP?U@_zdze}6qF zhTyM~{@UU2Ktlfb){F;!`uggs;NMaC{vn@4_S1mGFS6n9SorG*;RlX>Q27YKKGT3+ zP7FQ|dnp0@{gAh+LH*Hb@!iPeFGf-SQ1agv@UX7$$bN5AA3@GfBfgD!5&u+tKZ5-( zFMTM`4{Q5}di0xF-;UUSP4r!!uSei7WyOy~<}Gx3{raZy5d6pUi}ieL$36+b+F#qF zz;51Zd7s?w+t%J$_vz6uW8W6|GjzR@dMgLKt?gY6V3|q>u+fFQP=umXK#4;w|o00_TTUBCtUs5-}8_A`*zm)t!wgAZ1QQF-DdhV&^mnek{7p+kpKie4S~or$@+V$=>ze#d7+>mp5Jida<%-XhweRbL zU!Uiu_}tp+oFP1KMFnX8M+yFSLj2ZQ^7iZPz2NezI^HA9`qf^4eGvQ69sSdopBhGg zGw!{6^!2dy7X#i?pohF(GL+9kd+qvrEpmg@|IPTmd?|mvt%(=woA!RX3Eaao^ceA4 ztEoJ{(Es`=Z7jdtLchCZepOy7e(!e^mFLta*?$7p*L8n4_pg^Z|1SSH`ccJtQMuP{ z&cBlX#=m|#8w1hbrNO9$B#p9?>{`qe@6}e3~&$| z34X+}TO{~y!j1i=e%~Po^l1U`FKppMZEwHShL7an7s~L-JASbJZ9H%q-F-4S?utJd`6o^P$58Nz0e`#fZ{wezfkynND)QIo>SN-wD)Y}A z&jZW{D*OD3Hh(-4C+)R%yyIl;>vzN-orix1N{{U8ZBfs<_R@`&$jtgdf5G4a$F_ z%zGU8qr3m(b-jOsE#Ff#zRRm$G7-c3uB-LIg4cq{$6?8jMU?mG>RZRKB*_m)^Okf2 z6J|f!$be>RkLvP#+NrJV`A_Qjlxw`(XAMW|vvG|+WzO&)04syIlAGiDbty%=V zpJTm^dHB0W!!MxEehWe$hV%%h0h&RP;A8M>pY;u9;h#DG7L)uy{=tU_c>uwm8@ch% z1?}%Bee&_|-v5)|f5ZU5W&W_?Z})%R@26ugL9|{fF|VNTdSw1MCVXh}8{~f@{93H} zR5Sc^>jNXqx1NvJ4(n(B@dvb@A)W&m1AkMe2P0oEe!O&~ORviV8vks$Wu(CWpv zQ~Wm>d>6TXgYLJI^ScSZJ^p_==KiQ6zNVnRb$1__`aR!%A9TN=66o*y&+jkIRX)Eq_svIrSHyTLeSAucw&G6KewBcSK1z9?eAefj zg|prxfp5(oo@o8Ar6lj!>U&AxL(ad5us)0Qcb=p2oO~Pc8@>1xru`!L{I+oAa|`wB zy&rrF^sD6cy{_?#4x{fMhy04)Z^hsjOZpF-{x<)k;ppq=WJh`tB#)e!P(O4&J{g zZoXbCe-rSp3Y_m(e;W4VTev?aW60+m^)*_5jQ?uNK49BV%YA(l^4qv?JG=er(cd2q z{f$z-x{}W>?h}sJ))2|0lmCa{L`Ahy<7Z2^FvI)z&`){Ll9H`^sK)Z+|R|k zcb>ikNw3^|oaKG0ll>5NzGeKcOrky?lfC4KFU>=r8T1Q=e_-)XvA}n@_;%O&XQ8=% z`s1f2jQ-f9=4YpTEE#QDowmG2}12`wz`i4{!7QKmW)7^QFb? z893zQ%S*}MDhQhA?nOgbtncE4--izG;qrTM@FCpqOA!Anf#AnLpy@xvfuSGbz~CPM_^0Ep zU*Y7hZ@-6zsKmXDY8 zo}oXLE1J+7U@U%a*8` z2roXk@Vw(rNcwsEiW{%rkT*{ME1i7a?vO3;9LLAcrwbT)@{rWa`2cb;U%$J1K=5D1 z+?QuVe>D8R)5q?i`hvZqMesYdhrUHA=v(vWKYQ==S=Z{1-r)xu z^wW1-_+Y@}2lw#<*huEfGl=^o`BDkcN89V?340{_>>}AO&;P=1*yASpuh+GFtRDt) z`j1Qf@nwW>OZ?V$_uesgdlqE#0lPva_??OWcbfu%$?x~{c<9fYy1oB|nkP>C@o6s5 zoQFa^#BbOd|9J5M3to16L;v-d@j>aGAG?=VR6cWs{H@pkoB#6$)897uI}Ji#+n0Z{ zcOW-(^;^D>pGBScFtty^4e^mfI^y#u)A}o$7M%URzuDIdmv~+IomJ(&SycvD)fV|n z)Evm6pMcG;;FL;!$GZ20V&7K$gT4HA%RpgLX|MzE z1uR?zgAlNKkaWB}-Vg;On@qmM{2v3qJYH`4^rWARi@UrqC-<2-IuMKm95Uh|oDJgV zmwkYl0PcYWAg%%8>n%{0BjP?j(&Y8=>5K9#>6(R@qAG87ka7UG`pE1MOWy54@}d6u zGA?sbzF?mZ_JVh3_eqBDHvRjBztA$(*Y>H$Q(RdPKl_A2@`uO#ma7k+R~@dGZE{LJ zoFDLsjW3_h8y{}uJ@@#Z|KtDles*PuDhI6PEq3|+;?cKP13oSUuK^8+lV2?S)p2o( zXaGq1v_qg>*@uBgd4s}lD|^AjUmN8E9KoXams-^u zOnx@eUzz3uHh*b%+}6|;8*tlrh#oj@;1820lpl~Y2b4AUf}FoF qD6pAt+<(W^w_hI|dV2j7gGY*hJU^ws=5k+?PycTvj(^r|#Q*@SqL^d= literal 0 HcmV?d00001 diff --git a/docs/review-artifacts/s25-voice-lineage.md b/docs/review-artifacts/s25-voice-lineage.md new file mode 100644 index 0000000000000..07d96ff2ace33 --- /dev/null +++ b/docs/review-artifacts/s25-voice-lineage.md @@ -0,0 +1,33 @@ +# S25 WebUI voice donor lineage + +This directory preserves the independently reviewable donor for the browser voice slice. + +## Immutable donor + +- Commit: `791ca2b9f8677568c3d49b38c97549aa981e0706` +- Parent: `a79255d574bbf7279fadd2759782a4641c7976d9` +- Subject: `fix(web): use live speech for mobile voice input` +- Durable local ref: `refs/archive/s25-voice-donor-791ca2b9` +- Deterministically compressed binary format-patch: `docs/review-artifacts/791ca2b9.patch.gz` +- Compressed artifact SHA-256: `88fe5f0f1648cd3694de1c8dd76be1b61acd619b59433952581c2459b9610f0f` +- Uncompressed patch SHA-256: `b8b5ac97c9384601d7ad62b21d24e4e63f88c60a1832a5525a950ef1605e9bec` +- Stable patch-id: `79794abce4eb3e1a12eeb05e9a3bcf5c6626ddbd` + +After decompression, the stable patch-id computed from the committed patch equals the patch-id computed directly from commit `791ca2b9`, including the two donor screenshots. The committed patch remains sufficient if the non-published local archive ref is later removed. + +## Current-main port delta + +The candidate is a manual current-main port, not a cherry-pick. It retains the donor's browser `SpeechRecognition` direction, final/interim transcript accumulation, Android `onend` restart, fatal permission stop, tap-to-submit interaction, and typed Gboard fallback. Relative to the donor it: + +- ports onto `origin/main` at `f751a8c5467c41500e505d90cb0eb8b70929080f`; +- bounds restart attempts with exponential backoff and protects callbacks with a capture generation; +- normalizes and submits a non-empty transcript exactly once through a dedicated PTY helper; +- adds explicit disconnected, cancel, unavailable, start-failure, retry-budget, and no-empty-submit behavior tests; +- integrates the control into the current `ChatPage`/`ChatSidebar` layout; +- consumes structured `message.complete` events for browser speech output and cancels playback on barge-in; +- deliberately removes the unproven optional native bridge rather than selecting it from object shape alone; and +- omits the donor screenshots from the product commit because they are preserved byte-for-byte in the binary donor patch and do not constitute real-device proof. + +## Truth boundary + +Gboard is typed fallback only. Chrome Web Speech supplies browser transcript events. This slice does not claim private Gboard audio/VAD access, on-device or offline recognition, continuous or hands-free conversation, or verified behavior on a physical S25. Those claims remain gated on real-device evidence. diff --git a/web/src/components/ChatVoiceControl.test.tsx b/web/src/components/ChatVoiceControl.test.tsx index c26a2b5a80620..bde6000e2eedc 100644 --- a/web/src/components/ChatVoiceControl.test.tsx +++ b/web/src/components/ChatVoiceControl.test.tsx @@ -41,7 +41,6 @@ describe("ChatVoiceControl browser speech input", () => { afterEach(async () => { await act(async () => root.unmount()); - delete (window as Window & { hermesVoiceBridge?: unknown }).hermesVoiceBridge; host.remove(); vi.useRealTimers(); vi.unstubAllGlobals(); @@ -153,24 +152,4 @@ describe("ChatVoiceControl browser speech input", () => { expect(host.textContent).toContain("not connected"); }); - it("uses the optional native bridge and accepts partial and final events", async () => { - let listener: ((event: MessageEvent) => void) | undefined; - const bridge = { - postMessage: vi.fn(), - addEventListener: vi.fn((_name: string, cb: (event: MessageEvent) => void) => { listener = cb; }), - removeEventListener: vi.fn(), - }; - Object.defineProperty(window, "hermesVoiceBridge", { configurable: true, value: bridge }); - const { submit, surface } = await render(); - await act(async () => surface.click()); - expect(FakeRecognition.instances).toHaveLength(0); - expect(bridge.postMessage).toHaveBeenCalledWith(JSON.stringify({ version: 1, command: "start" })); - await act(async () => listener?.({ data: JSON.stringify({ version: 1, event: "partial", transcript: "Call" }) } as MessageEvent)); - await act(async () => listener?.({ data: JSON.stringify({ version: 1, event: "final", transcript: "Call the crew" }) } as MessageEvent)); - expect(host.textContent).toContain("Call the crew"); - await act(async () => surface.click()); - expect(submit).toHaveBeenCalledOnce(); - expect(submit).toHaveBeenCalledWith("Call the crew"); - expect(bridge.postMessage).toHaveBeenCalledWith(JSON.stringify({ version: 1, command: "stop" })); - }); }); diff --git a/web/src/components/ChatVoiceControl.tsx b/web/src/components/ChatVoiceControl.tsx index 5be2941579cdd..ef470900491bb 100644 --- a/web/src/components/ChatVoiceControl.tsx +++ b/web/src/components/ChatVoiceControl.tsx @@ -2,11 +2,6 @@ import { Mic, X } from "lucide-react"; import { useCallback, useEffect, useRef, useState } from "react"; import { Button } from "@nous-research/ui/ui/components/button"; -import { - connectNativeVoiceBridge, - type NativeVoiceBridgeConnection, - type NativeVoiceMessageTarget, -} from "@/lib/native-voice-bridge"; const RESTART_DELAY_MS = 250; const MAX_RESTARTS = 6; @@ -40,7 +35,6 @@ declare global { interface Window { SpeechRecognition?: SpeechRecognitionConstructor; webkitSpeechRecognition?: SpeechRecognitionConstructor; - hermesVoiceBridge?: NativeVoiceMessageTarget; } } @@ -65,7 +59,6 @@ export function ChatVoiceControl({ connected, submit, onBargeIn }: ChatVoiceCont const restartCountRef = useRef(0); const restartTimerRef = useRef(null); const generationRef = useRef(0); - const nativeBridgeRef = useRef(null); const finalRef = useRef(""); const interimRef = useRef(""); @@ -164,8 +157,7 @@ export function ChatVoiceControl({ connected, submit, onBargeIn }: ChatVoiceCont generationRef.current = generation; setError(""); setState("listening"); - if (nativeBridgeRef.current) nativeBridgeRef.current.command("start"); - else startRecognition(generation); + startRecognition(generation); }; const commit = () => { @@ -179,7 +171,6 @@ export function ChatVoiceControl({ connected, submit, onBargeIn }: ChatVoiceCont clearRestart(); recognitionRef.current?.abort(); recognitionRef.current = null; - nativeBridgeRef.current?.command("stop"); submit(transcript); updateFinal(""); updateInterim(""); @@ -193,39 +184,12 @@ export function ChatVoiceControl({ connected, submit, onBargeIn }: ChatVoiceCont clearRestart(); recognitionRef.current?.abort(); recognitionRef.current = null; - nativeBridgeRef.current?.command("cancel"); updateFinal(""); updateInterim(""); setError(""); setState("idle"); }, [clearRestart]); - useEffect(() => { - const connection = connectNativeVoiceBridge(window.hermesVoiceBridge, (event) => { - if (event.event === "partial") { - updateInterim(event.transcript ?? ""); - } else if (event.event === "final") { - updateFinal(event.transcript ?? ""); - updateInterim(""); - } else if (event.event === "error") { - setError(event.message ?? "Native speech recognition failed."); - if (event.fatal) { - listeningRef.current = false; - setState("error"); - } - } else if (event.event === "listening") { - setState("listening"); - } - }); - nativeBridgeRef.current = connection; - connection?.command("check"); - return () => { - connection?.command("cancel"); - connection?.disconnect(); - if (nativeBridgeRef.current === connection) nativeBridgeRef.current = null; - }; - }, []); - useEffect(() => cancel, [cancel]); const transcript = `${finalTranscript}${interimTranscript}`.trim(); diff --git a/web/src/lib/native-voice-bridge.test.ts b/web/src/lib/native-voice-bridge.test.ts deleted file mode 100644 index dde405e3eef8f..0000000000000 --- a/web/src/lib/native-voice-bridge.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; - -import { connectNativeVoiceBridge } from "./native-voice-bridge"; - -describe("origin-scoped native voice bridge", () => { - it("sends only the declared command envelope and accepts declared transcript events", () => { - let listener: ((event: MessageEvent) => void) | undefined; - const target = { - postMessage: vi.fn(), - addEventListener: vi.fn((_name: string, cb: (event: MessageEvent) => void) => { listener = cb; }), - removeEventListener: vi.fn(), - }; - const events = vi.fn(); - const bridge = connectNativeVoiceBridge(target, events); - expect(bridge).not.toBeNull(); - bridge?.command("check"); - bridge?.command("start"); - expect(target.postMessage).toHaveBeenNthCalledWith(1, JSON.stringify({ version: 1, command: "check" })); - expect(target.postMessage).toHaveBeenNthCalledWith(2, JSON.stringify({ version: 1, command: "start" })); - - listener?.({ data: JSON.stringify({ version: 1, event: "partial", transcript: "hello" }) } as MessageEvent); - listener?.({ data: JSON.stringify({ version: 1, event: "unknown", transcript: "ignored" }) } as MessageEvent); - expect(events).toHaveBeenCalledOnce(); - expect(events).toHaveBeenCalledWith({ version: 1, event: "partial", transcript: "hello" }); - }); - - it("fails closed for missing or malformed bridge objects and events", () => { - expect(connectNativeVoiceBridge(undefined, vi.fn())).toBeNull(); - const events = vi.fn(); - let listener: ((event: MessageEvent) => void) | undefined; - const target = { - postMessage: vi.fn(), - addEventListener: vi.fn((_name: string, cb: (event: MessageEvent) => void) => { listener = cb; }), - removeEventListener: vi.fn(), - }; - connectNativeVoiceBridge(target, events); - listener?.({ data: "not json" } as MessageEvent); - listener?.({ data: JSON.stringify({ version: 2, event: "final", transcript: "no" }) } as MessageEvent); - expect(events).not.toHaveBeenCalled(); - }); -}); diff --git a/web/src/lib/native-voice-bridge.ts b/web/src/lib/native-voice-bridge.ts deleted file mode 100644 index 2e86efebe09a9..0000000000000 --- a/web/src/lib/native-voice-bridge.ts +++ /dev/null @@ -1,79 +0,0 @@ -export type NativeVoiceCommand = "check" | "start" | "stop" | "cancel"; -export type NativeVoiceEventName = - | "availability" - | "ready" - | "listening" - | "partial" - | "final" - | "error" - | "ended"; - -export interface NativeVoiceEvent { - version: 1; - event: NativeVoiceEventName; - transcript?: string; - available?: boolean; - message?: string; - fatal?: boolean; -} - -export interface NativeVoiceMessageTarget { - postMessage(message: string): void; - addEventListener(type: "message", listener: (event: MessageEvent) => void): void; - removeEventListener(type: "message", listener: (event: MessageEvent) => void): void; -} - -const EVENT_NAMES = new Set([ - "availability", "ready", "listening", "partial", "final", "error", "ended", -]); - -function parseNativeVoiceEvent(data: unknown): NativeVoiceEvent | null { - if (typeof data !== "string") return null; - try { - const value = JSON.parse(data) as Record; - if (value.version !== 1 || typeof value.event !== "string" || !EVENT_NAMES.has(value.event as NativeVoiceEventName)) { - return null; - } - if (value.transcript !== undefined && typeof value.transcript !== "string") return null; - if (value.available !== undefined && typeof value.available !== "boolean") return null; - if (value.message !== undefined && typeof value.message !== "string") return null; - if (value.fatal !== undefined && typeof value.fatal !== "boolean") return null; - return value as unknown as NativeVoiceEvent; - } catch { - return null; - } -} - -export interface NativeVoiceBridgeConnection { - command(command: NativeVoiceCommand): void; - disconnect(): void; -} - -/** - * Bind the optional AndroidX WebKit WebMessage listener object. - * - * The native host owns origin allow-listing when it injects this target. The - * web app neither assumes a specific Android recognizer nor receives provider - * credentials. Missing/malformed targets fail closed to ordinary Web Speech. - */ -export function connectNativeVoiceBridge( - target: NativeVoiceMessageTarget | undefined, - onEvent: (event: NativeVoiceEvent) => void, -): NativeVoiceBridgeConnection | null { - if (!target || typeof target.postMessage !== "function" || typeof target.addEventListener !== "function" || typeof target.removeEventListener !== "function") { - return null; - } - const listener = (message: MessageEvent) => { - const event = parseNativeVoiceEvent(message.data); - if (event) onEvent(event); - }; - target.addEventListener("message", listener); - return { - command(command) { - target.postMessage(JSON.stringify({ version: 1, command })); - }, - disconnect() { - target.removeEventListener("message", listener); - }, - }; -} From 84b4cdece0e90f627fca56ef92122a92c0d8f4c0 Mon Sep 17 00:00:00 2001 From: lesseradmin Date: Fri, 28 Aug 2026 14:40:06 -0500 Subject: [PATCH 3/7] fix(web): synchronize browser voice submission --- web/src/lib/assistant-voice-output.test.ts | 11 +++- web/src/lib/assistant-voice-output.ts | 3 +- web/src/lib/pty-voice-submit.test.ts | 35 +++++++---- web/src/lib/pty-voice-submit.ts | 20 ++++--- web/src/pages/ChatPage.test.tsx | 70 +++++++++++++++++++++- web/src/pages/ChatPage.tsx | 38 +++++++++--- 6 files changed, 148 insertions(+), 29 deletions(-) diff --git a/web/src/lib/assistant-voice-output.test.ts b/web/src/lib/assistant-voice-output.test.ts index d984c9bbda25a..1a7214e22c815 100644 --- a/web/src/lib/assistant-voice-output.test.ts +++ b/web/src/lib/assistant-voice-output.test.ts @@ -22,11 +22,20 @@ describe("assistant voice output", () => { const synth = { speak: vi.fn(), cancel: vi.fn() }; const settled = vi.fn(); const playback = speakAssistantFinal(synth, "Done", settled); + expect(playback).not.toBeNull(); const utterance = synth.speak.mock.calls[0][0] as SpeechSynthesisUtterance; expect(utterance.text).toBe("Done"); utterance.onend?.({} as SpeechSynthesisEvent); - playback.cancel(); + playback?.cancel(); expect(settled).toHaveBeenCalledOnce(); expect(synth.cancel).toHaveBeenCalledOnce(); }); + + it("does not start playback when SpeechSynthesisUtterance is unavailable", () => { + vi.stubGlobal("SpeechSynthesisUtterance", undefined); + const synth = { speak: vi.fn(), cancel: vi.fn() }; + + expect(speakAssistantFinal(synth, "Done", vi.fn())).toBeNull(); + expect(synth.speak).not.toHaveBeenCalled(); + }); }); diff --git a/web/src/lib/assistant-voice-output.ts b/web/src/lib/assistant-voice-output.ts index cb23fe69f1c7f..d09d90f9c285f 100644 --- a/web/src/lib/assistant-voice-output.ts +++ b/web/src/lib/assistant-voice-output.ts @@ -21,7 +21,8 @@ export function speakAssistantFinal( synthesis: SpeechSynthesisLike, text: string, onSettled: () => void, -): { cancel(): void } { +): { cancel(): void } | null { + if (typeof SpeechSynthesisUtterance !== "function") return null; const utterance = new SpeechSynthesisUtterance(text); let settled = false; const settle = () => { diff --git a/web/src/lib/pty-voice-submit.test.ts b/web/src/lib/pty-voice-submit.test.ts index e3cceeb4839c0..ee9f4060a469a 100644 --- a/web/src/lib/pty-voice-submit.test.ts +++ b/web/src/lib/pty-voice-submit.test.ts @@ -3,33 +3,48 @@ import { describe, expect, it, vi } from "vitest"; import { submitVoiceTranscriptToPty } from "./pty-voice-submit"; describe("submitVoiceTranscriptToPty", () => { - it("sends transcript then exactly one Return in a separate PTY frame", () => { - vi.useFakeTimers(); + it("sends Return only after xterm reports the transcript render", () => { const socket = { readyState: WebSocket.OPEN, send: vi.fn() }; const current = () => socket; + let terminalReady: (() => void) | undefined; - expect(submitVoiceTranscriptToPty(current, "Call the crew")).toBe(true); + expect( + submitVoiceTranscriptToPty(current, "Call the crew", (ready) => { + terminalReady = ready; + }, vi.fn()), + ).toBe(true); expect(socket.send).toHaveBeenCalledTimes(1); expect(socket.send).toHaveBeenNthCalledWith(1, "Call the crew"); - vi.advanceTimersByTime(100); + terminalReady?.(); expect(socket.send).toHaveBeenCalledTimes(2); expect(socket.send).toHaveBeenNthCalledWith(2, "\r"); - vi.useRealTimers(); }); it("does not send Return through a replacement or closed socket", () => { - vi.useFakeTimers(); const first = { readyState: WebSocket.OPEN, send: vi.fn<(data: string) => void>() }; const second = { readyState: WebSocket.OPEN, send: vi.fn<(data: string) => void>() }; let active: { readyState: number; send(data: string): void } = first; - expect(submitVoiceTranscriptToPty(() => active, "Hello")).toBe(true); + let terminalReady: (() => void) | undefined; + const onReturnFailed = vi.fn(); + expect( + submitVoiceTranscriptToPty(() => active, "Hello", (ready) => { + terminalReady = ready; + }, onReturnFailed), + ).toBe(true); active = second; - vi.advanceTimersByTime(100); + terminalReady?.(); expect(first.send).toHaveBeenCalledOnce(); expect(second.send).not.toHaveBeenCalled(); + expect(onReturnFailed).toHaveBeenCalledOnce(); active = { readyState: WebSocket.CLOSED, send: vi.fn<(data: string) => void>() }; - expect(submitVoiceTranscriptToPty(() => active, "Nope")).toBe(false); - vi.useRealTimers(); + expect( + submitVoiceTranscriptToPty( + () => active, + "Nope", + () => undefined, + onReturnFailed, + ), + ).toBe(false); }); }); diff --git a/web/src/lib/pty-voice-submit.ts b/web/src/lib/pty-voice-submit.ts index 25c13916c8983..7cd56fc952531 100644 --- a/web/src/lib/pty-voice-submit.ts +++ b/web/src/lib/pty-voice-submit.ts @@ -1,5 +1,3 @@ -const PTY_TRANSCRIPT_RETURN_DELAY_MS = 100; - interface PtySocket { readonly readyState: number; send(data: string): void; @@ -8,23 +6,27 @@ interface PtySocket { /** * Submit one finalized browser transcript to Ink's composer. * - * Return is deliberately a separate frame: a combined mobile WebSocket frame - * can be classified as a paste and remain in the composer. The captured socket - * must still be current before Return is sent, so a reconnect cannot submit the - * transcript into a replacement PTY session. + * Return is deliberately a separate frame after xterm commits the PTY echo: a + * combined mobile WebSocket frame can be classified as a paste and remain in + * the composer. The captured socket must still be current before Return is + * sent, so a reconnect cannot submit into a replacement PTY session. */ export function submitVoiceTranscriptToPty( currentSocket: () => PtySocket | null, transcript: string, + afterNextTerminalWrite: (ready: () => void) => void, + onReturnFailed: () => void, ): boolean { const socket = currentSocket(); if (!socket || socket.readyState !== WebSocket.OPEN) return false; - socket.send(transcript); - globalThis.setTimeout(() => { + afterNextTerminalWrite(() => { if (currentSocket() === socket && socket.readyState === WebSocket.OPEN) { socket.send("\r"); + } else { + onReturnFailed(); } - }, PTY_TRANSCRIPT_RETURN_DELAY_MS); + }); + socket.send(transcript); return true; } diff --git a/web/src/pages/ChatPage.test.tsx b/web/src/pages/ChatPage.test.tsx index 97a4f009a8140..3511ed3030c52 100644 --- a/web/src/pages/ChatPage.test.tsx +++ b/web/src/pages/ChatPage.test.tsx @@ -17,6 +17,9 @@ class FakeWebglAddon { } class FakeTerminal { + static instances: FakeTerminal[] = []; + + writeCallbacks: Array<() => void> = []; options: Record; rows = 24; cols = 80; @@ -27,6 +30,7 @@ class FakeTerminal { constructor(options: Record) { this.options = options; + FakeTerminal.instances.push(this); } attachCustomKeyEventHandler() { @@ -75,7 +79,9 @@ class FakeTerminal { refresh() {} - write() {} + write(_data: string, callback?: () => void) { + if (callback) this.writeCallbacks.push(callback); + } } const maybeReloadForLoopbackWsAuthFailure = vi.fn(() => false); @@ -94,6 +100,11 @@ vi.mock("@/components/ChatSidebar", () => ({ vi.mock("@/components/ChatSessionList", () => ({ ChatSessionList: () => null, })); +vi.mock("@/components/ChatVoiceControl", () => ({ + ChatVoiceControl: ({ submit }: { submit(text: string): void }) => ( + + ), +})); vi.mock("@/components/Backdrop", () => ({ Backdrop: () => null })); vi.mock("@/plugins", () => ({ PluginSlot: () => null, @@ -190,6 +201,7 @@ async function render(ui: ReactNode) { } beforeEach(() => { + FakeTerminal.instances = []; FakeWebSocket.instances = []; maybeReloadForLoopbackWsAuthFailure.mockClear(); apiMocks.buildWsUrl.mockReset(); @@ -255,6 +267,62 @@ afterEach(async () => { }); describe("ChatPage", () => { + it("waits for the post-submission terminal write before sending voice Return", async () => { + const { default: ChatPage } = await import("./ChatPage"); + await render( + + + , + ); + await vi.waitFor(() => expect(FakeWebSocket.instances).toHaveLength(1)); + const socket = FakeWebSocket.instances[0]; + socket.send = vi.fn(); + socket.onmessage?.({ data: "older output" }); + + await act(async () => { + Array.from(container.querySelectorAll("button")) + .find((button) => button.textContent === "submit voice")! + .dispatchEvent( + new MouseEvent("click", { bubbles: true }), + ); + }); + expect(socket.send).toHaveBeenCalledOnce(); + expect(socket.send).toHaveBeenCalledWith("voice transcript"); + + const terminal = FakeTerminal.instances[0]; + terminal.writeCallbacks.shift()?.(); + expect(socket.send).toHaveBeenCalledOnce(); + + socket.onmessage?.({ data: "voice transcript" }); + terminal.writeCallbacks.shift()?.(); + expect(socket.send).toHaveBeenNthCalledWith(2, "\r"); + }); + + it("surfaces a voice submission failure when the socket closes before echo", async () => { + const { default: ChatPage } = await import("./ChatPage"); + await render( + + + , + ); + await vi.waitFor(() => expect(FakeWebSocket.instances).toHaveLength(1)); + const socket = FakeWebSocket.instances[0]; + socket.send = vi.fn(); + + await act(async () => { + Array.from(container.querySelectorAll("button")) + .find((button) => button.textContent === "submit voice")! + .dispatchEvent(new MouseEvent("click", { bubbles: true })); + socket.readyState = 3; + socket.onclose?.({ code: 1006, reason: "", wasClean: false }); + }); + + expect(container.textContent).toContain( + "Chat disconnected before the voice transcript could be submitted.", + ); + expect(socket.send).not.toHaveBeenCalledWith("\r"); + }); + it("treats loopback 4401 closes as stale-token reload candidates", async () => { const { default: ChatPage } = await import("./ChatPage"); diff --git a/web/src/pages/ChatPage.tsx b/web/src/pages/ChatPage.tsx index c69875714cdff..661e283ddc403 100644 --- a/web/src/pages/ChatPage.tsx +++ b/web/src/pages/ChatPage.tsx @@ -185,6 +185,7 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) { const wsRef = useRef(null); const voiceTurnRef = useRef(false); const voicePlaybackRef = useRef<{ cancel(): void } | null>(null); + const pendingVoiceReturnRef = useRef<(() => void) | null>(null); const stickToBottomRef = useRef(true); // Exposed to the main metrics-sync effect so it can refit the terminal // the moment `isActive` flips back to true (display:none → display:flex @@ -512,7 +513,14 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) { }; const submitVoiceTranscript = useCallback((transcript: string) => { - if (!submitVoiceTranscriptToPty(() => wsRef.current, transcript)) { + if (!submitVoiceTranscriptToPty( + () => wsRef.current, + transcript, + (ready) => { + pendingVoiceReturnRef.current = ready; + }, + () => setBanner("Chat disconnected before the voice transcript could be submitted."), + )) { setBanner("Chat disconnected before the voice transcript could be sent."); return; } @@ -532,7 +540,7 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) { const playback = speakAssistantFinal(window.speechSynthesis, text, () => { if (voicePlaybackRef.current === playback) voicePlaybackRef.current = null; }); - voicePlaybackRef.current = playback; + if (playback) voicePlaybackRef.current = playback; }, [cancelVoicePlayback]); useEffect(() => cancelVoicePlayback, [cancelVoicePlayback]); @@ -1344,17 +1352,24 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) { // bottom as each chunk COMMITS (xterm write callback) instead of // guessing with a fixed delay, and release the pin the moment the user // scrolls up to read the backlog (#59591). - const followScroll = shouldFollowPtyOutput( + const shouldFollowScroll = shouldFollowPtyOutput( effectiveResume, stickToBottomRef.current, - ) - ? () => termRef.current?.scrollToBottom() - : undefined; - term.write(rendered, followScroll); + ); + const submitReturn = pendingVoiceReturnRef.current; + term.write(rendered, () => { + if (shouldFollowScroll) termRef.current?.scrollToBottom(); + if (submitReturn && pendingVoiceReturnRef.current === submitReturn) { + pendingVoiceReturnRef.current = null; + submitReturn(); + } + }); noteResumePtyChunk(rendered); }; ws.onclose = (ev) => { + const failPendingVoiceReturn = pendingVoiceReturnRef.current; + pendingVoiceReturnRef.current = null; // Drain buffered sanitizer state. A buffered partial escape is dropped // (writing an unterminated CSI would wedge xterm's parser); a buffered // newline run is emitted collapsed. @@ -1379,6 +1394,15 @@ export default function ChatPage({ isActive = true }: { isActive?: boolean }) { const why = ev.reason ? ` reason=${ev.reason}` : ""; console.warn(`[chat] PTY WebSocket closed code=${ev.code}${why}`); setLastCloseCode(ev.code); + if (failPendingVoiceReturn) { + if (!ev.wasClean || ev.code === 1001 || ev.code === 1006) { + scheduleReconnect(ev.code); + } else { + setPtyState("closed"); + } + failPendingVoiceReturn(); + return; + } if (ev.code === 4401) { if (maybeReloadForLoopbackWsAuthFailure(ev.code)) { return; From 65539bc3361faf7afca31872db38eae70754052b Mon Sep 17 00:00:00 2001 From: lesseradmin Date: Fri, 28 Aug 2026 21:35:59 -0500 Subject: [PATCH 4/7] fix(web): respect reduced motion in voice control --- web/src/components/ChatVoiceControl.test.tsx | 8 ++++++++ web/src/components/ChatVoiceControl.tsx | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/web/src/components/ChatVoiceControl.test.tsx b/web/src/components/ChatVoiceControl.test.tsx index bde6000e2eedc..b4a25e780afcf 100644 --- a/web/src/components/ChatVoiceControl.test.tsx +++ b/web/src/components/ChatVoiceControl.test.tsx @@ -145,6 +145,14 @@ describe("ChatVoiceControl browser speech input", () => { expect(surface.textContent).toContain("Gboard typed fallback remains below"); }); + it("suppresses the listening pulse when reduced motion is requested", async () => { + const { surface } = await render(); + await act(async () => surface.click()); + const mic = surface.querySelector("svg"); + expect(mic?.classList.contains("animate-pulse")).toBe(true); + expect(mic?.classList.contains("motion-reduce:animate-none")).toBe(true); + }); + it("refuses to start while chat is disconnected", async () => { const { surface } = await render(vi.fn(), false); await act(async () => surface.click()); diff --git a/web/src/components/ChatVoiceControl.tsx b/web/src/components/ChatVoiceControl.tsx index ef470900491bb..52c6449979195 100644 --- a/web/src/components/ChatVoiceControl.tsx +++ b/web/src/components/ChatVoiceControl.tsx @@ -207,7 +207,7 @@ export function ChatVoiceControl({ connected, submit, onBargeIn }: ChatVoiceCont onClick={state === "listening" ? commit : begin} className="flex min-h-[42dvh] w-full touch-manipulation flex-1 flex-col items-center justify-center gap-5 overflow-hidden px-5 py-8 text-center active:bg-white/10 lg:min-h-44" > - + {headline} From b105ef594d055801d3f9e5f5796aaf8f642615a8 Mon Sep 17 00:00:00 2001 From: lesseradmin Date: Tue, 25 Aug 2026 21:29:25 -0500 Subject: [PATCH 5/7] feat(web): add verified Android voice bridge --- web/src/components/ChatVoiceControl.test.tsx | 124 ++++++++++++++++++ web/src/components/ChatVoiceControl.tsx | 127 ++++++++++++++++++- 2 files changed, 249 insertions(+), 2 deletions(-) diff --git a/web/src/components/ChatVoiceControl.test.tsx b/web/src/components/ChatVoiceControl.test.tsx index b4a25e780afcf..1b9084cfb8942 100644 --- a/web/src/components/ChatVoiceControl.test.tsx +++ b/web/src/components/ChatVoiceControl.test.tsx @@ -26,6 +26,17 @@ class FakeRecognition { } } +class FakeNativeBridge { + messages: string[] = []; + onmessage: ((event: MessageEvent) => void) | null = null; + postMessage(message: string) { this.messages.push(message); } + emit(event: string, text?: string) { + this.onmessage?.(new MessageEvent("message", { + data: JSON.stringify({ version: 1, event, ...(text === undefined ? {} : { text }) }), + })); + } +} + describe("ChatVoiceControl browser speech input", () => { let host: HTMLDivElement; let root: ReturnType; @@ -160,4 +171,117 @@ describe("ChatVoiceControl browser speech input", () => { expect(host.textContent).toContain("not connected"); }); + it("checks the native bridge and falls back when on-device recognition is unavailable", async () => { + const bridge = new FakeNativeBridge(); + Object.defineProperty(window, "zer0Voice", { configurable: true, value: bridge }); + const { surface } = await render(); + + await act(async () => surface.click()); + expect(bridge.messages).toEqual([JSON.stringify({ version: 1, command: "check" })]); + await act(async () => bridge.emit("availability", "on-device-unavailable;fallback-choice-required")); + + expect(FakeRecognition.instances).toHaveLength(1); + expect(FakeRecognition.instances[0].start).toHaveBeenCalledOnce(); + }); + + it("does not trust ready or final events before confirmed native availability", async () => { + const bridge = new FakeNativeBridge(); + Object.defineProperty(window, "zer0Voice", { configurable: true, value: bridge }); + const { submit, surface } = await render(); + + await act(async () => surface.click()); + await act(async () => bridge.emit("ready")); + await act(async () => bridge.emit("final", "must not send")); + await act(async () => vi.advanceTimersByTime(1_000)); + + expect(submit).not.toHaveBeenCalled(); + expect(FakeRecognition.instances).toHaveLength(1); + }); + + it("uses confirmed native partials for display and submits a final exactly once", async () => { + const bridge = new FakeNativeBridge(); + Object.defineProperty(window, "zer0Voice", { configurable: true, value: bridge }); + const { submit, surface } = await render(); + + await act(async () => surface.click()); + await act(async () => bridge.emit("availability", "on-device")); + expect(bridge.messages.at(-1)).toBe(JSON.stringify({ version: 1, command: "start" })); + await act(async () => bridge.emit("ready")); + await act(async () => bridge.emit("partial", "Call the cr")); + expect(host.textContent).toContain("Call the cr"); + expect(submit).not.toHaveBeenCalled(); + const staleHandler = bridge.onmessage; + await act(async () => bridge.emit("final", "Call the crew")); + await act(async () => staleHandler?.(new MessageEvent("message", { + data: JSON.stringify({ version: 1, event: "final", text: "duplicate" }), + }))); + + expect(submit).toHaveBeenCalledTimes(1); + expect(submit).toHaveBeenCalledWith("Call the crew"); + }); + + it("falls back after a bounded native handshake timeout", async () => { + const bridge = new FakeNativeBridge(); + Object.defineProperty(window, "zer0Voice", { configurable: true, value: bridge }); + const { surface } = await render(); + await act(async () => surface.click()); + await act(async () => vi.advanceTimersByTime(999)); + expect(FakeRecognition.instances).toHaveLength(0); + await act(async () => vi.advanceTimersByTime(1)); + expect(FakeRecognition.instances).toHaveLength(1); + expect(bridge.messages.at(-1)).toBe(JSON.stringify({ version: 1, command: "cancel" })); + }); + + it.each(["ended", "error"])("falls back when native emits %s while listening", async (event) => { + const bridge = new FakeNativeBridge(); + Object.defineProperty(window, "zer0Voice", { configurable: true, value: bridge }); + const { surface } = await render(); + await act(async () => surface.click()); + await act(async () => bridge.emit("availability", "on-device")); + await act(async () => bridge.emit("listening")); + await act(async () => bridge.emit(event, "native-ended")); + await act(async () => vi.advanceTimersByTime(500)); + expect(FakeRecognition.instances).toHaveLength(1); + expect(FakeRecognition.instances[0].start).toHaveBeenCalledOnce(); + }); + + it("accepts the concrete Android consumer final that follows speech-ended", async () => { + const bridge = new FakeNativeBridge(); + Object.defineProperty(window, "zer0Voice", { configurable: true, value: bridge }); + const { submit, surface } = await render(); + await act(async () => surface.click()); + await act(async () => bridge.emit("availability", "on-device")); + await act(async () => bridge.emit("ready")); + await act(async () => bridge.emit("ended", "speech-ended")); + await act(async () => bridge.emit("final", "Android final")); + await act(async () => vi.advanceTimersByTime(500)); + expect(submit).toHaveBeenCalledOnce(); + expect(submit).toHaveBeenCalledWith("Android final"); + expect(FakeRecognition.instances).toHaveLength(0); + }); + + it("falls back if native stop never produces a final or ended event", async () => { + const bridge = new FakeNativeBridge(); + Object.defineProperty(window, "zer0Voice", { configurable: true, value: bridge }); + const { surface } = await render(); + await act(async () => surface.click()); + await act(async () => bridge.emit("availability", "on-device")); + await act(async () => bridge.emit("ready")); + await act(async () => surface.click()); + expect(bridge.messages.at(-1)).toBe(JSON.stringify({ version: 1, command: "stop" })); + await act(async () => vi.advanceTimersByTime(1_000)); + expect(FakeRecognition.instances).toHaveLength(1); + }); + + it("cancels the native bridge and detaches its event handler on unmount", async () => { + const bridge = new FakeNativeBridge(); + Object.defineProperty(window, "zer0Voice", { configurable: true, value: bridge }); + const { surface } = await render(); + await act(async () => surface.click()); + await act(async () => root.unmount()); + expect(bridge.messages.at(-1)).toBe(JSON.stringify({ version: 1, command: "cancel" })); + expect(bridge.onmessage).toBeNull(); + root = createRoot(host); + }); + }); diff --git a/web/src/components/ChatVoiceControl.tsx b/web/src/components/ChatVoiceControl.tsx index 52c6449979195..851cf52ad59d5 100644 --- a/web/src/components/ChatVoiceControl.tsx +++ b/web/src/components/ChatVoiceControl.tsx @@ -5,6 +5,8 @@ import { Button } from "@nous-research/ui/ui/components/button"; const RESTART_DELAY_MS = 250; const MAX_RESTARTS = 6; +const NATIVE_HANDSHAKE_TIMEOUT_MS = 1_000; +const NATIVE_FINAL_GRACE_MS = 250; type VoiceState = "idle" | "listening" | "sent" | "error"; @@ -29,12 +31,24 @@ interface SpeechRecognitionLike { abort(): void; } +interface NativeVoiceBridge { + postMessage(message: string): void; + onmessage: ((event: MessageEvent) => void) | null; +} + +interface NativeVoiceEvent { + version: 1; + event: "availability" | "ready" | "listening" | "partial" | "final" | "error" | "ended"; + text?: string; +} + type SpeechRecognitionConstructor = new () => SpeechRecognitionLike; declare global { interface Window { SpeechRecognition?: SpeechRecognitionConstructor; webkitSpeechRecognition?: SpeechRecognitionConstructor; + zer0Voice?: NativeVoiceBridge; } } @@ -61,12 +75,26 @@ export function ChatVoiceControl({ connected, submit, onBargeIn }: ChatVoiceCont const generationRef = useRef(0); const finalRef = useRef(""); const interimRef = useRef(""); + const nativeRef = useRef(null); + const nativeTimerRef = useRef(null); + const nativeAvailableRef = useRef(false); + const nativeSelectedRef = useRef(false); + const submittedRef = useRef(false); const clearRestart = useCallback(() => { if (restartTimerRef.current !== null) window.clearTimeout(restartTimerRef.current); restartTimerRef.current = null; }, []); + const clearNativeTimer = useCallback(() => { + if (nativeTimerRef.current !== null) window.clearTimeout(nativeTimerRef.current); + nativeTimerRef.current = null; + }, []); + + const sendNativeCommand = (command: "check" | "start" | "stop" | "cancel") => { + nativeRef.current?.postMessage(JSON.stringify({ version: 1, command })); + }; + const updateFinal = (value: string) => { finalRef.current = value; setFinalTranscript(value); @@ -142,6 +170,82 @@ export function ChatVoiceControl({ connected, submit, onBargeIn }: ChatVoiceCont } } + function fallbackToBrowser(generation: number) { + if (generation !== generationRef.current || !listeningRef.current) return; + clearNativeTimer(); + const bridge = nativeRef.current; + nativeRef.current = null; + nativeAvailableRef.current = false; + nativeSelectedRef.current = false; + if (bridge) { + bridge.onmessage = null; + bridge.postMessage(JSON.stringify({ version: 1, command: "cancel" })); + } + startRecognition(generation); + } + + function startNative(bridge: NativeVoiceBridge, generation: number) { + nativeRef.current = bridge; + nativeAvailableRef.current = false; + nativeSelectedRef.current = false; + bridge.onmessage = (message) => { + if (generation !== generationRef.current || !listeningRef.current) return; + let event: NativeVoiceEvent; + try { + event = JSON.parse(message.data) as NativeVoiceEvent; + } catch { + fallbackToBrowser(generation); + return; + } + if (event.version !== 1) return; + switch (event.event) { + case "availability": + if (event.text !== "on-device") fallbackToBrowser(generation); + else { + nativeAvailableRef.current = true; + sendNativeCommand("start"); + } + break; + case "ready": + case "listening": + if (!nativeAvailableRef.current) return; + nativeSelectedRef.current = true; + clearNativeTimer(); + setError(""); + setState("listening"); + break; + case "partial": + if (nativeSelectedRef.current) updateInterim(event.text ?? ""); + break; + case "final": { + const transcript = (event.text ?? "").replace(/\s+/g, " ").trim(); + if (!nativeSelectedRef.current || !transcript || submittedRef.current) return; + submittedRef.current = true; + listeningRef.current = false; + generationRef.current += 1; + clearNativeTimer(); + bridge.onmessage = null; + nativeRef.current = null; + updateFinal(""); + updateInterim(""); + submit(transcript); + setError(""); + setState("sent"); + break; + } + case "error": + fallbackToBrowser(generation); + break; + case "ended": + clearNativeTimer(); + nativeTimerRef.current = window.setTimeout(() => fallbackToBrowser(generation), NATIVE_FINAL_GRACE_MS); + break; + } + }; + sendNativeCommand("check"); + nativeTimerRef.current = window.setTimeout(() => fallbackToBrowser(generation), NATIVE_HANDSHAKE_TIMEOUT_MS); + } + const begin = () => { if (!connected) { setError("Chat is not connected yet."); @@ -152,15 +256,25 @@ export function ChatVoiceControl({ connected, submit, onBargeIn }: ChatVoiceCont clearRestart(); fatalRef.current = false; restartCountRef.current = 0; + submittedRef.current = false; listeningRef.current = true; const generation = generationRef.current + 1; generationRef.current = generation; setError(""); setState("listening"); - startRecognition(generation); + const bridge = window.zer0Voice; + if (bridge?.postMessage) startNative(bridge, generation); + else startRecognition(generation); }; const commit = () => { + if (nativeRef.current && nativeSelectedRef.current) { + sendNativeCommand("stop"); + clearNativeTimer(); + const generation = generationRef.current; + nativeTimerRef.current = window.setTimeout(() => fallbackToBrowser(generation), NATIVE_HANDSHAKE_TIMEOUT_MS); + return; + } const transcript = `${finalRef.current} ${interimRef.current}`.replace(/\s+/g, " ").trim(); if (!transcript) { setError("No speech heard yet — keep talking, then tap anywhere to send."); @@ -182,13 +296,22 @@ export function ChatVoiceControl({ connected, submit, onBargeIn }: ChatVoiceCont listeningRef.current = false; generationRef.current += 1; clearRestart(); + clearNativeTimer(); + const bridge = nativeRef.current; + nativeRef.current = null; + nativeAvailableRef.current = false; + nativeSelectedRef.current = false; + if (bridge) { + bridge.onmessage = null; + bridge.postMessage(JSON.stringify({ version: 1, command: "cancel" })); + } recognitionRef.current?.abort(); recognitionRef.current = null; updateFinal(""); updateInterim(""); setError(""); setState("idle"); - }, [clearRestart]); + }, [clearNativeTimer, clearRestart]); useEffect(() => cancel, [cancel]); From af53edc41c6bced7311755fd88ea60bad76b52cb Mon Sep 17 00:00:00 2001 From: lesseradmin Date: Tue, 25 Aug 2026 21:35:24 -0500 Subject: [PATCH 6/7] fix(web): reject stale Android voice callbacks --- web/src/components/ChatVoiceControl.test.tsx | 29 ++++++++++++++++++++ web/src/components/ChatVoiceControl.tsx | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/web/src/components/ChatVoiceControl.test.tsx b/web/src/components/ChatVoiceControl.test.tsx index 1b9084cfb8942..13fa0e545e56c 100644 --- a/web/src/components/ChatVoiceControl.test.tsx +++ b/web/src/components/ChatVoiceControl.test.tsx @@ -232,6 +232,35 @@ describe("ChatVoiceControl browser speech input", () => { expect(bridge.messages.at(-1)).toBe(JSON.stringify({ version: 1, command: "cancel" })); }); + it("rejects every retained native callback after timeout fallback", async () => { + const bridge = new FakeNativeBridge(); + Object.defineProperty(window, "zer0Voice", { configurable: true, value: bridge }); + const { submit, surface } = await render(); + await act(async () => surface.click()); + const staleHandler = bridge.onmessage; + + await act(async () => vi.advanceTimersByTime(1_000)); + expect(FakeRecognition.instances).toHaveLength(1); + const displayAfterFallback = host.textContent; + + for (const [event, text] of [ + ["availability", "on-device"], + ["ready", undefined], + ["partial", "stale partial"], + ["final", "stale final"], + ["error", "stale error"], + ] as const) { + await act(async () => staleHandler?.(new MessageEvent("message", { + data: JSON.stringify({ version: 1, event, ...(text === undefined ? {} : { text }) }), + }))); + } + + expect(host.textContent).toBe(displayAfterFallback); + expect(submit).not.toHaveBeenCalled(); + expect(FakeRecognition.instances).toHaveLength(1); + expect(FakeRecognition.instances[0].start).toHaveBeenCalledOnce(); + }); + it.each(["ended", "error"])("falls back when native emits %s while listening", async (event) => { const bridge = new FakeNativeBridge(); Object.defineProperty(window, "zer0Voice", { configurable: true, value: bridge }); diff --git a/web/src/components/ChatVoiceControl.tsx b/web/src/components/ChatVoiceControl.tsx index 851cf52ad59d5..c39fb9020dd28 100644 --- a/web/src/components/ChatVoiceControl.tsx +++ b/web/src/components/ChatVoiceControl.tsx @@ -189,7 +189,7 @@ export function ChatVoiceControl({ connected, submit, onBargeIn }: ChatVoiceCont nativeAvailableRef.current = false; nativeSelectedRef.current = false; bridge.onmessage = (message) => { - if (generation !== generationRef.current || !listeningRef.current) return; + if (nativeRef.current !== bridge || generation !== generationRef.current || !listeningRef.current) return; let event: NativeVoiceEvent; try { event = JSON.parse(message.data) as NativeVoiceEvent; From f843f20d9ddb7d1ac7f74e6d6fb754413cb58c76 Mon Sep 17 00:00:00 2001 From: lesseradmin Date: Fri, 28 Aug 2026 16:44:38 -0500 Subject: [PATCH 7/7] fix(web): preserve native voice on malformed payloads --- web/src/components/ChatVoiceControl.test.tsx | 24 ++++++++++++++++++++ web/src/components/ChatVoiceControl.tsx | 4 ++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/web/src/components/ChatVoiceControl.test.tsx b/web/src/components/ChatVoiceControl.test.tsx index 13fa0e545e56c..eafdae115badc 100644 --- a/web/src/components/ChatVoiceControl.test.tsx +++ b/web/src/components/ChatVoiceControl.test.tsx @@ -75,6 +75,15 @@ describe("ChatVoiceControl browser speech input", () => { expect(submit).not.toHaveBeenCalled(); }); + it("uses the browser locale for speech recognition", async () => { + vi.stubGlobal("navigator", { language: "fr-CA" }); + const { surface } = await render(); + + await act(async () => surface.click()); + + expect(FakeRecognition.instances[0].lang).toBe("fr-CA"); + }); + it("restarts Android premature onend with bounded exponential backoff and no submit", async () => { const { submit, surface } = await render(); await act(async () => surface.click()); @@ -198,6 +207,21 @@ describe("ChatVoiceControl browser speech input", () => { expect(FakeRecognition.instances).toHaveLength(1); }); + it("logs and ignores a malformed native bridge payload", async () => { + const bridge = new FakeNativeBridge(); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + Object.defineProperty(window, "zer0Voice", { configurable: true, value: bridge }); + const { surface } = await render(); + + await act(async () => surface.click()); + await act(async () => bridge.onmessage?.(new MessageEvent("message", { data: "not-json" }))); + + expect(warn).toHaveBeenCalledWith("[voice] Ignoring malformed native bridge payload"); + expect(FakeRecognition.instances).toHaveLength(0); + await act(async () => bridge.emit("availability", "on-device")); + expect(bridge.messages.at(-1)).toBe(JSON.stringify({ version: 1, command: "start" })); + }); + it("uses confirmed native partials for display and submits a final exactly once", async () => { const bridge = new FakeNativeBridge(); Object.defineProperty(window, "zer0Voice", { configurable: true, value: bridge }); diff --git a/web/src/components/ChatVoiceControl.tsx b/web/src/components/ChatVoiceControl.tsx index c39fb9020dd28..d472e54e4122e 100644 --- a/web/src/components/ChatVoiceControl.tsx +++ b/web/src/components/ChatVoiceControl.tsx @@ -118,7 +118,7 @@ export function ChatVoiceControl({ connected, submit, onBargeIn }: ChatVoiceCont const recognition = new Constructor(); recognition.continuous = true; recognition.interimResults = true; - recognition.lang = "en-US"; + recognition.lang = navigator.language; recognition.onresult = (event) => { if (generation !== generationRef.current || !listeningRef.current) return; let appendedFinal = ""; @@ -194,7 +194,7 @@ export function ChatVoiceControl({ connected, submit, onBargeIn }: ChatVoiceCont try { event = JSON.parse(message.data) as NativeVoiceEvent; } catch { - fallbackToBrowser(generation); + console.warn("[voice] Ignoring malformed native bridge payload"); return; } if (event.version !== 1) return;