From 221f9bcd12a05301810c5e2373dd3325ce1fa6b6 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:19:36 +0200 Subject: [PATCH 01/10] fix(mobile): queue voice input after leaving a thread --- .../src/voice-input/controller.test.ts | 97 +++++++++++++++++-- .../src/voice-input/controller.ts | 62 ++++++++---- 2 files changed, 129 insertions(+), 30 deletions(-) diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts index 5f26b882b694..c6ce8b3af9c2 100644 --- a/packages/client-runtime/src/voice-input/controller.test.ts +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -298,11 +298,21 @@ describe("VoiceInputController", () => { }); it.each(["cancel", "dispose", "ownerChanged"] as const)( - "holds the session after %s until non-abortable transcription settles", + "queues the next recording after %s until transcription and audio cleanup settle", async (action) => { const transcription = deferred(); const transcriptionEntered = deferred(); + const audioRelease = deferred(); + const audioReleaseEntered = deferred(); + const releaseRecording = vi + .fn<() => Promise>() + .mockRejectedValueOnce(new Error("busy")) + .mockImplementationOnce(() => { + audioReleaseEntered.resolve(undefined); + return audioRelease.promise; + }); const harness = createHarness({ + releaseRecording, getTranscriber: () => ({ prepare: async () => preparedTranscription((_uri, { signal }) => { @@ -321,20 +331,31 @@ describe("VoiceInputController", () => { harness.controller[action](); expect(signal.aborted).toBe(true); - const next = createHarness(); - await next.controller.start(); - expect(next.controller.currentState.error).toContain("already active"); + const prepare = vi.fn(async () => preparedTranscription()); + const next = createHarness({ getTranscriber: () => ({ prepare }) }); + const nextStart = next.controller.start(); + expect(next.controller.currentState).toEqual({ + phase: "preparing", + error: null, + errorAction: null, + }); + expect(prepare).not.toHaveBeenCalled(); expect(next.recorder.record).not.toHaveBeenCalled(); transcription.resolve("late text"); + await audioReleaseEntered.promise; + expect(prepare).not.toHaveBeenCalled(); + expect(next.recorder.prepareToRecordAsync).not.toHaveBeenCalled(); + audioRelease.resolve(undefined); await stopping; + await nextStart; expect(harness.commits).toEqual([]); expect(harness.deleted).toEqual(["file:///voice.m4a"]); expect(harness.controller.currentState.phase).toBe("idle"); - await next.controller.start(); expect(next.controller.currentState.phase).toBe("recording"); + expect(prepare).toHaveBeenCalledTimes(1); await next.controller.interruptRecording(); }, ); @@ -431,7 +452,7 @@ describe("VoiceInputController", () => { expect(harness.controller.currentState.error).toContain("draft changed"); }); - it("keeps the app-wide session locked until canceled preparation settles", async () => { + it("queues the next recording until canceled preparation settles", async () => { const preparation = deferred(); const preparationEntered = deferred(); const first = createHarness({ @@ -447,15 +468,71 @@ describe("VoiceInputController", () => { first.controller.cancel(); expect(signal.aborted).toBe(true); - const blocked = createHarness(); - await blocked.controller.start(); - expect(blocked.controller.currentState.error).toContain("already active"); + const next = createHarness(); + const nextStart = next.controller.start(); + expect(next.controller.currentState.phase).toBe("preparing"); + expect(next.recorder.prepareToRecordAsync).not.toHaveBeenCalled(); preparation.resolve(preparedTranscription()); await firstStart; expect(first.recorder.record).not.toHaveBeenCalled(); - blocked.controller.cancel(); + await nextStart; + expect(next.controller.currentState.phase).toBe("recording"); + await next.controller.interruptRecording(); + }); + + it("still rejects a second start while the first recording is active", async () => { + const first = createHarness(); + await first.controller.start(); + const next = createHarness(); + await next.controller.start(); + expect(next.controller.currentState.error).toContain("already active"); + expect(next.recorder.record).not.toHaveBeenCalled(); + await first.controller.interruptRecording(); + }); + + it("does not start a canceled waiter or let it release another recording", async () => { + const preparation = deferred(); + const preparationEntered = deferred(); + const first = createHarness({ + getTranscriber: () => ({ + prepare: () => { + preparationEntered.resolve(undefined); + return preparation.promise; + }, + }), + }); + const firstStart = first.controller.start(); + await preparationEntered.promise; + first.controller.dispose(); + const next = createHarness(); + const canceledStart = next.controller.start(); + next.controller.cancel(); + const nextStart = next.controller.start(); + preparation.resolve(preparedTranscription()); + await Promise.all([firstStart, canceledStart, nextStart]); + + expect(next.recorder.record).toHaveBeenCalledTimes(1); + expect(next.deleted).toEqual([]); + const blocked = createHarness(); + await blocked.controller.start(); + expect(blocked.controller.currentState.error).toContain("already active"); + await next.controller.interruptRecording(); + }); + + it("releases an abandoned recording even if its native recorder was already disposed", async () => { + const harness = createHarness(); + await harness.controller.start(); + harness.recorder.stop.mockRejectedValueOnce(new Error("recorder released")); + Object.defineProperty(harness.recorder, "uri", { + get() { + throw new Error("recorder released"); + }, + }); + + await expect(harness.controller.interruptRecording()).resolves.toBeUndefined(); + expect(harness.deleted).toEqual(["file:///voice.m4a"]); const next = createHarness(); await next.controller.start(); expect(next.controller.currentState.phase).toBe("recording"); diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts index cb284ad9f85b..0e3c59d43c73 100644 --- a/packages/client-runtime/src/voice-input/controller.ts +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -121,18 +121,28 @@ export function resolveTranscriptCommit( }; } -let activeSession: symbol | null = null; +type VoiceInputSession = { + abandoned: boolean; + readonly released: Promise; + readonly release: () => void; +}; + +let activeSession: VoiceInputSession | null = null; let activeTranscriptionOperation: Promise | null = null; -function acquireSession(): symbol | null { +function acquireSession(): VoiceInputSession | null { if (activeSession) return null; - const token = Symbol("voice-input-session"); + const { promise, resolve } = Promise.withResolvers(); + const token = { abandoned: false, released: promise, release: resolve }; activeSession = token; return token; } -function releaseSession(token: symbol | null): void { - if (token && activeSession === token) activeSession = null; +function releaseSession(token: VoiceInputSession | null): void { + if (token && activeSession === token) { + activeSession = null; + token.release(); + } } async function runTranscriptionOperation(operation: () => Promise): Promise { @@ -177,7 +187,7 @@ export class VoiceInputController { private readonly dependencies: VoiceInputControllerDependencies; private state: VoiceInputState = IDLE_STATE; private operationToken = 0; - private sessionToken: symbol | null = null; + private sessionToken: VoiceInputSession | null = null; private transcription: PreparedVoiceTranscription | null = null; private transcriptionAbortController: AbortController | null = null; private capturedDraft: VoiceDraftSnapshot | null = null; @@ -201,19 +211,28 @@ export class VoiceInputController { this.setError("This draft is no longer available.", "retry"); return; } - const sessionToken = acquireSession(); - if (!sessionToken) { - this.setError("Another voice recording is already active.", "retry"); - return; - } - - this.sessionToken = sessionToken; + let sessionToken: VoiceInputSession | null = null; const operationToken = ++this.operationToken; const abortController = new AbortController(); this.transcriptionAbortController = abortController; this.setState({ phase: "preparing", error: null, errorAction: null }); try { + for ( + let previousSession = activeSession; + previousSession?.abandoned; + previousSession = activeSession + ) { + await previousSession.released; + if (!this.isCurrent(operationToken)) return; + } + sessionToken = acquireSession(); + if (!sessionToken) { + this.setError("Another voice recording is already active.", "retry"); + return; + } + this.sessionToken = sessionToken; + const transcriber = this.dependencies.getTranscriber(); if (!transcriber) { this.setError("Voice transcription is not available.", null); @@ -260,10 +279,12 @@ export class VoiceInputController { if (this.isCurrent(operationToken)) this.setError("Could not start voice recording.", "retry"); } finally { - if (this.isCurrent(operationToken) && this.state.phase === "error") { - await this.releaseResources(); - } else if (!this.isCurrent(operationToken) && !this.finishing) { - await this.releaseResources(); + if (sessionToken && this.sessionToken === sessionToken) { + if (this.isCurrent(operationToken) && this.state.phase === "error") { + await this.releaseResources(); + } else if (!this.isCurrent(operationToken) && !this.finishing) { + await this.releaseResources(); + } } } } @@ -426,9 +447,7 @@ export class VoiceInputController { ); try { await this.dependencies.recorder.stop(); - this.rememberRecordingUri(this.dependencies.recorder.uri); } catch { - this.rememberRecordingUri(this.dependencies.recorder.uri); } finally { await this.releaseResources(); } @@ -436,7 +455,9 @@ export class VoiceInputController { private async releaseResources(): Promise { this.rememberRecordingUri(this.recordingUri); - this.rememberRecordingUri(this.dependencies.recorder.uri); + try { + this.rememberRecordingUri(this.dependencies.recorder.uri); + } catch {} this.recordingUri = null; for (const uri of this.ownedRecordingUris) { try { @@ -470,6 +491,7 @@ export class VoiceInputController { private invalidateOperation(): void { this.operationToken += 1; + if (this.sessionToken) this.sessionToken.abandoned = true; this.transcriptionAbortController?.abort(); } From bed03ca986ccfb001da47c17af5b44d063021465 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:24:00 +0200 Subject: [PATCH 02/10] fix(mobile): create abort handle after voice session handoff --- .../src/voice-input/controller.test.ts | 28 +++++++++++++++++++ .../src/voice-input/controller.ts | 4 +-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts index c6ce8b3af9c2..d73c8e7505ba 100644 --- a/packages/client-runtime/src/voice-input/controller.test.ts +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -539,6 +539,34 @@ describe("VoiceInputController", () => { await next.controller.interruptRecording(); }); + it("keeps the new abort controller when restarting the same composer after cancellation", async () => { + const transcription = deferred(); + const transcriptionEntered = deferred(); + const prepare = vi + .fn() + .mockResolvedValueOnce( + preparedTranscription(() => { + transcriptionEntered.resolve(undefined); + return transcription.promise; + }), + ) + .mockResolvedValue(preparedTranscription()); + const harness = createHarness({ getTranscriber: () => ({ prepare }) }); + await harness.controller.start(); + const stopping = harness.controller.stop(); + await transcriptionEntered.promise; + harness.controller.cancel(); + const restarting = harness.controller.start(); + transcription.resolve("late text"); + await Promise.all([stopping, restarting]); + await harness.controller.stop(); + + expect(harness.commits).toEqual([ + { text: "hello new text", selection: { start: 14, end: 14 } }, + ]); + expect(harness.controller.currentState.phase).toBe("idle"); + }); + it("does not start the microphone for an owner that changed during preparation", async () => { const preparation = deferred(); const preparationEntered = deferred(); diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts index 0e3c59d43c73..80a8a528f335 100644 --- a/packages/client-runtime/src/voice-input/controller.ts +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -213,8 +213,6 @@ export class VoiceInputController { } let sessionToken: VoiceInputSession | null = null; const operationToken = ++this.operationToken; - const abortController = new AbortController(); - this.transcriptionAbortController = abortController; this.setState({ phase: "preparing", error: null, errorAction: null }); try { @@ -232,6 +230,8 @@ export class VoiceInputController { return; } this.sessionToken = sessionToken; + const abortController = new AbortController(); + this.transcriptionAbortController = abortController; const transcriber = this.dependencies.getTranscriber(); if (!transcriber) { From ac78b45202a8d4080490417cd31acd541e7e0d3e Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 04:35:08 +0200 Subject: [PATCH 03/10] fix(mobile): keep dictation active across navigation --- apps/mobile/src/App.tsx | 25 +- .../SettingsScheduledTasksRouteScreen.tsx | 52 ++-- .../scheduled-task-editor-state.test.ts | 57 ++++ .../settings/scheduled-task-editor-state.ts | 55 ++++ .../settings/scheduled-task-editor.tsx | 64 +++-- .../features/threads/NewTaskDraftScreen.tsx | 3 +- .../src/features/threads/ThreadComposer.tsx | 9 +- .../voice-input/GlobalVoiceInputControl.tsx | 47 ++++ .../voice-input/VoiceInputProvider.tsx | 222 +++++++++++++++ .../voice-input/useVoiceInputController.ts | 253 +++--------------- .../voice-input/voiceInputSession.test.ts | 155 +++++++++++ .../features/voice-input/voiceInputSession.ts | 55 ++++ 12 files changed, 725 insertions(+), 272 deletions(-) create mode 100644 apps/mobile/src/features/settings/scheduled-task-editor-state.test.ts create mode 100644 apps/mobile/src/features/settings/scheduled-task-editor-state.ts create mode 100644 apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx create mode 100644 apps/mobile/src/features/voice-input/VoiceInputProvider.tsx create mode 100644 apps/mobile/src/features/voice-input/voiceInputSession.test.ts create mode 100644 apps/mobile/src/features/voice-input/voiceInputSession.ts diff --git a/apps/mobile/src/App.tsx b/apps/mobile/src/App.tsx index 001c9d597cc8..d30eea1d286d 100644 --- a/apps/mobile/src/App.tsx +++ b/apps/mobile/src/App.tsx @@ -23,6 +23,7 @@ import { OverlayPortalHost } from "./components/OverlayPortal"; import { shouldHandleAppLink } from "./lib/appLinking"; import { useMobileNavigationTheme } from "./lib/useMobileNavigationTheme"; import { SubscriptionUsageCoordinator } from "./widgets/SubscriptionUsageCoordinator"; +import { VoiceInputProvider } from "./features/voice-input/VoiceInputProvider"; import "../global.css"; @@ -76,22 +77,24 @@ function AppContent() { - - {/* The navigation theme drives the NATIVE header appearance: native-stack + + + {/* The navigation theme drives the NATIVE header appearance: native-stack forwards `dark` as the nav bar's overrideUserInterfaceStyle. Without this, React Navigation defaults to its light theme and every native header (glass buttons, title, materials) is forced light even when the system is in dark mode. */} - - - - - - - - {/* Anchored-menu overlays render here — in-window, so the + + + + + + + + {/* Anchored-menu overlays render here — in-window, so the keyboard stays up while a dropdown is open. */} - + + diff --git a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx index 91b1d0923e6c..c8925d1baa66 100644 --- a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx @@ -15,7 +15,6 @@ import { import { useCallback, useEffect, - useId, useLayoutEffect, useMemo, useRef, @@ -372,14 +371,21 @@ export function SettingsScheduledTaskEditRouteScreen() { } function SettingsScheduledTaskEditorScreen({ title }: { readonly title: string }) { - const { editor, setEditor, hasChanges, draftForEnvironment } = useScheduledTaskEditor(); + const { + editor, + voiceOwnerKey, + readEditor, + setEditor, + startEditor, + hasChanges, + draftForEnvironment, + } = useScheduledTaskEditor(); const { availableTargets } = useSettingsEnvironmentFilter(); const navigation = useNavigation(); const insets = useSafeAreaInsets(); const [saving, setSaving] = useState(false); const [saved, setSaved] = useState(false); - const voiceOwnerId = useId(); - const ownerKey = editor ? `${voiceOwnerId}:${editor.environmentId}` : null; + const ownerKey = editor ? `${voiceOwnerKey}:${editor.environmentId}` : null; const prompt = editor?.draft.prompt ?? ""; const [selectionState, setSelectionState] = useState<{ readonly ownerKey: string | null; @@ -402,32 +408,36 @@ function SettingsScheduledTaskEditorScreen({ title }: { readonly title: string } ); const voiceInput = useVoiceInputController({ ownerKey, - draftMessage: prompt, - selection, + readDraftMessage: () => { + const current = readEditor(); + return current?.environmentId === editor?.environmentId + ? (current?.draft.prompt ?? null) + : null; + }, onChangeSelection: setSelection, - onChangeDraftMessage: setPrompt, + onChangeDraftMessage: (text) => { + if (readEditor()?.environmentId === editor?.environmentId) setPrompt(text); + }, disabled: saving, }); - const preventRemove = !saved && (hasChanges || saving || voiceInput.isBusy); + const preventRemove = !saved && (hasChanges || saving); usePreventRemove(preventRemove, ({ data }) => { if (saving) { Alert.alert("Saving task", "Wait for the task to finish saving before leaving."); return; } - Alert.alert( - "Discard changes?", - voiceInput.isBusy - ? "Your dictation and unsaved changes will be lost." - : "Your unsaved changes will be lost.", - [ - { text: "Keep editing", style: "cancel" }, - { - text: "Discard changes", - style: "destructive", - onPress: () => navigation.dispatch(data.action), + Alert.alert("Discard changes?", "Your unsaved changes will be lost.", [ + { text: "Keep editing", style: "cancel" }, + { + text: "Discard changes", + style: "destructive", + onPress: () => { + if (voiceInput.isBusy) voiceInput.cancel(); + startEditor(null); + navigation.dispatch(data.action); }, - ], - ); + }, + ]); }); useEffect(() => { if (!saved) return; diff --git a/apps/mobile/src/features/settings/scheduled-task-editor-state.test.ts b/apps/mobile/src/features/settings/scheduled-task-editor-state.test.ts new file mode 100644 index 000000000000..97a7d38a44bf --- /dev/null +++ b/apps/mobile/src/features/settings/scheduled-task-editor-state.test.ts @@ -0,0 +1,57 @@ +import { EnvironmentId } from "@t3tools/contracts"; +import { afterEach, describe, expect, it } from "vite-plus/test"; + +import { appAtomRegistry } from "../../state/atom-registry"; +import { createDraft } from "./scheduledTaskDraft"; +import { + readScheduledTaskEditor, + scheduledTaskEditorSessionAtom, + startScheduledTaskEditor, + updateScheduledTaskEditor, +} from "./scheduled-task-editor-state"; + +afterEach(() => appAtomRegistry.set(scheduledTaskEditorSessionAtom, null)); + +describe("scheduled task voice draft", () => { + const editor = { + environmentId: EnvironmentId.make("environment"), + environmentLabel: "Environment", + draft: createDraft(null, null), + }; + + it("keeps the prompt outside the screen and retains its original change baseline", () => { + startScheduledTaskEditor(editor); + const original = appAtomRegistry.get(scheduledTaskEditorSessionAtom)!; + const ownerKey = `scheduled-task:${original.id}`; + updateScheduledTaskEditor( + (current) => + current && { + ...current, + draft: { ...current.draft, prompt: "spoken prompt" }, + }, + null, + ); + + expect(readScheduledTaskEditor(ownerKey, null)?.draft.prompt).toBe("spoken prompt"); + expect(appAtomRegistry.get(scheduledTaskEditorSessionAtom)?.initial).toBe(editor); + }); + + it("rejects an old voice target when another task opens in the same environment", () => { + startScheduledTaskEditor(editor); + const original = appAtomRegistry.get(scheduledTaskEditorSessionAtom)!; + startScheduledTaskEditor({ ...editor, draft: { ...editor.draft, prompt: "other task" } }); + + expect(readScheduledTaskEditor(`scheduled-task:${original.id}`, editor)).toBeNull(); + expect(appAtomRegistry.get(scheduledTaskEditorSessionAtom)?.current?.draft.prompt).toBe( + "other task", + ); + }); + + it("does not restore the default prompt after the editor is discarded", () => { + startScheduledTaskEditor(null); + const session = appAtomRegistry.get(scheduledTaskEditorSessionAtom)!; + expect(readScheduledTaskEditor(`scheduled-task:${session.id}`, editor)).toBeNull(); + updateScheduledTaskEditor((current) => current, editor); + expect(appAtomRegistry.get(scheduledTaskEditorSessionAtom)?.current).toBeNull(); + }); +}); diff --git a/apps/mobile/src/features/settings/scheduled-task-editor-state.ts b/apps/mobile/src/features/settings/scheduled-task-editor-state.ts new file mode 100644 index 000000000000..3ac81c08b13d --- /dev/null +++ b/apps/mobile/src/features/settings/scheduled-task-editor-state.ts @@ -0,0 +1,55 @@ +import type { EnvironmentId } from "@t3tools/contracts"; +import { Atom } from "effect/unstable/reactivity"; + +import { appAtomRegistry } from "../../state/atom-registry"; +import type { ScheduledTaskDraft } from "./scheduledTaskDraft"; + +export type ScheduledTaskEditor = { + readonly environmentId: EnvironmentId; + readonly environmentLabel: string; + readonly draft: ScheduledTaskDraft; +}; + +type ScheduledTaskEditorSession = { + readonly id: number; + readonly initial: ScheduledTaskEditor | null; + readonly current: ScheduledTaskEditor | null; +}; + +export const scheduledTaskEditorSessionAtom = Atom.make( + null, +).pipe(Atom.keepAlive); +let nextEditorId = 0; + +export function startScheduledTaskEditor(next: ScheduledTaskEditor | null): void { + appAtomRegistry.set(scheduledTaskEditorSessionAtom, { + id: ++nextEditorId, + initial: next, + current: next, + }); +} + +export function updateScheduledTaskEditor( + update: + | ScheduledTaskEditor + | null + | ((current: ScheduledTaskEditor | null) => ScheduledTaskEditor | null), + defaultEditor: ScheduledTaskEditor | null, +): void { + const session = appAtomRegistry.get(scheduledTaskEditorSessionAtom); + const previous = session ? session.current : defaultEditor; + appAtomRegistry.set(scheduledTaskEditorSessionAtom, { + id: session?.id ?? 0, + initial: session ? session.initial : previous, + current: typeof update === "function" ? update(previous) : update, + }); +} + +export function readScheduledTaskEditor( + voiceOwnerKey: string, + defaultEditor: ScheduledTaskEditor | null, +): ScheduledTaskEditor | null { + const session = appAtomRegistry.get(scheduledTaskEditorSessionAtom); + if (`scheduled-task:${session?.id ?? 0}` !== voiceOwnerKey) return null; + return session ? session.current : defaultEditor; +} diff --git a/apps/mobile/src/features/settings/scheduled-task-editor.tsx b/apps/mobile/src/features/settings/scheduled-task-editor.tsx index f5d8337beeb4..265f42e0059d 100644 --- a/apps/mobile/src/features/settings/scheduled-task-editor.tsx +++ b/apps/mobile/src/features/settings/scheduled-task-editor.tsx @@ -1,10 +1,10 @@ +import { useAtomValue } from "@effect/atom-react"; import type { EnvironmentId } from "@t3tools/contracts"; import { createContext, use, useMemo, useCallback, - useState, type Dispatch, type ReactNode, type SetStateAction, @@ -20,14 +20,18 @@ import { import { useSettingsEnvironmentFilter } from "./settings-environment-filter"; import { settingsTargetsForProject } from "./settings-environment-filter.logic"; -type ScheduledTaskEditor = { - readonly environmentId: EnvironmentId; - readonly environmentLabel: string; - readonly draft: ScheduledTaskDraft; -}; +import { + scheduledTaskEditorSessionAtom, + startScheduledTaskEditor, + updateScheduledTaskEditor, + readScheduledTaskEditor, + type ScheduledTaskEditor, +} from "./scheduled-task-editor-state"; const ScheduledTaskEditorContext = createContext<{ readonly editor: ScheduledTaskEditor | null; + readonly voiceOwnerKey: string; + readonly readEditor: () => ScheduledTaskEditor | null; readonly setEditor: Dispatch>; readonly startEditor: (editor: ScheduledTaskEditor | null) => void; readonly hasChanges: boolean; @@ -76,12 +80,10 @@ export function ScheduledTaskEditorProvider({ children }: { readonly children: R : null, [defaultTarget, draftForEnvironment], ); - const [session, setSession] = useState<{ - readonly initial: ScheduledTaskEditor | null; - readonly current: ScheduledTaskEditor | null; - } | null>(null); - const editor = session?.current ?? defaultEditor; - const initial = session?.initial ?? defaultEditor; + const session = useAtomValue(scheduledTaskEditorSessionAtom); + const voiceOwnerKey = `scheduled-task:${session?.id ?? 0}`; + const editor = session ? session.current : defaultEditor; + const initial = session ? session.initial : defaultEditor; const hasChanges = Boolean( initial && editor && @@ -90,23 +92,37 @@ export function ScheduledTaskEditorProvider({ children }: { readonly children: R ); const setEditor = useCallback>>( (update) => { - setSession((current) => { - const previous = current?.current ?? defaultEditor; - return { - initial: current?.initial ?? previous, - current: typeof update === "function" ? update(previous) : update, - }; - }); + updateScheduledTaskEditor(update, defaultEditor); }, [defaultEditor], ); - const startEditor = useCallback((next: ScheduledTaskEditor | null) => { - setSession({ initial: next, current: next }); - }, []); + const startEditor = startScheduledTaskEditor; const resetEditor = useCallback(() => startEditor(defaultEditor), [defaultEditor, startEditor]); + const readEditor = useCallback( + () => readScheduledTaskEditor(voiceOwnerKey, defaultEditor), + [defaultEditor, voiceOwnerKey], + ); const value = useMemo( - () => ({ editor, setEditor, startEditor, hasChanges, resetEditor, draftForEnvironment }), - [editor, setEditor, startEditor, hasChanges, resetEditor, draftForEnvironment], + () => ({ + editor, + voiceOwnerKey, + readEditor, + setEditor, + startEditor, + hasChanges, + resetEditor, + draftForEnvironment, + }), + [ + editor, + voiceOwnerKey, + readEditor, + setEditor, + startEditor, + hasChanges, + resetEditor, + draftForEnvironment, + ], ); return {children}; } diff --git a/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx b/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx index 1ee4e9bc4e85..537c46551780 100644 --- a/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx +++ b/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx @@ -473,8 +473,7 @@ export function NewTaskDraftScreen(props: { }); const voiceInput = useVoiceInputController({ ownerKey: flow.draftKey, - draftMessage: flow.prompt, - selection: composerMenu.selection, + readDraftMessage: () => (flow.draftKey ? getComposerDraftSnapshot(flow.draftKey).text : null), disabled: isIncomingShareTransferPending || isImportingShare || flow.submitting, onChangeDraftMessage: flow.setPrompt, onChangeSelection: composerMenu.onSelectionChange, diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index a0eaaaf85548..8d9f11aeda58 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -54,6 +54,8 @@ import { themeColorWithAlpha } from "../../lib/mobileTheme"; import { armAgentAwarenessLiveActivityForLocalWork } from "../agent-awareness/remoteRegistration"; import { scopedThreadKey } from "../../lib/scopedEntities"; import { + getComposerDraftSnapshot, + setComposerDraftText, composerContextImportsAtom, countComposerDraftAttachmentsAfterSelection, } from "../../state/use-composer-drafts"; @@ -503,10 +505,9 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer usageLimitsOffered && props.draftAttachments.length === 0 ? openUsageLimits : undefined, }); const voiceInput = useVoiceInputController({ - ownerKey: composerOwnerKey, - draftMessage: props.draftMessage, - selection: composerMenu.selection, - onChangeDraftMessage: props.onChangeDraftMessage, + ownerKey: composerDraftKey, + readDraftMessage: () => getComposerDraftSnapshot(composerDraftKey).text, + onChangeDraftMessage: (text) => setComposerDraftText(composerDraftKey, text), onChangeSelection: composerMenu.onSelectionChange, }); const voicePresentation = resolveVoiceComposerPresentation( diff --git a/apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx b/apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx new file mode 100644 index 000000000000..fcabfb7af189 --- /dev/null +++ b/apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx @@ -0,0 +1,47 @@ +import { Platform, View } from "react-native"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; +import { FullWindowOverlay } from "react-native-screens"; + +import { APP_BAR_HEIGHT } from "../../lib/layoutMetrics"; +import { + ComposerDictationCancelAction, + ComposerDictationPrimaryAction, + ComposerDictationStatus, +} from "./ComposerDictationControl"; +import { useGlobalVoiceInput } from "./VoiceInputProvider"; +import { resolveVoiceComposerPresentation } from "./voiceInputPresentation"; + +export function GlobalVoiceInputControl() { + const voice = useGlobalVoiceInput(); + const insets = useSafeAreaInsets(); + const presentation = resolveVoiceComposerPresentation(voice.state, voice.elapsedSeconds); + if (!presentation.statusLabel || (voice.ownerKey && voice.focusedOwners.has(voice.ownerKey))) { + return null; + } + const content = ( + + + + + void voice.session.retry()} + onConfirm={voice.stop} + onCancel={voice.cancel} + /> + + + ); + return Platform.OS === "ios" ? {content} : content; +} diff --git a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx new file mode 100644 index 000000000000..2fe5221c94a1 --- /dev/null +++ b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx @@ -0,0 +1,222 @@ +import { + RecordingPresets, + requestRecordingPermissionsAsync, + setAudioModeAsync, + setIsAudioActiveAsync, + useAudioRecorder, + type RecordingStatus, +} from "expo-audio"; +import { File } from "expo-file-system"; +import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake"; +import { + createContext, + use, + useCallback, + useEffect, + useId, + useRef, + useState, + type ReactNode, +} from "react"; +import { AppState } from "react-native"; +import { useSharedValue } from "react-native-reanimated"; + +import { getLocalVoiceTranscriber } from "../../native/voiceTranscription"; +import { getNativeShowcaseScene } from "../showcase/nativeShowcaseScene"; +import { + VOICE_RECORDING_LIMIT_SECONDS, + voiceInputBlocksSubmission, + type VoiceInputState, +} from "@t3tools/client-runtime/voice-input"; +import { normalizeVoiceInputDecibels, VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering"; +import { VoiceInputSession } from "./voiceInputSession"; +import { GlobalVoiceInputControl } from "./GlobalVoiceInputControl"; + +const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null }; +const VOICE_METERING_INTERVAL_MS = 80; +const VOICE_RECORDING_OPTIONS = { + ...RecordingPresets.HIGH_QUALITY, + isMeteringEnabled: true, +}; + +async function releaseVoiceRecordingAudio(): Promise { + try { + await setAudioModeAsync({ allowsRecording: false }); + } finally { + // Expo does not deactivate AVAudioSession when recording stops or its + // category changes. Explicit deactivation resumes interrupted app audio. + await setIsAudioActiveAsync(false); + } +} + +async function configureVoiceRecordingAudio(): Promise { + try { + await setAudioModeAsync({ + allowsRecording: true, + interruptionMode: "doNotMix", + playsInSilentMode: true, + shouldPlayInBackground: false, + }); + await setIsAudioActiveAsync(true); + } catch (error) { + try { + await releaseVoiceRecordingAudio(); + } catch { + // Keep the setup error. The controller has not started a recorder yet. + } + throw error; + } +} + +const VoiceInputContext = createContext | null>(null); + +export function VoiceInputProvider({ children }: { readonly children: ReactNode }) { + const runtime = useVoiceInputRuntime(); + return ( + + {children} + + + ); +} + +export function useGlobalVoiceInput() { + const context = use(VoiceInputContext); + if (!context) throw new Error("Voice input provider is missing."); + return context; +} + +function useVoiceInputRuntime() { + const [{ state, ownerKey }, setState] = useState({ + state: INITIAL_STATE, + ownerKey: null as string | null, + }); + const [focusedOwners, setFocusedOwners] = useState>(new Set()); + const setOwnerFocused = useCallback((key: string, focused: boolean) => { + setFocusedOwners((current) => { + const next = new Set(current); + if (focused) next.add(key); + else next.delete(key); + return next; + }); + }, []); + const [elapsedSeconds, setElapsedSeconds] = useState(0); + const keepAwakeId = useId(); + const keepAwakeSessionRef = useRef(0); + const elapsedSecondsRef = useRef(0); + const audioLevelsRef = useRef(Array(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0)); + const audioLevels = useSharedValue(audioLevelsRef.current); + const sessionRef = useRef(null); + const handleRecorderStatus = useCallback((status: RecordingStatus) => { + sessionRef.current?.controller.handleRecorderStatus({ + isFinished: status.isFinished, + hasError: status.hasError || status.mediaServicesDidReset === true, + error: status.error, + url: status.url, + }); + }, []); + const recorder = useAudioRecorder(VOICE_RECORDING_OPTIONS, handleRecorderStatus); + + if (!sessionRef.current) { + sessionRef.current = new VoiceInputSession({ + recorder, + getTranscriber: getLocalVoiceTranscriber, + requestPermission: async () => { + const permission = await requestRecordingPermissionsAsync(); + return { granted: permission.granted, canAskAgain: permission.canAskAgain }; + }, + configureRecording: configureVoiceRecordingAudio, + releaseRecording: releaseVoiceRecordingAudio, + deleteRecording: (uri) => new File(uri).delete(), + onStateChange: (nextState) => + setState({ state: nextState, ownerKey: sessionRef.current?.ownerKey ?? null }), + }); + } + + const session = sessionRef.current; + const controller = session.controller; + + useEffect(() => { + const subscription = AppState.addEventListener("change", (nextState) => { + // iOS reports `inactive` while its permission dialog is open. Only the + // real background state cancels preparation; recorder status handles + // calls and route interruptions during capture. + if (nextState === "background") controller.appMovedToBackground(); + }); + return () => subscription.remove(); + }, [controller]); + + useEffect(() => () => controller.dispose(), [controller]); + + useEffect(() => { + if (state.phase !== "recording") return; + + const tag = `voice-input:${keepAwakeId}:${++keepAwakeSessionRef.current}`; + const activation = activateKeepAwakeAsync(tag); + void activation.catch(() => {}); + return () => { + // Release after activation settles, even if the recording ends immediately. + void activation.then(() => deactivateKeepAwake(tag)).catch(() => {}); + }; + }, [keepAwakeId, state.phase]); + + useEffect(() => { + if (state.phase !== "preparing" && state.phase !== "recording") return; + + if (audioLevelsRef.current.some((level) => level !== 0)) { + audioLevelsRef.current = Array(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0); + audioLevels.value = audioLevelsRef.current; + } + if (elapsedSecondsRef.current !== 0) { + elapsedSecondsRef.current = 0; + setElapsedSeconds(0); + } + if (state.phase !== "recording") return; + + const sampleRecording = () => { + if (controller.currentState.phase !== "recording") return; + const status = recorder.getStatus(); + if (!status.isRecording) return; + + const level = normalizeVoiceInputDecibels(status.metering); + const history = audioLevelsRef.current; + if (level !== 0 || history.some((sample) => sample !== 0)) { + const nextLevels = [...history.slice(1), level]; + audioLevelsRef.current = nextLevels; + audioLevels.value = nextLevels; + } + + const nextElapsedSeconds = Math.min( + VOICE_RECORDING_LIMIT_SECONDS, + Math.max(0, Math.floor(status.durationMillis / 1_000)), + ); + if (nextElapsedSeconds !== elapsedSecondsRef.current) { + elapsedSecondsRef.current = nextElapsedSeconds; + setElapsedSeconds(nextElapsedSeconds); + } + }; + + sampleRecording(); + const intervalId = setInterval(sampleRecording, VOICE_METERING_INTERVAL_MS); + return () => clearInterval(intervalId); + }, [audioLevels, controller, recorder, state.phase]); + + const stop = useCallback(() => controller.stop(), [controller]); + const cancel = useCallback(() => controller.cancel(), [controller]); + + return { + // Store screenshots show the dictation button even on simulators, whose + // on-device transcription is unavailable. + isAvailable: getLocalVoiceTranscriber() !== null || getNativeShowcaseScene() !== null, + state, + audioLevels, + elapsedSeconds, + isBusy: voiceInputBlocksSubmission(state), + ownerKey, + focusedOwners, + setOwnerFocused, + session, + stop, + cancel, + }; +} diff --git a/apps/mobile/src/features/voice-input/useVoiceInputController.ts b/apps/mobile/src/features/voice-input/useVoiceInputController.ts index 5fd15482d037..f8359ff22cda 100644 --- a/apps/mobile/src/features/voice-input/useVoiceInputController.ts +++ b/apps/mobile/src/features/voice-input/useVoiceInputController.ts @@ -1,235 +1,68 @@ -import { - RecordingPresets, - requestRecordingPermissionsAsync, - setAudioModeAsync, - setIsAudioActiveAsync, - useAudioRecorder, - type RecordingStatus, -} from "expo-audio"; -import { File } from "expo-file-system"; -import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake"; import { useFocusEffect } from "@react-navigation/native"; -import { useCallback, useEffect, useId, useRef, useState } from "react"; -import { AppState } from "react-native"; -import { useSharedValue } from "react-native-reanimated"; - -import type { ComposerEditorSelection } from "../../components/ComposerEditor"; -import { getLocalVoiceTranscriber } from "../../native/voiceTranscription"; -import { getNativeShowcaseScene } from "../showcase/nativeShowcaseScene"; +import { useCallback, useEffect, useRef } from "react"; import { - VoiceInputController, - VOICE_RECORDING_LIMIT_SECONDS, voiceInputBlocksSubmission, - voiceInputFreezesEditor, - type VoiceDraftSnapshot, type VoiceInputState, } from "@t3tools/client-runtime/voice-input"; -import { normalizeVoiceInputDecibels, VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering"; - -const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null }; -const VOICE_METERING_INTERVAL_MS = 80; -const VOICE_RECORDING_OPTIONS = { - ...RecordingPresets.HIGH_QUALITY, - isMeteringEnabled: true, -}; -async function releaseVoiceRecordingAudio(): Promise { - try { - await setAudioModeAsync({ allowsRecording: false }); - } finally { - // Expo does not deactivate AVAudioSession when recording stops or its - // category changes. Explicit deactivation resumes interrupted app audio. - await setIsAudioActiveAsync(false); - } -} +import type { ComposerEditorSelection } from "../../components/ComposerEditor"; +import { useGlobalVoiceInput } from "./VoiceInputProvider"; +import { createVoiceInputTarget } from "./voiceInputSession"; -async function configureVoiceRecordingAudio(): Promise { - try { - await setAudioModeAsync({ - allowsRecording: true, - interruptionMode: "doNotMix", - playsInSilentMode: true, - shouldPlayInBackground: false, - }); - await setIsAudioActiveAsync(true); - } catch (error) { - try { - await releaseVoiceRecordingAudio(); - } catch { - // Keep the setup error. The controller has not started a recorder yet. - } - throw error; - } -} +const IDLE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null }; export function useVoiceInputController(input: { readonly ownerKey: string | null; - readonly draftMessage: string; - readonly selection: ComposerEditorSelection; + readonly readDraftMessage: () => string | null; readonly disabled?: boolean; readonly onChangeDraftMessage: (value: string) => void; readonly onChangeSelection: (selection: ComposerEditorSelection) => void; }) { - const [state, setState] = useState(INITIAL_STATE); - const [elapsedSeconds, setElapsedSeconds] = useState(0); - const keepAwakeId = useId(); - const keepAwakeSessionRef = useRef(0); - const elapsedSecondsRef = useRef(0); - const audioLevelsRef = useRef(Array(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0)); - const audioLevels = useSharedValue(audioLevelsRef.current); - const controllerRef = useRef(null); - const previousDraftRef = useRef({ ownerKey: input.ownerKey, text: input.draftMessage }); - const revisionRef = useRef(0); - if ( - previousDraftRef.current.ownerKey !== input.ownerKey || - previousDraftRef.current.text !== input.draftMessage - ) { - previousDraftRef.current = { ownerKey: input.ownerKey, text: input.draftMessage }; - revisionRef.current += 1; - } - const latestInputRef = useRef(input); - latestInputRef.current = input; - - const handleRecorderStatus = useCallback((status: RecordingStatus) => { - controllerRef.current?.handleRecorderStatus({ - isFinished: status.isFinished, - hasError: status.hasError || status.mediaServicesDidReset === true, - error: status.error, - url: status.url, - }); - }, []); - const recorder = useAudioRecorder(VOICE_RECORDING_OPTIONS, handleRecorderStatus); - - if (!controllerRef.current) { - controllerRef.current = new VoiceInputController({ - recorder, - getTranscriber: getLocalVoiceTranscriber, - requestPermission: async () => { - const permission = await requestRecordingPermissionsAsync(); - return { granted: permission.granted, canAskAgain: permission.canAskAgain }; - }, - configureRecording: configureVoiceRecordingAudio, - releaseRecording: releaseVoiceRecordingAudio, - deleteRecording: (uri) => new File(uri).delete(), - readDraft: (): VoiceDraftSnapshot | null => { - const current = latestInputRef.current; - if (!current.ownerKey) return null; - return { - ownerKey: current.ownerKey, - text: current.draftMessage, - selection: current.selection, - revision: revisionRef.current, - }; - }, - commitDraft: (text, selection) => { - const current = latestInputRef.current; - current.onChangeSelection(selection); - current.onChangeDraftMessage(text); - }, - onStateChange: setState, - }); - } - - const controller = controllerRef.current; - const previousOwnerRef = useRef(input.ownerKey); - useEffect(() => { - if (previousOwnerRef.current === input.ownerKey) return; - previousOwnerRef.current = input.ownerKey; - controller.ownerChanged(); - }, [controller, input.ownerKey]); - - useFocusEffect( - useCallback( - () => () => { - controller.dispose(); - }, - [controller], - ), - ); - + const global = useGlobalVoiceInput(); + const { setOwnerFocused, session } = global; + const latestInput = useRef(input); + latestInput.current = input; + const mounted = useRef(true); useEffect(() => { - const subscription = AppState.addEventListener("change", (nextState) => { - // iOS reports `inactive` while its permission dialog is open. Only the - // real background state cancels preparation; recorder status handles - // calls and route interruptions during capture. - if (nextState === "background") controller.appMovedToBackground(); - }); - return () => subscription.remove(); - }, [controller]); - - useEffect(() => () => controller.dispose(), [controller]); - - useEffect(() => { - if (state.phase !== "recording") return; - - const tag = `voice-input:${keepAwakeId}:${++keepAwakeSessionRef.current}`; - const activation = activateKeepAwakeAsync(tag); - void activation.catch(() => {}); + mounted.current = true; return () => { - // Release after activation settles, even if the recording ends immediately. - void activation.then(() => deactivateKeepAwake(tag)).catch(() => {}); - }; - }, [keepAwakeId, state.phase]); - - useEffect(() => { - if (state.phase !== "preparing" && state.phase !== "recording") return; - - if (audioLevelsRef.current.some((level) => level !== 0)) { - audioLevelsRef.current = Array(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0); - audioLevels.value = audioLevelsRef.current; - } - if (elapsedSecondsRef.current !== 0) { - elapsedSecondsRef.current = 0; - setElapsedSeconds(0); - } - if (state.phase !== "recording") return; - - const sampleRecording = () => { - if (controller.currentState.phase !== "recording") return; - const status = recorder.getStatus(); - if (!status.isRecording) return; - - const level = normalizeVoiceInputDecibels(status.metering); - const history = audioLevelsRef.current; - if (level !== 0 || history.some((sample) => sample !== 0)) { - const nextLevels = [...history.slice(1), level]; - audioLevelsRef.current = nextLevels; - audioLevels.value = nextLevels; - } - - const nextElapsedSeconds = Math.min( - VOICE_RECORDING_LIMIT_SECONDS, - Math.max(0, Math.floor(status.durationMillis / 1_000)), - ); - if (nextElapsedSeconds !== elapsedSecondsRef.current) { - elapsedSecondsRef.current = nextElapsedSeconds; - setElapsedSeconds(nextElapsedSeconds); - } + mounted.current = false; }; + }, []); - sampleRecording(); - const intervalId = setInterval(sampleRecording, VOICE_METERING_INTERVAL_MS); - return () => clearInterval(intervalId); - }, [audioLevels, controller, recorder, state.phase]); + useFocusEffect( + useCallback(() => { + const ownerKey = input.ownerKey; + if (!ownerKey) return; + setOwnerFocused(ownerKey, true); + return () => setOwnerFocused(ownerKey, false); + }, [input.ownerKey, setOwnerFocused]), + ); const start = useCallback(() => { - if (!latestInputRef.current.disabled) void controller.start(); - }, [controller]); - const stop = useCallback(() => controller.stop(), [controller]); - const cancel = useCallback(() => controller.cancel(), [controller]); - + const captured = latestInput.current; + if (!captured.ownerKey || captured.disabled) return; + void session.start( + createVoiceInputTarget(captured.ownerKey, captured.readDraftMessage, (text, selection) => { + captured.onChangeDraftMessage(text); + if (mounted.current && latestInput.current.ownerKey === captured.ownerKey) { + latestInput.current.onChangeSelection(selection); + } + }), + ); + }, [session]); + const state = global.ownerKey === input.ownerKey ? global.state : IDLE_STATE; + const isBusy = voiceInputBlocksSubmission(state); return { - // Store screenshots show the dictation button even on simulators, whose - // on-device transcription is unavailable. - isAvailable: getLocalVoiceTranscriber() !== null || getNativeShowcaseScene() !== null, + isAvailable: global.isAvailable && (!global.isBusy || global.ownerKey === input.ownerKey), state, - audioLevels, - elapsedSeconds, - isBusy: voiceInputBlocksSubmission(state), - freezesEditor: voiceInputFreezesEditor(state), - blocksSubmission: voiceInputBlocksSubmission(state), + audioLevels: global.audioLevels, + elapsedSeconds: global.elapsedSeconds, + isBusy, + freezesEditor: isBusy, + blocksSubmission: isBusy, start, - stop, - cancel, + stop: global.stop, + cancel: global.cancel, }; } diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.test.ts b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts new file mode 100644 index 000000000000..019a0164aafd --- /dev/null +++ b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts @@ -0,0 +1,155 @@ +import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; +import type { PreparedVoiceTranscription } from "@t3tools/client-runtime/voice-input"; +import { resetVoiceInputGlobalsForTests } from "../../../../../packages/client-runtime/src/voice-input/controller"; + +import { createVoiceInputTarget, VoiceInputSession } from "./voiceInputSession"; + +function createSession() { + const recorder = { + uri: "file:///voice.m4a", + prepareToRecordAsync: vi.fn(async () => {}), + record: vi.fn(), + stop: vi.fn(async () => {}), + }; + const prepare = vi.fn(async (): Promise => ({ + locale: "en-US", + transcribe: async () => "spoken text", + })); + const session = new VoiceInputSession({ + recorder, + getTranscriber: () => ({ prepare }), + requestPermission: async () => ({ granted: true, canAskAgain: true }), + configureRecording: async () => {}, + releaseRecording: vi.fn(async () => {}), + deleteRecording: vi.fn(), + onStateChange: vi.fn(), + }); + return { session, recorder, prepare }; +} + +describe("global voice input", () => { + beforeEach(() => resetVoiceInputGlobalsForTests()); + + it("appends to the starting draft after its screen leaves and another draft opens", async () => { + const { session, recorder } = createSession(); + const drafts = new Map([ + ["first", "original prompt"], + ["second", "other prompt"], + ]); + let visibleDraft = "first"; + const targetKey = visibleDraft; + await session.start( + createVoiceInputTarget( + targetKey, + () => drafts.get(targetKey) ?? null, + (text) => drafts.set(targetKey, text), + ), + ); + visibleDraft = "second"; + expect(session.controller.currentState.phase).toBe("recording"); + expect(recorder.stop).not.toHaveBeenCalled(); + await session.controller.stop(); + + expect(drafts.get("first")).toBe("original prompt spoken text"); + expect(drafts.get(visibleDraft)).toBe("other prompt"); + expect(session.controller.currentState.phase).toBe("idle"); + }); + + it.each(["preparing", "recording", "transcribing"] as const)( + "keeps one recorder and its original target during %s", + async (phase) => { + const preparation = Promise.withResolvers(); + const preparationEntered = Promise.withResolvers(); + const transcription = Promise.withResolvers(); + const transcriptionEntered = Promise.withResolvers(); + const { session, recorder, prepare } = createSession(); + prepare.mockImplementationOnce(() => { + preparationEntered.resolve(); + return preparation.promise; + }); + const firstCommit = vi.fn(); + const secondCommit = vi.fn(); + const starting = session.start(createVoiceInputTarget("first", () => "first", firstCommit)); + await preparationEntered.promise; + let stopping: Promise | null = null; + if (phase !== "preparing") { + preparation.resolve({ + locale: "en-US", + transcribe: () => { + transcriptionEntered.resolve(); + return transcription.promise; + }, + }); + await starting; + } + if (phase === "transcribing") { + stopping = session.controller.stop(); + await transcriptionEntered.promise; + } + await session.start(createVoiceInputTarget("second", () => "second", secondCommit)); + expect(session.ownerKey).toBe("first"); + expect(session.controller.currentState.phase).toBe(phase); + expect(prepare).toHaveBeenCalledTimes(1); + preparation.resolve({ locale: "en-US", transcribe: async () => "spoken text" }); + await starting; + transcription.resolve("spoken text"); + await (stopping ?? session.controller.stop()); + expect(recorder.record).toHaveBeenCalledTimes(1); + expect(firstCommit).toHaveBeenCalledWith("first spoken text", { start: 17, end: 17 }); + expect(secondCommit).not.toHaveBeenCalled(); + }, + ); + + it.each(["changed", "removed"] as const)( + "does not overwrite a %s starting draft", + async (change) => { + const { session } = createSession(); + let text: string | null = "first"; + const commit = vi.fn(); + await session.start(createVoiceInputTarget("first", () => text, commit)); + text = change === "removed" ? null : "edited prompt"; + await session.controller.stop(); + expect(commit).not.toHaveBeenCalled(); + expect(session.controller.currentState.error).toContain("draft changed"); + }, + ); + + it("finishes the original draft at the recording limit while it is off screen", async () => { + const { session, recorder } = createSession(); + const commit = vi.fn(); + await session.start(createVoiceInputTarget("first", () => "first", commit)); + await session.controller.handleRecorderStatus({ + isFinished: true, + hasError: false, + error: null, + url: recorder.uri, + }); + expect(commit).toHaveBeenCalledWith("first spoken text", { start: 17, end: 17 }); + expect(session.controller.currentState.phase).toBe("idle"); + }); + + it("waits for canceled native work before starting a recording for another draft", async () => { + const preparation = Promise.withResolvers(); + const preparationEntered = Promise.withResolvers(); + const { session, recorder, prepare } = createSession(); + prepare.mockImplementationOnce(() => { + preparationEntered.resolve(); + return preparation.promise; + }); + const oldCommit = vi.fn(); + const nextCommit = vi.fn(); + const firstStart = session.start(createVoiceInputTarget("first", () => "first", oldCommit)); + await preparationEntered.promise; + session.controller.cancel(); + const nextStart = session.start(createVoiceInputTarget("second", () => "second", nextCommit)); + expect(prepare).toHaveBeenCalledTimes(1); + expect(recorder.record).not.toHaveBeenCalled(); + preparation.resolve({ locale: "en-US", transcribe: async () => "old transcript" }); + await firstStart; + await nextStart; + await session.controller.stop(); + expect(recorder.record).toHaveBeenCalledTimes(1); + expect(oldCommit).not.toHaveBeenCalled(); + expect(nextCommit).toHaveBeenCalledWith("second spoken text", { start: 18, end: 18 }); + }); +}); diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.ts b/apps/mobile/src/features/voice-input/voiceInputSession.ts new file mode 100644 index 000000000000..bc4681448b43 --- /dev/null +++ b/apps/mobile/src/features/voice-input/voiceInputSession.ts @@ -0,0 +1,55 @@ +import { + VoiceInputController, + voiceInputBlocksSubmission, + type VoiceDraftSnapshot, + type VoiceInputControllerDependencies, +} from "@t3tools/client-runtime/voice-input"; + +export type VoiceInputTarget = { + readonly ownerKey: string; + readonly readDraft: () => VoiceDraftSnapshot | null; + readonly commitDraft: VoiceInputControllerDependencies["commitDraft"]; +}; + +export function createVoiceInputTarget( + ownerKey: string, + readText: () => string | null, + commitDraft: VoiceInputTarget["commitDraft"], +): VoiceInputTarget { + return { + ownerKey, + readDraft: () => { + const text = readText(); + if (text === null) return null; + return { ownerKey, text, selection: { start: text.length, end: text.length }, revision: 0 }; + }, + commitDraft, + }; +} + +export class VoiceInputSession { + readonly controller: VoiceInputController; + private target: VoiceInputTarget | null = null; + + constructor(dependencies: Omit) { + this.controller = new VoiceInputController({ + ...dependencies, + readDraft: () => this.target?.readDraft() ?? null, + commitDraft: (text, selection) => this.target?.commitDraft(text, selection), + }); + } + + get ownerKey(): string | null { + return this.target?.ownerKey ?? null; + } + + retry(): Promise { + return this.target ? this.start(this.target) : Promise.resolve(); + } + + start(target: VoiceInputTarget): Promise { + if (voiceInputBlocksSubmission(this.controller.currentState)) return Promise.resolve(); + this.target = target; + return this.controller.start(); + } +} From 5a5e887bdba0d1847efdbc0014458e017c9f89f4 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 04:39:30 +0200 Subject: [PATCH 04/10] fix(mobile): remove global dictation import cycle --- apps/mobile/src/App.tsx | 2 ++ .../src/features/voice-input/VoiceInputProvider.tsx | 8 +------- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/apps/mobile/src/App.tsx b/apps/mobile/src/App.tsx index d30eea1d286d..c2ccd631127b 100644 --- a/apps/mobile/src/App.tsx +++ b/apps/mobile/src/App.tsx @@ -24,6 +24,7 @@ import { shouldHandleAppLink } from "./lib/appLinking"; import { useMobileNavigationTheme } from "./lib/useMobileNavigationTheme"; import { SubscriptionUsageCoordinator } from "./widgets/SubscriptionUsageCoordinator"; import { VoiceInputProvider } from "./features/voice-input/VoiceInputProvider"; +import { GlobalVoiceInputControl } from "./features/voice-input/GlobalVoiceInputControl"; import "../global.css"; @@ -94,6 +95,7 @@ function AppContent() { {/* Anchored-menu overlays render here — in-window, so the keyboard stays up while a dropdown is open. */} + diff --git a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx index 2fe5221c94a1..c31ab943ad4b 100644 --- a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx +++ b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx @@ -30,7 +30,6 @@ import { } from "@t3tools/client-runtime/voice-input"; import { normalizeVoiceInputDecibels, VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering"; import { VoiceInputSession } from "./voiceInputSession"; -import { GlobalVoiceInputControl } from "./GlobalVoiceInputControl"; const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null }; const VOICE_METERING_INTERVAL_MS = 80; @@ -72,12 +71,7 @@ const VoiceInputContext = createContext export function VoiceInputProvider({ children }: { readonly children: ReactNode }) { const runtime = useVoiceInputRuntime(); - return ( - - {children} - - - ); + return {children}; } export function useGlobalVoiceInput() { From f4a56c281df52d7ccbd7c881fbabc91ba6a3c878 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 04:41:02 +0200 Subject: [PATCH 05/10] fix(mobile): retain the starting dictation selection --- .../SettingsScheduledTasksRouteScreen.tsx | 1 + .../features/threads/NewTaskDraftScreen.tsx | 1 + .../src/features/threads/ThreadComposer.tsx | 1 + .../voice-input/useVoiceInputController.ts | 18 +++++--- .../voice-input/voiceInputSession.test.ts | 45 ++++++++++++++++--- .../features/voice-input/voiceInputSession.ts | 3 +- 6 files changed, 56 insertions(+), 13 deletions(-) diff --git a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx index c8925d1baa66..1a4ff14b1859 100644 --- a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx @@ -408,6 +408,7 @@ function SettingsScheduledTaskEditorScreen({ title }: { readonly title: string } ); const voiceInput = useVoiceInputController({ ownerKey, + selection, readDraftMessage: () => { const current = readEditor(); return current?.environmentId === editor?.environmentId diff --git a/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx b/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx index 537c46551780..64f664f9a996 100644 --- a/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx +++ b/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx @@ -474,6 +474,7 @@ export function NewTaskDraftScreen(props: { const voiceInput = useVoiceInputController({ ownerKey: flow.draftKey, readDraftMessage: () => (flow.draftKey ? getComposerDraftSnapshot(flow.draftKey).text : null), + selection: composerMenu.selection, disabled: isIncomingShareTransferPending || isImportingShare || flow.submitting, onChangeDraftMessage: flow.setPrompt, onChangeSelection: composerMenu.onSelectionChange, diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index 8d9f11aeda58..e59350616bc4 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -507,6 +507,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer const voiceInput = useVoiceInputController({ ownerKey: composerDraftKey, readDraftMessage: () => getComposerDraftSnapshot(composerDraftKey).text, + selection: composerMenu.selection, onChangeDraftMessage: (text) => setComposerDraftText(composerDraftKey, text), onChangeSelection: composerMenu.onSelectionChange, }); diff --git a/apps/mobile/src/features/voice-input/useVoiceInputController.ts b/apps/mobile/src/features/voice-input/useVoiceInputController.ts index f8359ff22cda..a7229a5315c1 100644 --- a/apps/mobile/src/features/voice-input/useVoiceInputController.ts +++ b/apps/mobile/src/features/voice-input/useVoiceInputController.ts @@ -14,6 +14,7 @@ const IDLE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: n export function useVoiceInputController(input: { readonly ownerKey: string | null; readonly readDraftMessage: () => string | null; + readonly selection: ComposerEditorSelection; readonly disabled?: boolean; readonly onChangeDraftMessage: (value: string) => void; readonly onChangeSelection: (selection: ComposerEditorSelection) => void; @@ -43,12 +44,17 @@ export function useVoiceInputController(input: { const captured = latestInput.current; if (!captured.ownerKey || captured.disabled) return; void session.start( - createVoiceInputTarget(captured.ownerKey, captured.readDraftMessage, (text, selection) => { - captured.onChangeDraftMessage(text); - if (mounted.current && latestInput.current.ownerKey === captured.ownerKey) { - latestInput.current.onChangeSelection(selection); - } - }), + createVoiceInputTarget( + captured.ownerKey, + captured.readDraftMessage, + (text, selection) => { + captured.onChangeDraftMessage(text); + if (mounted.current && latestInput.current.ownerKey === captured.ownerKey) { + latestInput.current.onChangeSelection(selection); + } + }, + captured.selection, + ), ); }, [session]); const state = global.ownerKey === input.ownerKey ? global.state : IDLE_STATE; diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.test.ts b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts index 019a0164aafd..7ae31c28de50 100644 --- a/apps/mobile/src/features/voice-input/voiceInputSession.test.ts +++ b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts @@ -30,6 +30,26 @@ function createSession() { describe("global voice input", () => { beforeEach(() => resetVoiceInputGlobalsForTests()); + it.each([ + { selection: { start: 6, end: 11 }, expected: "hello spoken text", cursor: 17 }, + { selection: { start: 6, end: 6 }, expected: "hello spoken text world", cursor: 18 }, + ])( + "keeps the starting selection $selection after navigation", + async ({ selection, expected, cursor }) => { + const { session } = createSession(); + const commit = vi.fn(); + await session.start(createVoiceInputTarget("first", () => "hello world", commit, selection)); + await session.start( + createVoiceInputTarget("second", () => "other prompt", vi.fn(), { start: 12, end: 12 }), + ); + await session.controller.stop(); + expect(commit).toHaveBeenCalledWith(expected, { + start: cursor, + end: cursor, + }); + }, + ); + it("appends to the starting draft after its screen leaves and another draft opens", async () => { const { session, recorder } = createSession(); const drafts = new Map([ @@ -43,6 +63,7 @@ describe("global voice input", () => { targetKey, () => drafts.get(targetKey) ?? null, (text) => drafts.set(targetKey, text), + { start: 15, end: 15 }, ), ); visibleDraft = "second"; @@ -69,7 +90,9 @@ describe("global voice input", () => { }); const firstCommit = vi.fn(); const secondCommit = vi.fn(); - const starting = session.start(createVoiceInputTarget("first", () => "first", firstCommit)); + const starting = session.start( + createVoiceInputTarget("first", () => "first", firstCommit, { start: 5, end: 5 }), + ); await preparationEntered.promise; let stopping: Promise | null = null; if (phase !== "preparing") { @@ -86,7 +109,9 @@ describe("global voice input", () => { stopping = session.controller.stop(); await transcriptionEntered.promise; } - await session.start(createVoiceInputTarget("second", () => "second", secondCommit)); + await session.start( + createVoiceInputTarget("second", () => "second", secondCommit, { start: 6, end: 6 }), + ); expect(session.ownerKey).toBe("first"); expect(session.controller.currentState.phase).toBe(phase); expect(prepare).toHaveBeenCalledTimes(1); @@ -106,7 +131,9 @@ describe("global voice input", () => { const { session } = createSession(); let text: string | null = "first"; const commit = vi.fn(); - await session.start(createVoiceInputTarget("first", () => text, commit)); + await session.start( + createVoiceInputTarget("first", () => text, commit, { start: 5, end: 5 }), + ); text = change === "removed" ? null : "edited prompt"; await session.controller.stop(); expect(commit).not.toHaveBeenCalled(); @@ -117,7 +144,9 @@ describe("global voice input", () => { it("finishes the original draft at the recording limit while it is off screen", async () => { const { session, recorder } = createSession(); const commit = vi.fn(); - await session.start(createVoiceInputTarget("first", () => "first", commit)); + await session.start( + createVoiceInputTarget("first", () => "first", commit, { start: 5, end: 5 }), + ); await session.controller.handleRecorderStatus({ isFinished: true, hasError: false, @@ -138,10 +167,14 @@ describe("global voice input", () => { }); const oldCommit = vi.fn(); const nextCommit = vi.fn(); - const firstStart = session.start(createVoiceInputTarget("first", () => "first", oldCommit)); + const firstStart = session.start( + createVoiceInputTarget("first", () => "first", oldCommit, { start: 5, end: 5 }), + ); await preparationEntered.promise; session.controller.cancel(); - const nextStart = session.start(createVoiceInputTarget("second", () => "second", nextCommit)); + const nextStart = session.start( + createVoiceInputTarget("second", () => "second", nextCommit, { start: 6, end: 6 }), + ); expect(prepare).toHaveBeenCalledTimes(1); expect(recorder.record).not.toHaveBeenCalled(); preparation.resolve({ locale: "en-US", transcribe: async () => "old transcript" }); diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.ts b/apps/mobile/src/features/voice-input/voiceInputSession.ts index bc4681448b43..a51a486143f6 100644 --- a/apps/mobile/src/features/voice-input/voiceInputSession.ts +++ b/apps/mobile/src/features/voice-input/voiceInputSession.ts @@ -15,13 +15,14 @@ export function createVoiceInputTarget( ownerKey: string, readText: () => string | null, commitDraft: VoiceInputTarget["commitDraft"], + selection: VoiceDraftSnapshot["selection"], ): VoiceInputTarget { return { ownerKey, readDraft: () => { const text = readText(); if (text === null) return null; - return { ownerKey, text, selection: { start: text.length, end: text.length }, revision: 0 }; + return { ownerKey, text, selection, revision: 0 }; }, commitDraft, }; From f8a124f756baae6c4509125786e7af42d3c4213f Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 04:46:43 +0200 Subject: [PATCH 06/10] fix(mobile): cancel dictation when queued edits are discarded --- .../features/threads/ThreadDetailScreen.tsx | 7 +++- .../voice-input/voiceInputSession.test.ts | 35 ++++++++++++++++++- .../features/voice-input/voiceInputSession.ts | 4 +++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index d1470315a566..14f8bac554e5 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -131,6 +131,7 @@ import { ComposerQueuedEditBanner } from "./ComposerQueuedEdit"; import { useThreadQueuedCount } from "./ThreadQueueControl"; import type { ThreadContentPresentation } from "./threadContentPresentation"; import { resolveThreadFeedSubmissionAnchor } from "./thread-feed-live-follow"; +import { useGlobalVoiceInput } from "../voice-input/VoiceInputProvider"; export interface ThreadDetailScreenProps { readonly worktreeSetup?: WorktreeSetupCardProps | null; @@ -303,6 +304,7 @@ const USER_INPUT_TOGGLE_TIMING = { export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: ThreadDetailScreenProps) { const navigation = useNavigation(); + const { session: voiceInputSession } = useGlobalVoiceInput(); const deviceState = useEnvironmentQuery( deviceEnvironment.state({ environmentId: props.environmentId, input: {} }), ); @@ -1155,7 +1157,10 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread > { + voiceInputSession.cancel(props.composerDraftKey); + props.onCancelQueuedRunEdit(); + }} /> ) : null} diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.test.ts b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts index 7ae31c28de50..063527b1d0b4 100644 --- a/apps/mobile/src/features/voice-input/voiceInputSession.test.ts +++ b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts @@ -157,6 +157,39 @@ describe("global voice input", () => { expect(session.controller.currentState.phase).toBe("idle"); }); + it("stops recording when its queued edit is discarded", async () => { + const { session, recorder } = createSession(); + const commit = vi.fn(); + let draft: string | null = "queued prompt"; + const ownerKey = "thread~queued-edit~run"; + await session.start( + createVoiceInputTarget(ownerKey, () => draft, commit, { start: 13, end: 13 }), + ); + session.cancel(ownerKey); + draft = null; + await session.controller.stop(); + expect(recorder.stop).toHaveBeenCalledTimes(1); + expect(session.controller.currentState).toEqual({ + phase: "idle", + error: null, + errorAction: null, + }); + expect(commit).not.toHaveBeenCalled(); + }); + + it("keeps another prompt's recording when a queued edit is discarded", async () => { + const { session, recorder } = createSession(); + const commit = vi.fn(); + await session.start( + createVoiceInputTarget("other prompt", () => "hello", commit, { start: 5, end: 5 }), + ); + session.cancel("thread~queued-edit~run"); + expect(recorder.stop).not.toHaveBeenCalled(); + expect(session.controller.currentState.phase).toBe("recording"); + await session.controller.stop(); + expect(commit).toHaveBeenCalledWith("hello spoken text", { start: 17, end: 17 }); + }); + it("waits for canceled native work before starting a recording for another draft", async () => { const preparation = Promise.withResolvers(); const preparationEntered = Promise.withResolvers(); @@ -171,7 +204,7 @@ describe("global voice input", () => { createVoiceInputTarget("first", () => "first", oldCommit, { start: 5, end: 5 }), ); await preparationEntered.promise; - session.controller.cancel(); + session.cancel("first"); const nextStart = session.start( createVoiceInputTarget("second", () => "second", nextCommit, { start: 6, end: 6 }), ); diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.ts b/apps/mobile/src/features/voice-input/voiceInputSession.ts index a51a486143f6..7eccea142303 100644 --- a/apps/mobile/src/features/voice-input/voiceInputSession.ts +++ b/apps/mobile/src/features/voice-input/voiceInputSession.ts @@ -44,6 +44,10 @@ export class VoiceInputSession { return this.target?.ownerKey ?? null; } + cancel(ownerKey: string | null): void { + if (ownerKey !== null && this.ownerKey === ownerKey) this.controller.cancel(); + } + retry(): Promise { return this.target ? this.start(this.target) : Promise.resolve(); } From 93aef26cb2ba376c8ba41b10bfc31b39845f548e Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 04:53:53 +0200 Subject: [PATCH 07/10] fix(mobile): track dictation draft changes across navigation --- .../SettingsScheduledTasksRouteScreen.tsx | 4 + .../features/threads/NewTaskDraftScreen.tsx | 2 + .../src/features/threads/ThreadComposer.tsx | 3 + .../voice-input/useVoiceInputController.ts | 2 + .../voice-input/voiceInputSession.test.ts | 91 +++++++++++++++---- .../features/voice-input/voiceInputSession.ts | 24 ++++- 6 files changed, 105 insertions(+), 21 deletions(-) diff --git a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx index 1a4ff14b1859..ce9162525832 100644 --- a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx @@ -62,6 +62,8 @@ import { } from "./scheduledTaskDraft"; import { settingsTargetsForProject } from "./settings-environment-filter.logic"; import { useScheduledTaskEditor } from "./scheduled-task-editor"; +import { scheduledTaskEditorSessionAtom } from "./scheduled-task-editor-state"; +import { appAtomRegistry } from "../../state/atom-registry"; import { formatNextScheduledTaskRun, formatScheduledTaskInterval, @@ -408,6 +410,8 @@ function SettingsScheduledTaskEditorScreen({ title }: { readonly title: string } ); const voiceInput = useVoiceInputController({ ownerKey, + subscribeToDraftChanges: (onChange) => + appAtomRegistry.subscribe(scheduledTaskEditorSessionAtom, onChange), selection, readDraftMessage: () => { const current = readEditor(); diff --git a/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx b/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx index 64f664f9a996..e576787301da 100644 --- a/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx +++ b/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx @@ -96,6 +96,7 @@ import { captureComposerDraftInsertion, countComposerDraftAttachmentsAfterSelection, getComposerDraftSnapshot, + composerDraftsAtom, mergeComposerDraftContent, restoreComposerDraftSnapshot, updateComposerDraftSettings, @@ -474,6 +475,7 @@ export function NewTaskDraftScreen(props: { const voiceInput = useVoiceInputController({ ownerKey: flow.draftKey, readDraftMessage: () => (flow.draftKey ? getComposerDraftSnapshot(flow.draftKey).text : null), + subscribeToDraftChanges: (onChange) => appAtomRegistry.subscribe(composerDraftsAtom, onChange), selection: composerMenu.selection, disabled: isIncomingShareTransferPending || isImportingShare || flow.submitting, onChangeDraftMessage: flow.setPrompt, diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index 695eecbb9063..66efe4f43bb3 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -55,10 +55,12 @@ import { armAgentAwarenessLiveActivityForLocalWork } from "../agent-awareness/re import { scopedThreadKey } from "../../lib/scopedEntities"; import { getComposerDraftSnapshot, + composerDraftsAtom, setComposerDraftText, composerContextImportsAtom, countComposerDraftAttachmentsAfterSelection, } from "../../state/use-composer-drafts"; +import { appAtomRegistry } from "../../state/atom-registry"; import type { ComposerDocumentAttachment } from "../../lib/composerContext"; import { useProject, useThreadShells } from "../../state/entities"; import { scopeProjectRef } from "@t3tools/client-runtime/environment"; @@ -508,6 +510,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer const voiceInput = useVoiceInputController({ ownerKey: composerDraftKey, readDraftMessage: () => getComposerDraftSnapshot(composerDraftKey).text, + subscribeToDraftChanges: (onChange) => appAtomRegistry.subscribe(composerDraftsAtom, onChange), selection: composerMenu.selection, onChangeDraftMessage: (text) => setComposerDraftText(composerDraftKey, text), onChangeSelection: composerMenu.onSelectionChange, diff --git a/apps/mobile/src/features/voice-input/useVoiceInputController.ts b/apps/mobile/src/features/voice-input/useVoiceInputController.ts index a7229a5315c1..06a35e3c60a2 100644 --- a/apps/mobile/src/features/voice-input/useVoiceInputController.ts +++ b/apps/mobile/src/features/voice-input/useVoiceInputController.ts @@ -14,6 +14,7 @@ const IDLE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: n export function useVoiceInputController(input: { readonly ownerKey: string | null; readonly readDraftMessage: () => string | null; + readonly subscribeToDraftChanges: (onChange: () => void) => () => void; readonly selection: ComposerEditorSelection; readonly disabled?: boolean; readonly onChangeDraftMessage: (value: string) => void; @@ -54,6 +55,7 @@ export function useVoiceInputController(input: { } }, captured.selection, + captured.subscribeToDraftChanges, ), ); }, [session]); diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.test.ts b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts index 063527b1d0b4..07fc6348f8af 100644 --- a/apps/mobile/src/features/voice-input/voiceInputSession.test.ts +++ b/apps/mobile/src/features/voice-input/voiceInputSession.test.ts @@ -1,8 +1,22 @@ import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; import type { PreparedVoiceTranscription } from "@t3tools/client-runtime/voice-input"; +import { Atom, AtomRegistry } from "effect/unstable/reactivity"; import { resetVoiceInputGlobalsForTests } from "../../../../../packages/client-runtime/src/voice-input/controller"; -import { createVoiceInputTarget, VoiceInputSession } from "./voiceInputSession"; +import { + createVoiceInputTarget, + VoiceInputSession, + type VoiceInputTarget, +} from "./voiceInputSession"; + +function createTarget( + ownerKey: string, + readText: () => string | null, + commit: VoiceInputTarget["commitDraft"], + selection: { start: number; end: number }, +) { + return createVoiceInputTarget(ownerKey, readText, commit, selection, () => () => {}); +} function createSession() { const recorder = { @@ -38,9 +52,9 @@ describe("global voice input", () => { async ({ selection, expected, cursor }) => { const { session } = createSession(); const commit = vi.fn(); - await session.start(createVoiceInputTarget("first", () => "hello world", commit, selection)); + await session.start(createTarget("first", () => "hello world", commit, selection)); await session.start( - createVoiceInputTarget("second", () => "other prompt", vi.fn(), { start: 12, end: 12 }), + createTarget("second", () => "other prompt", vi.fn(), { start: 12, end: 12 }), ); await session.controller.stop(); expect(commit).toHaveBeenCalledWith(expected, { @@ -59,7 +73,7 @@ describe("global voice input", () => { let visibleDraft = "first"; const targetKey = visibleDraft; await session.start( - createVoiceInputTarget( + createTarget( targetKey, () => drafts.get(targetKey) ?? null, (text) => drafts.set(targetKey, text), @@ -91,7 +105,7 @@ describe("global voice input", () => { const firstCommit = vi.fn(); const secondCommit = vi.fn(); const starting = session.start( - createVoiceInputTarget("first", () => "first", firstCommit, { start: 5, end: 5 }), + createTarget("first", () => "first", firstCommit, { start: 5, end: 5 }), ); await preparationEntered.promise; let stopping: Promise | null = null; @@ -110,7 +124,7 @@ describe("global voice input", () => { await transcriptionEntered.promise; } await session.start( - createVoiceInputTarget("second", () => "second", secondCommit, { start: 6, end: 6 }), + createTarget("second", () => "second", secondCommit, { start: 6, end: 6 }), ); expect(session.ownerKey).toBe("first"); expect(session.controller.currentState.phase).toBe(phase); @@ -131,9 +145,7 @@ describe("global voice input", () => { const { session } = createSession(); let text: string | null = "first"; const commit = vi.fn(); - await session.start( - createVoiceInputTarget("first", () => text, commit, { start: 5, end: 5 }), - ); + await session.start(createTarget("first", () => text, commit, { start: 5, end: 5 })); text = change === "removed" ? null : "edited prompt"; await session.controller.stop(); expect(commit).not.toHaveBeenCalled(); @@ -144,9 +156,7 @@ describe("global voice input", () => { it("finishes the original draft at the recording limit while it is off screen", async () => { const { session, recorder } = createSession(); const commit = vi.fn(); - await session.start( - createVoiceInputTarget("first", () => "first", commit, { start: 5, end: 5 }), - ); + await session.start(createTarget("first", () => "first", commit, { start: 5, end: 5 })); await session.controller.handleRecorderStatus({ isFinished: true, hasError: false, @@ -157,14 +167,41 @@ describe("global voice input", () => { expect(session.controller.currentState.phase).toBe("idle"); }); + it("rejects a transcript when its off-screen draft changes and returns to the original text", async () => { + const registry = AtomRegistry.make(); + const draft = Atom.make("hello world"); + const unsubscribe = vi.fn(); + const { session } = createSession(); + const commit = vi.fn(); + const target = createVoiceInputTarget( + "first", + () => registry.get(draft), + commit, + { start: 6, end: 6 }, + (onChange) => { + const stop = registry.subscribe(draft, onChange); + return () => { + stop(); + unsubscribe(); + }; + }, + ); + await session.start(target); + registry.set(draft, "changed"); + registry.set(draft, "hello world"); + await session.controller.stop(); + expect(commit).not.toHaveBeenCalled(); + expect(session.controller.currentState.error).toContain("draft changed"); + expect(unsubscribe).toHaveBeenCalledTimes(1); + registry.dispose(); + }); + it("stops recording when its queued edit is discarded", async () => { const { session, recorder } = createSession(); const commit = vi.fn(); let draft: string | null = "queued prompt"; const ownerKey = "thread~queued-edit~run"; - await session.start( - createVoiceInputTarget(ownerKey, () => draft, commit, { start: 13, end: 13 }), - ); + await session.start(createTarget(ownerKey, () => draft, commit, { start: 13, end: 13 })); session.cancel(ownerKey); draft = null; await session.controller.stop(); @@ -177,12 +214,26 @@ describe("global voice input", () => { expect(commit).not.toHaveBeenCalled(); }); + it.each(["complete", "cancel"] as const)( + "releases draft observation after %s", + async (finish) => { + const { session } = createSession(); + const unsubscribe = vi.fn(); + const subscribe = vi.fn(() => unsubscribe); + await session.start( + createVoiceInputTarget("first", () => "first", vi.fn(), { start: 5, end: 5 }, subscribe), + ); + expect(subscribe).toHaveBeenCalledTimes(1); + if (finish === "complete") await session.controller.stop(); + else session.cancel("first"); + expect(unsubscribe).toHaveBeenCalledTimes(1); + }, + ); + it("keeps another prompt's recording when a queued edit is discarded", async () => { const { session, recorder } = createSession(); const commit = vi.fn(); - await session.start( - createVoiceInputTarget("other prompt", () => "hello", commit, { start: 5, end: 5 }), - ); + await session.start(createTarget("other prompt", () => "hello", commit, { start: 5, end: 5 })); session.cancel("thread~queued-edit~run"); expect(recorder.stop).not.toHaveBeenCalled(); expect(session.controller.currentState.phase).toBe("recording"); @@ -201,12 +252,12 @@ describe("global voice input", () => { const oldCommit = vi.fn(); const nextCommit = vi.fn(); const firstStart = session.start( - createVoiceInputTarget("first", () => "first", oldCommit, { start: 5, end: 5 }), + createTarget("first", () => "first", oldCommit, { start: 5, end: 5 }), ); await preparationEntered.promise; session.cancel("first"); const nextStart = session.start( - createVoiceInputTarget("second", () => "second", nextCommit, { start: 6, end: 6 }), + createTarget("second", () => "second", nextCommit, { start: 6, end: 6 }), ); expect(prepare).toHaveBeenCalledTimes(1); expect(recorder.record).not.toHaveBeenCalled(); diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.ts b/apps/mobile/src/features/voice-input/voiceInputSession.ts index 7eccea142303..52d8df2378bb 100644 --- a/apps/mobile/src/features/voice-input/voiceInputSession.ts +++ b/apps/mobile/src/features/voice-input/voiceInputSession.ts @@ -9,6 +9,7 @@ export type VoiceInputTarget = { readonly ownerKey: string; readonly readDraft: () => VoiceDraftSnapshot | null; readonly commitDraft: VoiceInputControllerDependencies["commitDraft"]; + readonly subscribe: () => () => void; }; export function createVoiceInputTarget( @@ -16,25 +17,45 @@ export function createVoiceInputTarget( readText: () => string | null, commitDraft: VoiceInputTarget["commitDraft"], selection: VoiceDraftSnapshot["selection"], + subscribeToChanges: (onChange: () => void) => () => void, ): VoiceInputTarget { + let revision = 0; return { ownerKey, readDraft: () => { const text = readText(); if (text === null) return null; - return { ownerKey, text, selection, revision: 0 }; + return { ownerKey, text, selection, revision }; }, commitDraft, + subscribe: () => { + let previousText = readText(); + return subscribeToChanges(() => { + const text = readText(); + if (text !== previousText) { + previousText = text; + revision += 1; + } + }); + }, }; } export class VoiceInputSession { readonly controller: VoiceInputController; private target: VoiceInputTarget | null = null; + private unsubscribe: (() => void) | null = null; constructor(dependencies: Omit) { this.controller = new VoiceInputController({ ...dependencies, + onStateChange: (state) => { + if (!voiceInputBlocksSubmission(state)) { + this.unsubscribe?.(); + this.unsubscribe = null; + } + dependencies.onStateChange(state); + }, readDraft: () => this.target?.readDraft() ?? null, commitDraft: (text, selection) => this.target?.commitDraft(text, selection), }); @@ -55,6 +76,7 @@ export class VoiceInputSession { start(target: VoiceInputTarget): Promise { if (voiceInputBlocksSubmission(this.controller.currentState)) return Promise.resolve(); this.target = target; + this.unsubscribe = target.subscribe(); return this.controller.start(); } } From f7694fb964057b3f9dddc8fc0b7a472ec09b2ece Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 3 Oct 2026 21:39:54 -0700 Subject: [PATCH 08/10] fix(mobile): collapse off-screen dictation into an edge pill The global dictation card spanned the top of every screen, covering the first thread row, and never said which draft it was recording into. It is now a small pill on the trailing edge showing the recording time; tapping it expands a card naming the target thread with the dictation controls. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../SettingsScheduledTasksRouteScreen.tsx | 1 + .../features/threads/NewTaskDraftScreen.tsx | 1 + .../src/features/threads/ThreadComposer.tsx | 1 + .../voice-input/GlobalVoiceInputControl.tsx | 133 +++++++++++++++--- .../voice-input/VoiceInputProvider.tsx | 10 +- .../voice-input/useVoiceInputController.ts | 9 +- .../features/voice-input/voiceInputSession.ts | 6 + 7 files changed, 133 insertions(+), 28 deletions(-) diff --git a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx index ce9162525832..a3219888241a 100644 --- a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx @@ -410,6 +410,7 @@ function SettingsScheduledTaskEditorScreen({ title }: { readonly title: string } ); const voiceInput = useVoiceInputController({ ownerKey, + label: editor?.draft.title.trim() || title, subscribeToDraftChanges: (onChange) => appAtomRegistry.subscribe(scheduledTaskEditorSessionAtom, onChange), selection, diff --git a/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx b/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx index e576787301da..3e2e1837e1da 100644 --- a/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx +++ b/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx @@ -474,6 +474,7 @@ export function NewTaskDraftScreen(props: { }); const voiceInput = useVoiceInputController({ ownerKey: flow.draftKey, + label: selectedProject ? `New task in ${selectedProject.title}` : "New task", readDraftMessage: () => (flow.draftKey ? getComposerDraftSnapshot(flow.draftKey).text : null), subscribeToDraftChanges: (onChange) => appAtomRegistry.subscribe(composerDraftsAtom, onChange), selection: composerMenu.selection, diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index 66efe4f43bb3..5ea142d08b4d 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -509,6 +509,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer }); const voiceInput = useVoiceInputController({ ownerKey: composerDraftKey, + label: props.selectedThread.title || "Untitled thread", readDraftMessage: () => getComposerDraftSnapshot(composerDraftKey).text, subscribeToDraftChanges: (onChange) => appAtomRegistry.subscribe(composerDraftsAtom, onChange), selection: composerMenu.selection, diff --git a/apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx b/apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx index fcabfb7af189..1b034ffbc086 100644 --- a/apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx +++ b/apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx @@ -1,8 +1,11 @@ -import { Platform, View } from "react-native"; +import { useState } from "react"; +import { ActivityIndicator, Platform, Pressable, useWindowDimensions, View } from "react-native"; +import Animated, { FadeIn, FadeOut, ReduceMotion } from "react-native-reanimated"; import { useSafeAreaInsets } from "react-native-safe-area-context"; import { FullWindowOverlay } from "react-native-screens"; -import { APP_BAR_HEIGHT } from "../../lib/layoutMetrics"; +import { AppText as Text } from "../../components/AppText"; +import { SymbolView } from "../../components/AppSymbol"; import { ComposerDictationCancelAction, ComposerDictationPrimaryAction, @@ -11,37 +14,121 @@ import { import { useGlobalVoiceInput } from "./VoiceInputProvider"; import { resolveVoiceComposerPresentation } from "./voiceInputPresentation"; +const EXPANDED_MAX_WIDTH = 360; +const ENTERING = FadeIn.duration(180).reduceMotion(ReduceMotion.System); +const EXITING = FadeOut.duration(120).reduceMotion(ReduceMotion.System); + +/** Keeps an off-screen dictation reachable as a pill on the screen's trailing edge. */ export function GlobalVoiceInputControl() { const voice = useGlobalVoiceInput(); - const insets = useSafeAreaInsets(); const presentation = resolveVoiceComposerPresentation(voice.state, voice.elapsedSeconds); if (!presentation.statusLabel || (voice.ownerKey && voice.focusedOwners.has(voice.ownerKey))) { return null; } - const content = ( + // Mounted only while visible, so each dictation starts collapsed. + const content = ; + return Platform.OS === "ios" ? {content} : content; +} + +function EdgeDictationPill() { + const voice = useGlobalVoiceInput(); + const insets = useSafeAreaInsets(); + const { width, height } = useWindowDimensions(); + const [expanded, setExpanded] = useState(false); + const presentation = resolveVoiceComposerPresentation(voice.state, voice.elapsedSeconds); + const label = voice.label ?? "Draft"; + const isError = presentation.statusKind === "error"; + const elapsedLabel = `${Math.floor(voice.elapsedSeconds / 60)}:${String(voice.elapsedSeconds % 60).padStart(2, "0")}`; + + return ( - - - void voice.session.retry()} - onConfirm={voice.stop} - onCancel={voice.cancel} - /> + {expanded ? ( + + + + Dictating into + + {label} + + + setExpanded(false)} + > + + + + + + + void voice.session.retry()} + onConfirm={voice.stop} + onCancel={voice.cancel} + /> + + + ) : ( + + setExpanded(true)} + > + {isError ? ( + + ) : voice.state.phase === "recording" ? ( + <> + + + {elapsedLabel} + + + ) : ( + + )} + + + )} ); - return Platform.OS === "ios" ? {content} : content; } diff --git a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx index c31ab943ad4b..f6deefcea87b 100644 --- a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx +++ b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx @@ -81,9 +81,10 @@ export function useGlobalVoiceInput() { } function useVoiceInputRuntime() { - const [{ state, ownerKey }, setState] = useState({ + const [{ state, ownerKey, label }, setState] = useState({ state: INITIAL_STATE, ownerKey: null as string | null, + label: null as string | null, }); const [focusedOwners, setFocusedOwners] = useState>(new Set()); const setOwnerFocused = useCallback((key: string, focused: boolean) => { @@ -123,7 +124,11 @@ function useVoiceInputRuntime() { releaseRecording: releaseVoiceRecordingAudio, deleteRecording: (uri) => new File(uri).delete(), onStateChange: (nextState) => - setState({ state: nextState, ownerKey: sessionRef.current?.ownerKey ?? null }), + setState({ + state: nextState, + ownerKey: sessionRef.current?.ownerKey ?? null, + label: sessionRef.current?.label ?? null, + }), }); } @@ -207,6 +212,7 @@ function useVoiceInputRuntime() { elapsedSeconds, isBusy: voiceInputBlocksSubmission(state), ownerKey, + label, focusedOwners, setOwnerFocused, session, diff --git a/apps/mobile/src/features/voice-input/useVoiceInputController.ts b/apps/mobile/src/features/voice-input/useVoiceInputController.ts index 06a35e3c60a2..d1dc234f39f5 100644 --- a/apps/mobile/src/features/voice-input/useVoiceInputController.ts +++ b/apps/mobile/src/features/voice-input/useVoiceInputController.ts @@ -13,6 +13,8 @@ const IDLE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: n export function useVoiceInputController(input: { readonly ownerKey: string | null; + /** Shown by the global dictation pill when this composer is off screen. */ + readonly label: string; readonly readDraftMessage: () => string | null; readonly subscribeToDraftChanges: (onChange: () => void) => () => void; readonly selection: ComposerEditorSelection; @@ -44,8 +46,8 @@ export function useVoiceInputController(input: { const start = useCallback(() => { const captured = latestInput.current; if (!captured.ownerKey || captured.disabled) return; - void session.start( - createVoiceInputTarget( + void session.start({ + ...createVoiceInputTarget( captured.ownerKey, captured.readDraftMessage, (text, selection) => { @@ -57,7 +59,8 @@ export function useVoiceInputController(input: { captured.selection, captured.subscribeToDraftChanges, ), - ); + label: captured.label, + }); }, [session]); const state = global.ownerKey === input.ownerKey ? global.state : IDLE_STATE; const isBusy = voiceInputBlocksSubmission(state); diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.ts b/apps/mobile/src/features/voice-input/voiceInputSession.ts index 52d8df2378bb..45983654601f 100644 --- a/apps/mobile/src/features/voice-input/voiceInputSession.ts +++ b/apps/mobile/src/features/voice-input/voiceInputSession.ts @@ -7,6 +7,8 @@ import { export type VoiceInputTarget = { readonly ownerKey: string; + /** Names the draft being dictated into while its composer is off screen. */ + readonly label?: string; readonly readDraft: () => VoiceDraftSnapshot | null; readonly commitDraft: VoiceInputControllerDependencies["commitDraft"]; readonly subscribe: () => () => void; @@ -65,6 +67,10 @@ export class VoiceInputSession { return this.target?.ownerKey ?? null; } + get label(): string | null { + return this.target?.label ?? null; + } + cancel(ownerKey: string | null): void { if (ownerKey !== null && this.ownerKey === ownerKey) this.controller.cancel(); } From 5bc83a045852ed8cf395912e42b673ee971af010 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 3 Oct 2026 22:03:30 -0700 Subject: [PATCH 09/10] fix(mobile): dock the dictation pill above the bottom toolbar Moves the off-screen dictation pill down to sit just above the bottom toolbar or composer, morphs its width when it expands, and slims the expanded state to one row: cancel, the target thread's title, a compact waveform with the elapsed time, and finish. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../voice-input/ComposerDictationControl.tsx | 33 ++- .../voice-input/GlobalVoiceInputControl.tsx | 235 +++++++++++++----- 2 files changed, 197 insertions(+), 71 deletions(-) diff --git a/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx b/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx index 602cbb158b54..f27d2359639a 100644 --- a/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx +++ b/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx @@ -157,8 +157,10 @@ export function ComposerDictationToolbar(props: { const WaveformBar = memo(function WaveformBar(props: { readonly audioLevels: SharedValue; readonly sampleIndex: number; + readonly height?: number; }) { const { audioLevels, sampleIndex } = props; + const height = props.height ?? WAVEFORM_BAR_HEIGHT; const animatedStyle = useAnimatedStyle(() => { const level = audioLevels.value[sampleIndex] ?? 0; return { @@ -166,8 +168,7 @@ const WaveformBar = memo(function WaveformBar(props: { transform: [ { scaleY: withTiming( - (WAVEFORM_MIN_BAR_HEIGHT + level * (WAVEFORM_BAR_HEIGHT - WAVEFORM_MIN_BAR_HEIGHT)) / - WAVEFORM_BAR_HEIGHT, + (WAVEFORM_MIN_BAR_HEIGHT + level * (height - WAVEFORM_MIN_BAR_HEIGHT)) / height, WAVEFORM_TIMING, ), }, @@ -178,11 +179,37 @@ const WaveformBar = memo(function WaveformBar(props: { return ( ); }); +const COMPACT_WAVEFORM_BAR_COUNT = 8; +const COMPACT_WAVEFORM_BAR_HEIGHT = 14; + +/** A fixed-size waveform of the latest samples, for tight spaces like the global dictation pill. */ +export const CompactVoiceWaveform = memo(function CompactVoiceWaveform(props: { + readonly audioLevels: SharedValue; +}) { + return ( + + {Array.from({ length: COMPACT_WAVEFORM_BAR_COUNT }, (_, index) => ( + + ))} + + ); +}); + const VoiceWaveform = memo(function VoiceWaveform(props: { readonly audioLevels: SharedValue; }) { diff --git a/apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx b/apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx index 1b034ffbc086..c86c82cfd935 100644 --- a/apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx +++ b/apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx @@ -1,22 +1,44 @@ -import { useState } from "react"; -import { ActivityIndicator, Platform, Pressable, useWindowDimensions, View } from "react-native"; -import Animated, { FadeIn, FadeOut, ReduceMotion } from "react-native-reanimated"; +import { useEffect, useState } from "react"; +import { + ActivityIndicator, + Linking, + Platform, + Pressable, + useWindowDimensions, + View, +} from "react-native"; +import Animated, { + Easing, + Extrapolation, + FadeIn, + interpolate, + ReduceMotion, + useAnimatedStyle, + useSharedValue, + withTiming, +} from "react-native-reanimated"; import { useSafeAreaInsets } from "react-native-safe-area-context"; import { FullWindowOverlay } from "react-native-screens"; import { AppText as Text } from "../../components/AppText"; import { SymbolView } from "../../components/AppSymbol"; -import { - ComposerDictationCancelAction, - ComposerDictationPrimaryAction, - ComposerDictationStatus, -} from "./ComposerDictationControl"; +import { useAndroidControlSizing } from "../../components/useAndroidControlSizing"; +import { cn } from "../../lib/cn"; +import { CompactVoiceWaveform } from "./ComposerDictationControl"; import { useGlobalVoiceInput } from "./VoiceInputProvider"; import { resolveVoiceComposerPresentation } from "./voiceInputPresentation"; -const EXPANDED_MAX_WIDTH = 360; +const PILL_HEIGHT = 36; +const COLLAPSED_WIDTH = 64; +const EXPANDED_MAX_WIDTH = 280; +// Clears Home's floating bottom toolbar (56) and a collapsed thread composer (60). +const IOS_BOTTOM_CHROME_CLEARANCE = 64; +const MORPH_TIMING = { + duration: 260, + easing: Easing.out(Easing.cubic), + reduceMotion: ReduceMotion.System, +} as const; const ENTERING = FadeIn.duration(180).reduceMotion(ReduceMotion.System); -const EXITING = FadeOut.duration(120).reduceMotion(ReduceMotion.System); /** Keeps an off-screen dictation reachable as a pill on the screen's trailing edge. */ export function GlobalVoiceInputControl() { @@ -33,75 +55,61 @@ export function GlobalVoiceInputControl() { function EdgeDictationPill() { const voice = useGlobalVoiceInput(); const insets = useSafeAreaInsets(); - const { width, height } = useWindowDimensions(); + const { width } = useWindowDimensions(); + const { fabSize } = useAndroidControlSizing(); const [expanded, setExpanded] = useState(false); + const progress = useSharedValue(0); + useEffect(() => { + progress.value = withTiming(expanded ? 1 : 0, MORPH_TIMING); + }, [expanded, progress]); + + const expandedWidth = Math.min(width - insets.left - insets.right - 32, EXPANDED_MAX_WIDTH); + // Each face keeps its final width and stays pinned to the edge, so the + // morphing pill reveals the expanded row instead of reflowing it. + const pillStyle = useAnimatedStyle(() => ({ + width: interpolate(progress.value, [0, 1], [COLLAPSED_WIDTH, expandedWidth]), + })); + const collapsedStyle = useAnimatedStyle(() => ({ + opacity: interpolate(progress.value, [0, 0.4], [1, 0], Extrapolation.CLAMP), + })); + const expandedStyle = useAnimatedStyle(() => ({ + opacity: interpolate(progress.value, [0.3, 1], [0, 1], Extrapolation.CLAMP), + })); + + // Sit just above the bottom chrome: the toolbar or composer on iOS, the + // new-thread FAB on Android. + const bottom = + Platform.OS === "android" + ? Math.max(insets.bottom, 16) + 16 + fabSize + 12 + : Math.max(insets.bottom, 12) + IOS_BOTTOM_CHROME_CLEARANCE; + const presentation = resolveVoiceComposerPresentation(voice.state, voice.elapsedSeconds); + const phase = voice.state.phase; + const isError = phase === "error"; + const openSettings = isError && voice.state.errorAction === "settings"; const label = voice.label ?? "Draft"; - const isError = presentation.statusKind === "error"; const elapsedLabel = `${Math.floor(voice.elapsedSeconds / 60)}:${String(voice.elapsedSeconds % 60).padStart(2, "0")}`; return ( - - {expanded ? ( + - - - Dictating into - - {label} - - - setExpanded(false)} - > - - - - - - - void voice.session.retry()} - onConfirm={voice.stop} - onCancel={voice.cancel} - /> - - - ) : ( - setExpanded(true)} > @@ -112,7 +120,7 @@ function EdgeDictationPill() { tintColorClassName="accent-danger-foreground" type="monochrome" /> - ) : voice.state.phase === "recording" ? ( + ) : phase === "recording" ? ( <> - )} - + + + + + + setExpanded(false)} + > + + {isError ? presentation.statusLabel : label} + + + {phase === "recording" ? ( + <> + + + {elapsedLabel} + + + + + + + + ) : isError ? ( + voice.isAvailable ? ( + { + if (!openSettings) { + void voice.session.retry(); + return; + } + voice.cancel(); + void Linking.openSettings(); + }} + > + + + ) : null + ) : ( + + + + )} + + + ); } From 166f2eb95ee717edbe125c9733b9ebba7ba221bf Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 3 Oct 2026 22:29:33 -0700 Subject: [PATCH 10/10] fix(mobile): collapse the dictation pill on any outside touch The expanded pill could only be collapsed by tapping its title. Any touch that starts elsewhere in the app, including a scroll, now collapses it without claiming the touch. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/mobile/src/App.tsx | 7 ++- .../voice-input/GlobalVoiceInputControl.tsx | 44 +++++++++++++++---- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/apps/mobile/src/App.tsx b/apps/mobile/src/App.tsx index c2ccd631127b..645bc0b8a2fb 100644 --- a/apps/mobile/src/App.tsx +++ b/apps/mobile/src/App.tsx @@ -1,7 +1,7 @@ import * as Linking from "expo-linking"; import * as SplashScreen from "expo-splash-screen"; import { useEffect } from "react"; -import { StatusBar, View } from "react-native"; +import { StatusBar } from "react-native"; import { GestureHandlerRootView } from "react-native-gesture-handler"; import { KeyboardProvider } from "react-native-keyboard-controller"; import { SafeAreaProvider } from "react-native-safe-area-context"; @@ -85,17 +85,16 @@ function AppContent() { this, React Navigation defaults to its light theme and every native header (glass buttons, title, materials) is forced light even when the system is in dark mode. */} - + - + {/* Anchored-menu overlays render here — in-window, so the keyboard stays up while a dropdown is open. */} - diff --git a/apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx b/apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx index c86c82cfd935..4159041e2853 100644 --- a/apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx +++ b/apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from "react"; +import { useEffect, useRef, useState, type ReactNode, type RefObject } from "react"; import { ActivityIndicator, Linking, @@ -40,19 +40,37 @@ const MORPH_TIMING = { } as const; const ENTERING = FadeIn.duration(180).reduceMotion(ReduceMotion.System); -/** Keeps an off-screen dictation reachable as a pill on the screen's trailing edge. */ -export function GlobalVoiceInputControl() { +/** + * Keeps an off-screen dictation reachable as a pill on the screen's trailing + * edge. Wraps the app content so any touch that starts outside the pill, + * including a scroll, collapses it without claiming the touch. + */ +export function GlobalVoiceInputControl(props: { readonly children: ReactNode }) { const voice = useGlobalVoiceInput(); + const collapseRef = useRef<(() => void) | null>(null); const presentation = resolveVoiceComposerPresentation(voice.state, voice.elapsedSeconds); - if (!presentation.statusLabel || (voice.ownerKey && voice.focusedOwners.has(voice.ownerKey))) { - return null; - } + const visible = + presentation.statusLabel !== null && + !(voice.ownerKey && voice.focusedOwners.has(voice.ownerKey)); // Mounted only while visible, so each dictation starts collapsed. - const content = ; - return Platform.OS === "ios" ? {content} : content; + const pill = visible ? : null; + return ( + <> + { + collapseRef.current?.(); + return false; + }} + > + {props.children} + + {pill && Platform.OS === "ios" ? {pill} : pill} + + ); } -function EdgeDictationPill() { +function EdgeDictationPill(props: { readonly collapseRef: RefObject<(() => void) | null> }) { const voice = useGlobalVoiceInput(); const insets = useSafeAreaInsets(); const { width } = useWindowDimensions(); @@ -62,6 +80,14 @@ function EdgeDictationPill() { useEffect(() => { progress.value = withTiming(expanded ? 1 : 0, MORPH_TIMING); }, [expanded, progress]); + const { collapseRef } = props; + useEffect(() => { + if (!expanded) return; + collapseRef.current = () => setExpanded(false); + return () => { + collapseRef.current = null; + }; + }, [collapseRef, expanded]); const expandedWidth = Math.min(width - insets.left - insets.right - 32, EXPANDED_MAX_WIDTH); // Each face keeps its final width and stays pinned to the edge, so the