From f76337e0bdfc11b7e7419081fd7cabf84546fd21 Mon Sep 17 00:00:00 2001 From: Bailey Mullens Date: Sat, 1 Aug 2026 21:29:42 -0500 Subject: [PATCH] feat(messages): read any message aloud with the huddle voice Add a ChatGPT-style speaker action to every chat message. On desktop the new speak_message_read_aloud/stop_message_read_aloud Tauri commands drive the huddle's Pocket TTS pipeline with the user's selected Settings voice (browser speechSynthesis remains the fallback outside Tauri, e.g. the web dev server and the e2e mock bridge). A playback bar above the composer (channels, threads, and the Home inbox) shows preparing/playing/error states with stop, pause/resume (web engine), and retry. Markdown is flattened word-for-word before synthesis; code spans are preserved verbatim and retries reuse the flattened text so code content is never re-parsed as Markdown. Huddle-started cards are excluded from the action. Mobile keeps the iOS AVSpeechSynthesizer bridge, un-gated to all messages. The initial-window reveal helpers move from lib.rs to initial_window.rs to keep lib.rs under the file-size ratchet after registering the new commands. Co-Authored-By: Claude Fable 5 Signed-off-by: Bailey Mullens --- desktop/playwright.config.ts | 4 + .../src/huddle/message_read_aloud.rs | 134 +++++++ desktop/src-tauri/src/huddle/mod.rs | 1 + desktop/src-tauri/src/huddle/tts_settings.rs | 2 +- desktop/src-tauri/src/initial_window.rs | 72 ++++ desktop/src-tauri/src/lib.rs | 71 +--- .../ui/ChannelComposerActivityAccessory.tsx | 64 +-- .../features/communities/useCommunityInit.ts | 2 + .../src/features/home/ui/InboxDetailPane.tsx | 2 + .../messages/lib/messageReadAloud.test.mjs | 27 ++ .../features/messages/lib/messageReadAloud.ts | 375 ++++++++++++++++++ .../features/messages/ui/MessageActionBar.tsx | 103 +++++ .../messages/ui/MessageReadAloudBar.tsx | 107 +++++ .../message-read-aloud-screenshots.spec.ts | 114 ++++++ mobile/ios/Runner/AppDelegate.swift | 108 ++++- .../channels/channel_detail_page.dart | 10 + .../channel_detail_page/message_bubble.dart | 1 + .../features/channels/message_actions.dart | 225 +++++++---- .../features/channels/message_read_aloud.dart | 245 ++++++++++++ .../channels/message_read_aloud_bar.dart | 122 ++++++ .../features/channels/thread_detail_page.dart | 10 + .../channels/message_read_aloud_test.dart | 31 ++ 22 files changed, 1648 insertions(+), 182 deletions(-) create mode 100644 desktop/src-tauri/src/huddle/message_read_aloud.rs create mode 100644 desktop/src-tauri/src/initial_window.rs create mode 100644 desktop/src/features/messages/lib/messageReadAloud.test.mjs create mode 100644 desktop/src/features/messages/lib/messageReadAloud.ts create mode 100644 desktop/src/features/messages/ui/MessageReadAloudBar.tsx create mode 100644 desktop/tests/e2e/message-read-aloud-screenshots.spec.ts create mode 100644 mobile/lib/features/channels/message_read_aloud.dart create mode 100644 mobile/lib/features/channels/message_read_aloud_bar.dart create mode 100644 mobile/test/features/channels/message_read_aloud_test.dart diff --git a/desktop/playwright.config.ts b/desktop/playwright.config.ts index 7ce7f48389a..ec4c6593475 100644 --- a/desktop/playwright.config.ts +++ b/desktop/playwright.config.ts @@ -11,6 +11,9 @@ export default defineConfig({ ], use: { baseURL: "http://127.0.0.1:4173", + launchOptions: process.env.PLAYWRIGHT_CHROME_EXECUTABLE + ? { executablePath: process.env.PLAYWRIGHT_CHROME_EXECUTABLE } + : undefined, screenshot: "only-on-failure", trace: "on-first-retry", video: "retain-on-failure", @@ -35,6 +38,7 @@ export default defineConfig({ "**/invites-settings-screenshots.spec.ts", "**/messaging.spec.ts", "**/message-feedback-snapshots.spec.ts", + "**/message-read-aloud-screenshots.spec.ts", "**/custom-emoji.spec.ts", "**/profile-custom-emoji-status.spec.ts", "**/custom-emoji-ui.spec.ts", diff --git a/desktop/src-tauri/src/huddle/message_read_aloud.rs b/desktop/src-tauri/src/huddle/message_read_aloud.rs new file mode 100644 index 00000000000..6596d4b5f07 --- /dev/null +++ b/desktop/src-tauri/src/huddle/message_read_aloud.rs @@ -0,0 +1,134 @@ +//! On-demand read-aloud of a single chat message through the local Pocket +//! TTS engine — the same pipeline (and selected Settings voice) the huddle +//! uses for agent speech. +//! +//! Mirrors the `preview_pocket_voice` structure: a short-lived pipeline per +//! request, completion detected by polling the pipeline's active flag. Only +//! one message plays at a time; starting a new one cancels the previous. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use tauri::{AppHandle, Emitter, State}; + +use crate::app_state::AppState; + +use super::models; +use super::tts::TtsPipeline; +use super::tts_settings::{current_settings, pocket_voice_reference}; + +/// Emitted (payload: the caller's `session_id`) when audio playback actually +/// starts, so the UI can move from "preparing" to "playing". +const STARTED_EVENT: &str = "message-read-aloud-started"; + +/// How long synthesis may run before the first audible sample. Matches the +/// voice-preview timeout. +const PLAYBACK_START_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + +/// Poll interval for the playback progress flags. +const POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(25); + +/// Cancel flag for the in-flight message read-aloud playback, if any. +/// Starting a new read-aloud cancels the previous one; stop raises it. +static ACTIVE_CANCEL: Mutex>> = Mutex::new(None); + +/// Swap the shared cancel slot, returning the previous occupant. +fn swap_active_cancel(next: Option>) -> Option> { + let mut slot = ACTIVE_CANCEL + .lock() + .unwrap_or_else(|error| error.into_inner()); + std::mem::replace(&mut *slot, next) +} + +/// Speak `text` through the local Pocket TTS engine with the user's selected +/// Settings voice. +/// +/// Resolves `Ok(true)` once playback finishes, `Ok(false)` when cancelled by +/// [`stop_message_read_aloud`] or a newer read-aloud request. Errors when the +/// voice files are not ready, no Pocket voice is available, or audio never +/// starts. +#[tauri::command] +pub async fn speak_message_read_aloud( + session_id: String, + text: String, + app: AppHandle, + state: State<'_, AppState>, +) -> Result { + if text.trim().is_empty() { + return Err("Nothing to read aloud".to_string()); + } + if !models::is_tts_ready() { + return Err("Voice files are still downloading. Try again shortly.".to_string()); + } + let model_dir = models::tts_model_dir().ok_or("Pocket voice files are unavailable")?; + let settings = current_settings(&state)?; + let voice_name = pocket_voice_reference(&app, &settings.voice_preferences)?; + let output_device = state + .huddle_audio + .output_device + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone(); + + // Only one message plays at a time — cancel whichever was still running. + let cancel = Arc::new(AtomicBool::new(false)); + if let Some(previous) = swap_active_cancel(Some(Arc::clone(&cancel))) { + previous.store(true, Ordering::Release); + } + let cancel_worker = Arc::clone(&cancel); + + let result = tokio::task::spawn_blocking(move || { + let active = Arc::new(AtomicBool::new(false)); + let pipeline = TtsPipeline::new_with_voice( + model_dir, + Arc::clone(&active), + Arc::clone(&cancel_worker), + &voice_name, + output_device, + )?; + pipeline.speak(text)?; + let started = std::time::Instant::now(); + let mut heard_audio = false; + loop { + if cancel_worker.load(Ordering::Acquire) { + return Ok(false); + } + let is_active = active.load(Ordering::Acquire); + if is_active && !heard_audio { + heard_audio = true; + let _ = app.emit(STARTED_EVENT, &session_id); + } + if heard_audio && !is_active { + return Ok(true); + } + if !heard_audio && started.elapsed() > PLAYBACK_START_TIMEOUT { + return Err( + "Playback did not start. Check your audio output and try again.".to_string(), + ); + } + std::thread::sleep(POLL_INTERVAL); + } + }) + .await + .map_err(|error| format!("Read-aloud task failed: {error}"))?; + + // Release the slot if it still belongs to this request so a stale Arc + // doesn't linger after playback ends. + { + let mut slot = ACTIVE_CANCEL + .lock() + .unwrap_or_else(|error| error.into_inner()); + if slot.as_ref().is_some_and(|held| Arc::ptr_eq(held, &cancel)) { + *slot = None; + } + } + result +} + +/// Stop the in-flight message read-aloud, if any. +#[tauri::command] +pub fn stop_message_read_aloud() { + if let Some(cancel) = swap_active_cancel(None) { + cancel.store(true, Ordering::Release); + } +} diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index 03264f80f4c..74520532757 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -27,6 +27,7 @@ mod agent_tts_routing; pub mod agents; pub mod audio_output; pub mod jitter; +pub mod message_read_aloud; pub mod models; pub mod pipeline; pub mod playout; diff --git a/desktop/src-tauri/src/huddle/tts_settings.rs b/desktop/src-tauri/src/huddle/tts_settings.rs index 1b378af8237..1068027aa82 100644 --- a/desktop/src-tauri/src/huddle/tts_settings.rs +++ b/desktop/src-tauri/src/huddle/tts_settings.rs @@ -463,7 +463,7 @@ async fn apply_tts_settings( Ok(voice_change_wait) } -fn current_settings(state: &AppState) -> Result { +pub(crate) fn current_settings(state: &AppState) -> Result { state .huddle_audio .tts diff --git a/desktop/src-tauri/src/initial_window.rs b/desktop/src-tauri/src/initial_window.rs new file mode 100644 index 00000000000..6b9f294e733 --- /dev/null +++ b/desktop/src-tauri/src/initial_window.rs @@ -0,0 +1,72 @@ +//! First-reveal choreography for the main window: keep it hidden (with an +//! opaque backing on macOS) until the initial render is ready and the +//! window-state plugin has settled restored geometry, then show and focus. + +#[cfg(target_os = "macos")] +pub(crate) const INITIAL_RENDER_READY_EVENT: &str = "initial-render-ready"; + +pub(crate) fn reveal_initial_window(window: &tauri::Window) { + if let Err(error) = window.show() { + eprintln!("buzz-desktop: failed to reveal main window: {error}"); + return; + } + if let Err(error) = window.set_focus() { + eprintln!("buzz-desktop: failed to focus main window: {error}"); + } +} + +#[cfg(target_os = "macos")] +pub(crate) fn set_initial_window_backing(window: &tauri::Window) { + // The window remains transparent at runtime for vibrancy. Use an opaque + // native backing only across the first visible frames so the previous app + // cannot show through before WebKit has submitted its first surface. + if let Err(error) = window.set_background_color(Some(tauri::window::Color(17, 21, 24, 255))) { + eprintln!("buzz-desktop: failed to set initial window backing: {error}"); + } +} + +#[cfg(target_os = "macos")] +pub(crate) async fn clear_initial_window_backing(window: &tauri::Window) { + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + if let Err(error) = window.set_background_color(None) { + eprintln!("buzz-desktop: failed to clear initial window backing: {error}"); + } +} + +#[cfg(target_os = "macos")] +pub(crate) async fn wait_for_stable_initial_window_geometry( + window: &tauri::Window, +) { + const MAX_POLLS: usize = 120; + const REQUIRED_STABLE_POLLS: usize = 4; + + let mut previous_bounds = None; + let mut stable_polls = 0; + + for _ in 0..MAX_POLLS { + // Accept whatever geometry the window-state plugin restores — maximized + // or a normal saved size. macOS applies the restore asynchronously, so + // we only need consecutive identical outer bounds to know it settled. + // Gating on `is_maximized()` here would leave `bounds` permanently + // `None` for restored non-maximized windows and stall the reveal until + // the poll timeout. + let bounds = match (window.outer_position(), window.outer_size()) { + (Ok(position), Ok(size)) => Some((position.x, position.y, size.width, size.height)), + _ => None, + }; + + if bounds.is_some() && bounds == previous_bounds { + stable_polls += 1; + if stable_polls >= REQUIRED_STABLE_POLLS { + return; + } + } else { + stable_polls = 0; + } + previous_bounds = bounds; + + tokio::time::sleep(std::time::Duration::from_millis(16)).await; + } + + eprintln!("buzz-desktop: initial window geometry did not settle before reveal timeout"); +} diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index c4b733e3e07..5afda46dbc2 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -9,6 +9,7 @@ mod event_sync; mod events; mod huddle; mod identity_storage; +mod initial_window; mod key_backup; mod linux_media; mod managed_agents; @@ -54,6 +55,7 @@ use huddle::{ join_huddle, leave_huddle, push_audio_pcm, set_huddle_transcription_enabled, set_tts_enabled, set_voice_input_mode, speak_agent_message, start_huddle, start_stt_pipeline, }; +use initial_window::*; use managed_agents::{ backfill_persona_snapshots, ensure_nest, list_managed_agent_runtimes, put_managed_agent_runtime_lifecycle, reconcile_managed_agent_runtimes, @@ -73,73 +75,6 @@ use tauri_plugin_window_state::StateFlags; #[cfg(target_os = "macos")] use tray_menu::show_main_window; -#[cfg(target_os = "macos")] -const INITIAL_RENDER_READY_EVENT: &str = "initial-render-ready"; - -fn reveal_initial_window(window: &tauri::Window) { - if let Err(error) = window.show() { - eprintln!("buzz-desktop: failed to reveal main window: {error}"); - return; - } - if let Err(error) = window.set_focus() { - eprintln!("buzz-desktop: failed to focus main window: {error}"); - } -} - -#[cfg(target_os = "macos")] -fn set_initial_window_backing(window: &tauri::Window) { - // The window remains transparent at runtime for vibrancy. Use an opaque - // native backing only across the first visible frames so the previous app - // cannot show through before WebKit has submitted its first surface. - if let Err(error) = window.set_background_color(Some(tauri::window::Color(17, 21, 24, 255))) { - eprintln!("buzz-desktop: failed to set initial window backing: {error}"); - } -} - -#[cfg(target_os = "macos")] -async fn clear_initial_window_backing(window: &tauri::Window) { - tokio::time::sleep(std::time::Duration::from_millis(250)).await; - if let Err(error) = window.set_background_color(None) { - eprintln!("buzz-desktop: failed to clear initial window backing: {error}"); - } -} - -#[cfg(target_os = "macos")] -async fn wait_for_stable_initial_window_geometry(window: &tauri::Window) { - const MAX_POLLS: usize = 120; - const REQUIRED_STABLE_POLLS: usize = 4; - - let mut previous_bounds = None; - let mut stable_polls = 0; - - for _ in 0..MAX_POLLS { - // Accept whatever geometry the window-state plugin restores — maximized - // or a normal saved size. macOS applies the restore asynchronously, so - // we only need consecutive identical outer bounds to know it settled. - // Gating on `is_maximized()` here would leave `bounds` permanently - // `None` for restored non-maximized windows and stall the reveal until - // the poll timeout. - let bounds = match (window.outer_position(), window.outer_size()) { - (Ok(position), Ok(size)) => Some((position.x, position.y, size.width, size.height)), - _ => None, - }; - - if bounds.is_some() && bounds == previous_bounds { - stable_polls += 1; - if stable_polls >= REQUIRED_STABLE_POLLS { - return; - } - } else { - stable_polls = 0; - } - previous_bounds = bounds; - - tokio::time::sleep(std::time::Duration::from_millis(16)).await; - } - - eprintln!("buzz-desktop: initial window geometry did not settle before reveal timeout"); -} - #[cfg_attr(mobile, tauri::mobile_entry_point)] pub fn run() { // mesh-llm's async chains (model download, node start/join) overflow @@ -898,6 +833,8 @@ pub fn run() { huddle::tts_settings::preview_pocket_voice, huddle::tts_settings::import_pocket_voice, huddle::tts_settings::delete_pocket_voice, + huddle::message_read_aloud::speak_message_read_aloud, + huddle::message_read_aloud::stop_message_read_aloud, speak_agent_message, add_agent_to_huddle, check_pipeline_hotstart, diff --git a/desktop/src/features/channels/ui/ChannelComposerActivityAccessory.tsx b/desktop/src/features/channels/ui/ChannelComposerActivityAccessory.tsx index f99888f0112..529d8c2cf94 100644 --- a/desktop/src/features/channels/ui/ChannelComposerActivityAccessory.tsx +++ b/desktop/src/features/channels/ui/ChannelComposerActivityAccessory.tsx @@ -4,6 +4,7 @@ import { CardMintComposerChip } from "@/features/agents/ui/CardMintComposerChip" import { useCardMintJobs } from "@/features/agents/cardMintStore"; import { BotActivityComposerAction } from "@/features/channels/ui/BotActivityBar"; import { ComposerActivityAccessory } from "@/features/messages/ui/ComposerActivityAccessory"; +import { MessageReadAloudBar } from "@/features/messages/ui/MessageReadAloudBar"; import { TypingIndicatorRow } from "@/features/messages/ui/TypingIndicatorRow"; type ChannelComposerActivityAccessoryProps = { @@ -35,36 +36,41 @@ export function ChannelComposerActivityAccessory({ }: ChannelComposerActivityAccessoryProps) { const cardMintJobs = useCardMintJobs(); return ( - -
- {cardMintJobs.length > 0 ? : null} - {workingBotPubkeys.length > 0 ? ( -
- +
+ +
+ +
+ {cardMintJobs.length > 0 ? : null} + {workingBotPubkeys.length > 0 ? ( +
+ +
+ ) : null} + {typingPubkeys.length > 0 ? ( + -
- ) : null} - {typingPubkeys.length > 0 ? ( - - ) : null} -
- + ) : null} +
+ + ); } diff --git a/desktop/src/features/communities/useCommunityInit.ts b/desktop/src/features/communities/useCommunityInit.ts index afa69f913f6..6564c69dda7 100644 --- a/desktop/src/features/communities/useCommunityInit.ts +++ b/desktop/src/features/communities/useCommunityInit.ts @@ -31,6 +31,7 @@ import { resetAvatarProfileSync } from "@/features/profile/avatarProfileSync"; import { resetSidebarRelayConnectionCardState } from "@/features/sidebar/ui/useSidebarRelayConnectionCard"; import { clearMarkdownNodeCache } from "@/shared/ui/markdown/nodeCache"; import { resetVideoPlayerState } from "@/shared/ui/videoPlayerState"; +import { stopMessageReadAloud } from "@/features/messages/lib/messageReadAloud"; import { initFirstCommunity, @@ -65,6 +66,7 @@ function resetCommunityState({ } resetSidebarRelayConnectionCardState(); resetMediaCaches(); + stopMessageReadAloud(); resetVideoPlayerState(); resetRenderScopedReactionHydration(); clearSearchHitEventCache(); diff --git a/desktop/src/features/home/ui/InboxDetailPane.tsx b/desktop/src/features/home/ui/InboxDetailPane.tsx index 9b6ff57e172..c22aa4a6f63 100644 --- a/desktop/src/features/home/ui/InboxDetailPane.tsx +++ b/desktop/src/features/home/ui/InboxDetailPane.tsx @@ -34,6 +34,7 @@ import { orderMentionPubkeysByText } from "@/features/messages/lib/orderMentionP import { getThreadReference } from "@/features/messages/lib/threading"; import { normalizePubkey } from "@/shared/lib/pubkey"; import { MessageComposer } from "@/features/messages/ui/MessageComposer"; +import { MessageReadAloudBar } from "@/features/messages/ui/MessageReadAloudBar"; import { useAnchoredScroll } from "@/features/messages/ui/useAnchoredScroll"; import { useComposerHeightPadding } from "@/features/messages/ui/useComposerHeightPadding"; import { UpdateIndicator } from "@/features/settings/UpdateIndicator"; @@ -633,6 +634,7 @@ function InboxMessageDetailPane({ }} />
+ { + assert.equal( + messageTextForSpeech( + "## **Update**\n\nRead [the guide](https://example.com), then run `buzz check`.\n\n```sh\nbuzz status\n```", + ), + "Update\n\nRead the guide, then run buzz check.\n\nbuzz status", + ); +}); + +test("messageTextForSpeech keeps image alt text and list content", () => { + assert.equal( + messageTextForSpeech("- First\n- ![Diagram](https://example.com/a.png)"), + "First\nDiagram", + ); +}); + +test("messageTextForSpeech preserves code identifiers exactly", () => { + assert.equal( + messageTextForSpeech("Use `message_read_aloud.dart` and `__init__` next."), + "Use message_read_aloud.dart and __init__ next.", + ); +}); diff --git a/desktop/src/features/messages/lib/messageReadAloud.ts b/desktop/src/features/messages/lib/messageReadAloud.ts new file mode 100644 index 00000000000..8cddc409518 --- /dev/null +++ b/desktop/src/features/messages/lib/messageReadAloud.ts @@ -0,0 +1,375 @@ +import * as React from "react"; + +export type MessageReadAloudStatus = + | "idle" + | "preparing" + | "playing" + | "paused" + | "finished" + | "error"; + +/** + * Which speech backend is driving the current playback. + * + * "native" is the Pocket TTS pipeline in the Tauri backend — the same engine + * and selected Settings voice the huddle uses. "web" is the browser's + * `speechSynthesis`, kept as the fallback for non-Tauri contexts (web dev + * server, e2e mock bridge). Native supports play/stop only; pause/resume is + * web-only. + */ +export type MessageReadAloudEngine = "native" | "web"; + +export type MessageReadAloudState = { + author: string; + engine: MessageReadAloudEngine; + error: string | null; + messageId: string | null; + status: MessageReadAloudStatus; + text: string; +}; + +const IDLE_STATE: MessageReadAloudState = { + author: "", + engine: "web", + error: null, + messageId: null, + status: "idle", + text: "", +}; + +let state = IDLE_STATE; +let activeUtterance: SpeechSynthesisUtterance | null = null; +let nativeSessionCounter = 0; +let currentNativeSessionId: string | null = null; +let nativeStartedListenerReady: Promise | null = null; +const listeners = new Set<() => void>(); + +function publish(next: MessageReadAloudState) { + state = next; + for (const listener of listeners) listener(); +} + +function speechEngine() { + if (typeof window === "undefined") return null; + return window.speechSynthesis ?? null; +} + +function nativeEngineAvailable() { + if (typeof window === "undefined") return false; + const globals = window as unknown as Record; + // The e2e mock bridge defines __TAURI_INTERNALS__ but not the read-aloud + // commands — specs drive the web engine through a speechSynthesis mock. + return ( + "__TAURI_INTERNALS__" in globals && + !("__BUZZ_E2E_INVOKE_MOCK_COMMAND__" in globals) + ); +} + +/** Flatten common Markdown markers while preserving every readable word. */ +export function messageTextForSpeech(markdown: string) { + const codeSpans: string[] = []; + const preserveCode = (text: string) => { + const index = codeSpans.push(text) - 1; + return `${index}`; + }; + + return markdown + .replace(/```(?:[^\n`]*)\n?([\s\S]*?)```/g, (_match, code: string) => + preserveCode(code), + ) + .replace(/`([^`]+)`/g, (_match, code: string) => preserveCode(code)) + .replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1") + .replace(/\[([^\]]+)\]\([^)]*\)/g, "$1") + .replace(/^\s{0,3}(?:#{1,6}|>|[-+*]|\d+[.)])\s+/gm, "") + .replace(/\*\*([^*\n]+)\*\*/g, "$1") + .replace(/(? codeSpans[Number(index)] ?? "", + ) + .replace(/\n{3,}/g, "\n\n") + .trim(); +} + +function readAloudErrorMessage(error: unknown) { + if (typeof error === "string" && error.trim().length > 0) return error; + return "Couldn't play audio. Try again."; +} + +function ensureNativeStartedListener() { + if (!nativeStartedListenerReady) { + nativeStartedListenerReady = import("@tauri-apps/api/event").then( + ({ listen }) => + listen("message-read-aloud-started", (event) => { + if (event.payload !== currentNativeSessionId) return; + if (state.status !== "preparing") return; + publish({ ...state, status: "playing" }); + }).then(() => undefined), + ); + } + return nativeStartedListenerReady; +} + +async function beginNativeSpeech( + messageId: string, + author: string, + text: string, +) { + const sessionId = String(++nativeSessionCounter); + currentNativeSessionId = sessionId; + activeUtterance = null; + speechEngine()?.cancel(); + publish({ + author, + engine: "native", + error: null, + messageId, + status: "preparing", + text, + }); + try { + await ensureNativeStartedListener(); + const { invoke } = await import("@tauri-apps/api/core"); + const finished = await invoke("speak_message_read_aloud", { + sessionId, + text, + }); + if (currentNativeSessionId !== sessionId) return; + currentNativeSessionId = null; + if (finished) { + publish({ + author, + engine: "native", + error: null, + messageId, + status: "finished", + text, + }); + } else if (state.messageId === messageId && state.status !== "idle") { + publish(IDLE_STATE); + } + } catch (error) { + if (currentNativeSessionId !== sessionId) return; + currentNativeSessionId = null; + console.error("message read aloud failed", error); + publish({ + author, + engine: "native", + error: readAloudErrorMessage(error), + messageId, + status: "error", + text, + }); + } +} + +function beginWebSpeech(messageId: string, author: string, text: string) { + const engine = speechEngine(); + if (!engine || typeof SpeechSynthesisUtterance === "undefined") { + publish({ + author, + engine: "web", + error: "Couldn't play audio. Try again.", + messageId, + status: "error", + text, + }); + return; + } + + engine.cancel(); + const utterance = new SpeechSynthesisUtterance(text); + activeUtterance = utterance; + publish({ + author, + engine: "web", + error: null, + messageId, + status: "preparing", + text, + }); + + utterance.onstart = () => { + if (activeUtterance !== utterance) return; + publish({ + author, + engine: "web", + error: null, + messageId, + status: "playing", + text, + }); + }; + utterance.onpause = () => { + if (activeUtterance !== utterance) return; + publish({ + author, + engine: "web", + error: null, + messageId, + status: "paused", + text, + }); + }; + utterance.onresume = () => { + if (activeUtterance !== utterance) return; + publish({ + author, + engine: "web", + error: null, + messageId, + status: "playing", + text, + }); + }; + utterance.onend = () => { + if (activeUtterance !== utterance) return; + activeUtterance = null; + publish({ + author, + engine: "web", + error: null, + messageId, + status: "finished", + text, + }); + }; + utterance.onerror = (event) => { + if (activeUtterance !== utterance || event.error === "canceled") return; + activeUtterance = null; + console.error("message read aloud failed", event.error); + publish({ + author, + engine: "web", + error: "Couldn't play audio. Try again.", + messageId, + status: "error", + text, + }); + }; + + engine.speak(utterance); +} + +function speakText(messageId: string, author: string, text: string) { + if (!text) { + publish({ + author, + engine: "web", + error: "Couldn't play audio. Try again.", + messageId, + status: "error", + text, + }); + return; + } + if (nativeEngineAvailable()) { + void beginNativeSpeech(messageId, author, text); + return; + } + beginWebSpeech(messageId, author, text); +} + +function beginSpeech(messageId: string, author: string, markdown: string) { + speakText(messageId, author, messageTextForSpeech(markdown)); +} + +export function listenToMessage( + messageId: string, + author: string, + markdown: string, +) { + if ( + state.messageId === messageId && + state.status === "paused" && + state.engine === "web" + ) { + speechEngine()?.resume(); + return; + } + if (state.messageId === messageId && state.status === "playing") { + if (state.engine === "native") { + stopMessageReadAloud(); + } else { + speechEngine()?.pause(); + } + return; + } + beginSpeech(messageId, author, markdown); +} + +export function pauseMessageReadAloud() { + if (state.status === "playing" && state.engine === "web") { + speechEngine()?.pause(); + } +} + +export function resumeMessageReadAloud() { + if (state.status === "paused" && state.engine === "web") { + speechEngine()?.resume(); + } +} + +export function retryMessageReadAloud() { + // `state.text` is already flattened — do not re-run messageTextForSpeech, + // which would treat code content (e.g. `__init__`) as Markdown markers. + if (state.messageId) speakText(state.messageId, state.author, state.text); +} + +function stopNativeSpeech() { + currentNativeSessionId = null; + void import("@tauri-apps/api/core") + .then(({ invoke }) => invoke("stop_message_read_aloud")) + .catch((error) => { + console.error("message read aloud stop failed", error); + }); +} + +export function stopMessageReadAloud() { + if ( + state.engine === "native" && + (state.status === "preparing" || state.status === "playing") + ) { + stopNativeSpeech(); + } + activeUtterance = null; + speechEngine()?.cancel(); + publish(IDLE_STATE); +} + +if (typeof window !== "undefined") { + window.addEventListener("hashchange", stopMessageReadAloud); + window.addEventListener("pagehide", stopMessageReadAloud); +} + +export function getMessageReadAloudState() { + return state; +} + +export function subscribeMessageReadAloud(listener: () => void) { + listeners.add(listener); + return () => listeners.delete(listener); +} + +export function useMessageReadAloud() { + return React.useSyncExternalStore( + subscribeMessageReadAloud, + getMessageReadAloudState, + getMessageReadAloudState, + ); +} + +export function useMessageReadAloudForMessage(messageId: string) { + const getSnapshot = React.useCallback( + () => (state.messageId === messageId ? state : IDLE_STATE), + [messageId], + ); + return React.useSyncExternalStore( + subscribeMessageReadAloud, + getSnapshot, + getSnapshot, + ); +} diff --git a/desktop/src/features/messages/ui/MessageActionBar.tsx b/desktop/src/features/messages/ui/MessageActionBar.tsx index 967e50f5d2f..d927357aa47 100644 --- a/desktop/src/features/messages/ui/MessageActionBar.tsx +++ b/desktop/src/features/messages/ui/MessageActionBar.tsx @@ -7,11 +7,17 @@ import { EllipsisVertical, Flag, Link2, + LoaderCircle, MailCheck, MailOpen, + Pause, Pencil, + Play, + RotateCcw, SmilePlus, + Square, Trash2, + Volume2, } from "lucide-react"; import * as React from "react"; @@ -35,6 +41,10 @@ import { copyTextToClipboard } from "@/shared/lib/clipboard"; import { emojiDisplayName } from "@/shared/lib/emojiName"; import { rewriteRelayUrl } from "@/shared/lib/mediaUrl"; import { KIND_HUDDLE_STARTED } from "@/shared/constants/kinds"; +import { + listenToMessage, + useMessageReadAloudForMessage, +} from "@/features/messages/lib/messageReadAloud"; import { Button } from "@/shared/ui/button"; import { DeleteMessageConfirmDialog } from "./DeleteMessageConfirmDialog"; import { @@ -60,6 +70,9 @@ function MoreActionsMenu({ onMarkUnread, onMarkRead, onOpenChange, + onListen, + listenDisabled, + listenLabel, onRemindLater, onUnfollowThread, open, @@ -76,6 +89,9 @@ function MoreActionsMenu({ onMarkUnread?: (message: TimelineMessage) => void; onMarkRead?: (message: TimelineMessage) => void; onOpenChange: (open: boolean) => void; + onListen?: () => void; + listenDisabled?: boolean; + listenLabel?: string; onRemindLater?: (message: TimelineMessage) => void; onUnfollowThread?: (message: TimelineMessage) => void; open: boolean; @@ -149,6 +165,13 @@ function MoreActionsMenu({ ) : null} + {onListen ? ( + + + {listenLabel ?? "Listen to message"} + + ) : null} + {onMarkRead || onMarkUnread ? ( { + if (!isThisMessage || playback.status !== "preparing") { + setShowPreparing(false); + return; + } + const timeout = window.setTimeout(() => setShowPreparing(true), 200); + return () => window.clearTimeout(timeout); + }, [isThisMessage, playback.status]); + const listenLabel = isThisMessage + ? playback.status === "preparing" + ? "Preparing audio…" + : playback.status === "playing" + ? playback.engine === "native" + ? "Stop reading" + : "Pause reading" + : playback.status === "paused" + ? "Resume reading" + : playback.status === "finished" + ? "Replay message" + : "Listen to message" + : "Listen to message"; + const canListen = + !message.pending && + message.kind !== KIND_HUDDLE_STARTED && + message.body.trim().length > 0; + const handleListen = React.useCallback(() => { + listenToMessage(message.id, message.author, message.body); + }, [message.author, message.body, message.id]); const hasMoreMenuActions = Boolean(onEdit) || @@ -398,6 +452,7 @@ export const MessageActionBar = React.memo(function MessageActionBar({ Boolean(onFollowThread) || Boolean(onUnfollowThread) || Boolean(onRemindLater) || + canListen || !message.pending; const wouldAddReaction = React.useCallback( @@ -534,6 +589,51 @@ export const MessageActionBar = React.memo(function MessageActionBar({ ) : null} + {canListen ? ( + + + + + {listenLabel} + + ) : null} + {hasMoreMenuActions ? ( diff --git a/desktop/src/features/messages/ui/MessageReadAloudBar.tsx b/desktop/src/features/messages/ui/MessageReadAloudBar.tsx new file mode 100644 index 00000000000..8bfada33a48 --- /dev/null +++ b/desktop/src/features/messages/ui/MessageReadAloudBar.tsx @@ -0,0 +1,107 @@ +import * as React from "react"; +import { LoaderCircle, Pause, Play, RotateCcw, Volume2, X } from "lucide-react"; + +import { + pauseMessageReadAloud, + resumeMessageReadAloud, + retryMessageReadAloud, + stopMessageReadAloud, + useMessageReadAloud, +} from "@/features/messages/lib/messageReadAloud"; +import { Button } from "@/shared/ui/button"; + +export function MessageReadAloudBar() { + const playback = useMessageReadAloud(); + const [showPreparing, setShowPreparing] = React.useState(false); + React.useEffect(() => { + if (playback.status !== "preparing") { + setShowPreparing(false); + return; + } + const timeout = window.setTimeout(() => setShowPreparing(true), 200); + return () => window.clearTimeout(timeout); + }, [playback.status]); + if ( + playback.status === "idle" || + playback.status === "finished" || + (playback.status === "preparing" && !showPreparing) || + !playback.messageId + ) { + return null; + } + + const isPaused = playback.status === "paused"; + const isError = playback.status === "error"; + const isPreparing = playback.status === "preparing"; + // The native Pocket engine supports play/stop only — the X button stops. + const canPause = playback.engine === "web"; + + return ( +
+ {isPreparing ? ( + + ) : ( + + )} +

+ {isError ? ( + (playback.error ?? "Couldn't play audio. Try again.") + ) : isPreparing ? ( + "Preparing audio…" + ) : ( + <> + Reading message from{" "} + + {playback.author} + + + )} +

+ {isError ? ( + + ) : isPreparing || !canPause ? null : ( + + )} + +
+ ); +} diff --git a/desktop/tests/e2e/message-read-aloud-screenshots.spec.ts b/desktop/tests/e2e/message-read-aloud-screenshots.spec.ts new file mode 100644 index 00000000000..a4b65b2f3a1 --- /dev/null +++ b/desktop/tests/e2e/message-read-aloud-screenshots.spec.ts @@ -0,0 +1,114 @@ +import { expect, test } from "@playwright/test"; + +import { installMockBridge, TEST_IDENTITIES } from "../helpers/bridge"; + +test.beforeEach(async ({ page }) => { + await page.addInitScript(() => { + class MockSpeechSynthesisUtterance extends EventTarget { + error: string | null = null; + onend: (() => void) | null = null; + onerror: ((event: { error: string }) => void) | null = null; + onpause: (() => void) | null = null; + onresume: (() => void) | null = null; + onstart: (() => void) | null = null; + + constructor(public text: string) { + super(); + } + } + + let active: MockSpeechSynthesisUtterance | null = null; + const speechSynthesis = { + cancel() { + active = null; + }, + getVoices() { + return []; + }, + pause() { + active?.onpause?.(); + }, + paused: false, + pending: false, + resume() { + active?.onresume?.(); + }, + speak(utterance: MockSpeechSynthesisUtterance) { + active = utterance; + window.setTimeout(() => { + if (active === utterance) utterance.onstart?.(); + }, 250); + }, + speaking: false, + }; + + Object.defineProperty(window, "SpeechSynthesisUtterance", { + configurable: true, + value: MockSpeechSynthesisUtterance, + }); + Object.defineProperty(window, "speechSynthesis", { + configurable: true, + value: speechSynthesis, + }); + }); + await installMockBridge(page, { + searchProfiles: [ + { + pubkey: TEST_IDENTITIES.alice.pubkey, + displayName: "Honey", + ownerPubkey: TEST_IDENTITIES.tyler.pubkey, + isAgent: true, + }, + ], + }); +}); + +test("messages expose private read-aloud controls", async ({ + page, +}, testInfo) => { + await page.setViewportSize({ width: 1280, height: 820 }); + await page.goto("/"); + await page.getByTestId("channel-general").click(); + + const agentMessage = page + .getByTestId("message-row") + .filter({ hasText: "Hey team — checking in." }); + const humanMessage = page + .getByTestId("message-row") + .filter({ hasText: "Welcome to #general" }); + + await agentMessage.hover(); + const listenButton = agentMessage.getByRole("button", { + name: "Listen to message", + }); + await expect(listenButton).toBeVisible(); + await humanMessage.hover(); + await expect( + humanMessage.getByRole("button", { name: "Listen to message" }), + ).toBeVisible(); + await agentMessage.hover(); + await page.screenshot({ + path: testInfo.outputPath("agent-listen-hover.png"), + }); + + await listenButton.click(); + const readAloudBar = page.getByTestId("message-read-aloud-bar"); + await expect(readAloudBar).toContainText("Reading message from Honey"); + await expect( + readAloudBar.getByRole("button", { name: "Pause reading" }), + ).toBeVisible(); + await page.screenshot({ + path: testInfo.outputPath("agent-now-reading.png"), + }); + + await readAloudBar.getByRole("button", { name: "Pause reading" }).click(); + await expect( + readAloudBar.getByRole("button", { name: "Resume reading" }), + ).toBeVisible(); + await readAloudBar.getByRole("button", { name: "Resume reading" }).click(); + await expect( + readAloudBar.getByRole("button", { name: "Pause reading" }), + ).toBeVisible(); + await readAloudBar.getByRole("button", { name: "Stop reading" }).click(); + await expect(readAloudBar).toHaveCount(0); +}); diff --git a/mobile/ios/Runner/AppDelegate.swift b/mobile/ios/Runner/AppDelegate.swift index f6fe3aaf489..5a1de0b71bf 100644 --- a/mobile/ios/Runner/AppDelegate.swift +++ b/mobile/ios/Runner/AppDelegate.swift @@ -4,11 +4,17 @@ import UIKit import UserNotifications @main -@objc class AppDelegate: FlutterAppDelegate, FlutterImplicitEngineDelegate { +@objc class AppDelegate: FlutterAppDelegate, FlutterImplicitEngineDelegate, + AVSpeechSynthesizerDelegate +{ private var mediaUploadChannel: FlutterMethodChannel? private var qrScannerChannel: FlutterMethodChannel? private var inlinePhotoPickerSupportChannel: FlutterMethodChannel? private var nativeAttachmentPopoverCoordinator: NativeAttachmentPopoverCoordinator? + private var messageReadAloudChannel: FlutterMethodChannel? + private let messageSpeechSynthesizer = AVSpeechSynthesizer() + private var speakingMessageId: String? + private var speakingUtterance: AVSpeechUtterance? override func application( _ application: UIApplication, @@ -50,6 +56,14 @@ import UserNotifications result(false) } } + messageReadAloudChannel = FlutterMethodChannel( + name: "buzz/message_read_aloud", + binaryMessenger: messenger + ) + messageSpeechSynthesizer.delegate = self + messageReadAloudChannel?.setMethodCallHandler { [weak self] call, result in + self?.handleMessageReadAloudMethodCall(call, result: result) + } if let inlinePhotoPickerRegistrar = engineBridge.pluginRegistry.registrar( forPlugin: "BuzzInlinePhotoPicker" @@ -72,6 +86,98 @@ import UserNotifications ) } + private func handleMessageReadAloudMethodCall( + _ call: FlutterMethodCall, + result: @escaping FlutterResult + ) { + switch call.method { + case "speak": + guard + let arguments = call.arguments as? [String: Any], + let messageId = arguments["messageId"] as? String, + let author = arguments["author"] as? String, + let text = arguments["text"] as? String, + !text.isEmpty + else { + result( + FlutterError( + code: "invalid_arguments", + message: "Expected a message id, author, and non-empty text.", + details: nil + ) + ) + return + } + + if messageSpeechSynthesizer.isSpeaking || messageSpeechSynthesizer.isPaused { + speakingMessageId = nil + speakingUtterance = nil + messageSpeechSynthesizer.stopSpeaking(at: .immediate) + } + do { + let audioSession = AVAudioSession.sharedInstance() + try audioSession.setCategory( + .playback, + mode: .spokenAudio, + options: [.duckOthers] + ) + try audioSession.setActive(true) + } catch { + result( + FlutterError( + code: "audio_session_failed", + message: "Unable to start message audio.", + details: error.localizedDescription + ) + ) + return + } + let utterance = AVSpeechUtterance(string: text) + utterance.rate = AVSpeechUtteranceDefaultSpeechRate + speakingMessageId = messageId + speakingUtterance = utterance + messageSpeechSynthesizer.speak(utterance) + UIAccessibility.post( + notification: .announcement, + argument: "Reading message from \(author)" + ) + result(nil) + case "pause": + result(messageSpeechSynthesizer.pauseSpeaking(at: .word)) + case "resume": + result(messageSpeechSynthesizer.continueSpeaking()) + case "stop": + speakingMessageId = nil + speakingUtterance = nil + if messageSpeechSynthesizer.isSpeaking || messageSpeechSynthesizer.isPaused { + messageSpeechSynthesizer.stopSpeaking(at: .immediate) + } + try? AVAudioSession.sharedInstance().setActive( + false, + options: [.notifyOthersOnDeactivation] + ) + result(nil) + default: + result(FlutterMethodNotImplemented) + } + } + + func speechSynthesizer( + _ synthesizer: AVSpeechSynthesizer, + didFinish utterance: AVSpeechUtterance + ) { + guard speakingUtterance === utterance, let messageId = speakingMessageId else { + return + } + speakingMessageId = nil + speakingUtterance = nil + try? AVAudioSession.sharedInstance().setActive( + false, + options: [.notifyOthersOnDeactivation] + ) + messageReadAloudChannel?.invokeMethod("finished", arguments: messageId) + } + private static func handleQrScannerMethodCall( _ call: FlutterMethodCall, result: @escaping FlutterResult diff --git a/mobile/lib/features/channels/channel_detail_page.dart b/mobile/lib/features/channels/channel_detail_page.dart index 044342d101f..ef60284134a 100644 --- a/mobile/lib/features/channels/channel_detail_page.dart +++ b/mobile/lib/features/channels/channel_detail_page.dart @@ -40,6 +40,8 @@ import 'manage_channel_sheet.dart'; import 'members_sheet.dart'; import 'message_actions.dart'; import 'message_content.dart'; +import 'message_read_aloud.dart'; +import 'message_read_aloud_bar.dart'; import 'read_state/deferred_read_state_update.dart'; import 'read_state/read_state_provider.dart'; import 'read_state/read_state_time.dart'; @@ -194,6 +196,13 @@ class ChannelDetailPage extends HookConsumerWidget { return null; }, [channel.id]); + final readAloudNotifier = ref.read(messageReadAloudProvider.notifier); + useEffect( + () => + () => readAloudNotifier.stop(), + [channel.id, readAloudNotifier], + ); + useEffect(() { final messageId = initialMessageId; if (messageId == null || channel.isForum) return null; @@ -381,6 +390,7 @@ class ChannelDetailPage extends HookConsumerWidget { ? const SizedBox.shrink() : ChannelTypingIndicator(entries: typingEntries), ), + const MessageReadAloudBar(), if (!resolvedChannel.isForum && resolvedChannel.isMember && !resolvedChannel.isArchived) diff --git a/mobile/lib/features/channels/channel_detail_page/message_bubble.dart b/mobile/lib/features/channels/channel_detail_page/message_bubble.dart index cb6ab670659..88a626a0afc 100644 --- a/mobile/lib/features/channels/channel_detail_page/message_bubble.dart +++ b/mobile/lib/features/channels/channel_detail_page/message_bubble.dart @@ -99,6 +99,7 @@ class _MessageBubble extends ConsumerWidget { message: message, channelId: currentChannelId, canManageMessage: canManageMessage, + messageAuthor: displayName, allMessages: allMessages, currentPubkey: currentPubkey, isMember: isMember, diff --git a/mobile/lib/features/channels/message_actions.dart b/mobile/lib/features/channels/message_actions.dart index 519d25898c0..402e1a54dc7 100644 --- a/mobile/lib/features/channels/message_actions.dart +++ b/mobile/lib/features/channels/message_actions.dart @@ -22,6 +22,7 @@ import '../../shared/reminders/remind_me_later_sheet.dart'; import '../../shared/reminders/reminder_service.dart'; import 'channel_management_provider.dart'; import 'emoji_picker.dart'; +import 'message_read_aloud.dart'; import 'reaction_row.dart'; import 'recent_emoji_provider.dart'; import 'read_state/message_read_state.dart'; @@ -41,6 +42,7 @@ void showMessageActions({ required TimelineMessage message, required String channelId, required bool canManageMessage, + String? messageAuthor, List? allMessages, String? currentPubkey, bool isMember = false, @@ -50,97 +52,152 @@ void showMessageActions({ context: context, isScrollControlled: true, showDragHandle: true, - builder: (sheetContext) => SafeArea( - child: ConstrainedBox( - constraints: BoxConstraints( - maxHeight: MediaQuery.sizeOf(sheetContext).height * 0.7, - ), - child: SingleChildScrollView( - child: Padding( - padding: const EdgeInsets.fromLTRB( - Grid.gutter, - 0, - Grid.gutter, - Grid.xs, + builder: (sheetContext) => Consumer( + builder: (sheetContext, sheetRef, _) { + final playback = sheetRef.watch(messageReadAloudProvider); + final isThisMessage = playback.messageId == message.id; + final listenLabel = isThisMessage + ? switch (playback.status) { + MessageReadAloudStatus.preparing => 'Preparing audio…', + MessageReadAloudStatus.playing => 'Pause reading', + MessageReadAloudStatus.paused => 'Resume reading', + MessageReadAloudStatus.finished => 'Replay message', + _ => 'Listen to message', + } + : 'Listen to message'; + return SafeArea( + child: ConstrainedBox( + constraints: BoxConstraints( + maxHeight: MediaQuery.sizeOf(sheetContext).height * 0.7, ), - child: Column( - mainAxisSize: MainAxisSize.min, - children: [ - _QuickReactionRow( - message: message, - sheetContext: sheetContext, - pageContext: context, - pageRef: ref, + child: SingleChildScrollView( + child: Padding( + padding: const EdgeInsets.fromLTRB( + Grid.gutter, + 0, + Grid.gutter, + Grid.xs, ), - const SizedBox(height: Grid.xs), - if (!message.isSystem) ...[ - // Fast actions: respond now, hand off context, defer. - _FastActionsRow( - message: message, - channelId: channelId, - allMessages: allMessages, - currentPubkey: currentPubkey, - isMember: isMember, - isArchived: isArchived, - pageContext: context, - ), - const SizedBox(height: Grid.xs), - // Triage: come back to this message later. - _MarkReadUnreadTile(message: message, channelId: channelId), - _FollowThreadTile(message: message), - const SheetDivider(), - // Export: take the content out of the conversation. - ListTile( - leading: const Icon(LucideIcons.copy), - title: const Text('Copy text'), - onTap: () { - Navigator.of(sheetContext).pop(); - // Copy to clipboard - final data = ClipboardData(text: message.content); - Clipboard.setData(data); - }, - ), - ], - if (canManageMessage) ...[ - if (!message.isSystem) const SheetDivider(), - ListTile( - leading: const Icon(LucideIcons.pencil), - title: const Text('Edit message'), - onTap: () { - Navigator.of(sheetContext).pop(); - _showEditSheet( - context: context, - ref: ref, + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + _QuickReactionRow( + message: message, + sheetContext: sheetContext, + pageContext: context, + pageRef: ref, + ), + const SizedBox(height: Grid.xs), + if (!message.isSystem) ...[ + // Fast actions: respond now, hand off context, defer. + _FastActionsRow( message: message, channelId: channelId, - ); - }, - ), - ListTile( - leading: Icon( - LucideIcons.trash2, - color: sheetContext.colors.error, - ), - title: Text( - 'Delete message', - style: TextStyle(color: sheetContext.colors.error), - ), - onTap: () { - Navigator.of(sheetContext).pop(); - _confirmDelete( - context: context, - ref: ref, + allMessages: allMessages, + currentPubkey: currentPubkey, + isMember: isMember, + isArchived: isArchived, + pageContext: context, + ), + const SizedBox(height: Grid.xs), + if (defaultTargetPlatform == TargetPlatform.iOS && + message.content.trim().isNotEmpty) + ListTile( + leading: Icon( + isThisMessage && + playback.status == + MessageReadAloudStatus.playing + ? LucideIcons.pause + : isThisMessage && + playback.status == + MessageReadAloudStatus.paused + ? LucideIcons.play + : isThisMessage && + playback.status == + MessageReadAloudStatus.finished + ? LucideIcons.rotateCcw + : LucideIcons.volume2, + ), + title: Text(listenLabel), + onTap: + isThisMessage && + playback.status == + MessageReadAloudStatus.preparing + ? null + : () { + Navigator.of(sheetContext).pop(); + unawaited( + sheetRef + .read(messageReadAloudProvider.notifier) + .listen( + messageId: message.id, + author: messageAuthor ?? 'Agent', + markdown: message.content, + ), + ); + }, + ), + // Triage: come back to this message later. + _MarkReadUnreadTile( + message: message, channelId: channelId, - messageId: message.id, - ); - }, - ), - ], - ], + ), + _FollowThreadTile(message: message), + const SheetDivider(), + // Export: take the content out of the conversation. + ListTile( + leading: const Icon(LucideIcons.copy), + title: const Text('Copy text'), + onTap: () { + Navigator.of(sheetContext).pop(); + // Copy to clipboard + final data = ClipboardData(text: message.content); + Clipboard.setData(data); + }, + ), + ], + if (canManageMessage) ...[ + if (!message.isSystem) const SheetDivider(), + ListTile( + leading: const Icon(LucideIcons.pencil), + title: const Text('Edit message'), + onTap: () { + Navigator.of(sheetContext).pop(); + _showEditSheet( + context: context, + ref: ref, + message: message, + channelId: channelId, + ); + }, + ), + ListTile( + leading: Icon( + LucideIcons.trash2, + color: sheetContext.colors.error, + ), + title: Text( + 'Delete message', + style: TextStyle(color: sheetContext.colors.error), + ), + onTap: () { + Navigator.of(sheetContext).pop(); + _confirmDelete( + context: context, + ref: ref, + channelId: channelId, + messageId: message.id, + ); + }, + ), + ], + ], + ), + ), ), ), - ), - ), + ); + }, ), ); } diff --git a/mobile/lib/features/channels/message_read_aloud.dart b/mobile/lib/features/channels/message_read_aloud.dart new file mode 100644 index 00000000000..2b55be5e4bb --- /dev/null +++ b/mobile/lib/features/channels/message_read_aloud.dart @@ -0,0 +1,245 @@ +import 'dart:async'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter/widgets.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; + +enum MessageReadAloudStatus { + idle, + preparing, + playing, + paused, + finished, + error, +} + +@immutable +class MessageReadAloudState { + final String? messageId; + final String author; + final String text; + final MessageReadAloudStatus status; + final String? error; + + const MessageReadAloudState({ + this.messageId, + this.author = '', + this.text = '', + this.status = MessageReadAloudStatus.idle, + this.error, + }); +} + +String messageTextForSpeech(String markdown) { + final codeSpans = []; + String preserveCode(String text) { + final index = codeSpans.length; + codeSpans.add(text); + return '\uE000$index\uE001'; + } + + return markdown + .replaceAllMapped( + RegExp(r'```(?:[^\n`]*)\n?([\s\S]*?)```'), + (match) => preserveCode(match.group(1) ?? ''), + ) + .replaceAllMapped( + RegExp(r'`([^`]+)`'), + (match) => preserveCode(match.group(1) ?? ''), + ) + .replaceAllMapped( + RegExp(r'!\[([^\]]*)\]\([^)]*\)'), + (match) => match.group(1) ?? '', + ) + .replaceAllMapped( + RegExp(r'\[([^\]]+)\]\([^)]*\)'), + (match) => match.group(1) ?? '', + ) + .replaceAll( + RegExp(r'^\s{0,3}(?:#{1,6}|>|[-+*]|\d+[.)])\s+', multiLine: true), + '', + ) + .replaceAllMapped( + RegExp(r'\*\*([^*\n]+)\*\*'), + (match) => match.group(1) ?? '', + ) + .replaceAllMapped( + RegExp(r'(? match.group(1) ?? '', + ) + .replaceAllMapped( + RegExp(r'(? match.group(1) ?? '', + ) + .replaceAllMapped( + RegExp(r'(? match.group(1) ?? '', + ) + .replaceAllMapped( + RegExp(r'~~([^~\n]+)~~'), + (match) => match.group(1) ?? '', + ) + .replaceAllMapped( + RegExp(r'\uE000(\d+)\uE001'), + (match) => codeSpans[int.parse(match.group(1)!)], + ) + .replaceAll(RegExp(r'\n{3,}'), '\n\n') + .trim(); +} + +class MessageReadAloudNotifier extends Notifier { + static const _channel = MethodChannel('buzz/message_read_aloud'); + AppLifecycleListener? _lifecycleListener; + bool get _isSupported => defaultTargetPlatform == TargetPlatform.iOS; + + @override + MessageReadAloudState build() { + _channel.setMethodCallHandler(_handleNativeCallback); + _lifecycleListener?.dispose(); + _lifecycleListener = AppLifecycleListener(onPause: stop, onDetach: stop); + ref.onDispose(() { + _lifecycleListener?.dispose(); + _lifecycleListener = null; + _channel.setMethodCallHandler(null); + if (_isSupported) unawaited(_channel.invokeMethod('stop')); + }); + return const MessageReadAloudState(); + } + + Future listen({ + required String messageId, + required String author, + required String markdown, + }) async { + if (state.messageId == messageId) { + if (state.status == MessageReadAloudStatus.playing || + state.status == MessageReadAloudStatus.preparing) { + await pause(); + return; + } + if (state.status == MessageReadAloudStatus.paused) { + await resume(); + return; + } + } + + await _speakText( + messageId: messageId, + author: author, + text: messageTextForSpeech(markdown), + ); + } + + Future _speakText({ + required String messageId, + required String author, + required String text, + }) async { + state = MessageReadAloudState( + messageId: messageId, + author: author, + text: text, + status: MessageReadAloudStatus.preparing, + ); + + if (!_isSupported || text.isEmpty) { + _fail(messageId); + return; + } + + try { + await _channel.invokeMethod('speak', { + 'messageId': messageId, + 'author': author, + 'text': text, + }); + if (state.messageId == messageId && + state.status == MessageReadAloudStatus.preparing) { + state = MessageReadAloudState( + messageId: messageId, + author: author, + text: text, + status: MessageReadAloudStatus.playing, + ); + } + } catch (error) { + debugPrint('message read aloud failed: $error'); + _fail(messageId); + } + } + + Future pause() async { + if (state.status != MessageReadAloudStatus.playing) return; + final paused = await _channel.invokeMethod('pause') ?? false; + if (paused) { + state = MessageReadAloudState( + messageId: state.messageId, + author: state.author, + text: state.text, + status: MessageReadAloudStatus.paused, + ); + } + } + + Future resume() async { + if (state.status != MessageReadAloudStatus.paused) return; + final resumed = await _channel.invokeMethod('resume') ?? false; + if (resumed) { + state = MessageReadAloudState( + messageId: state.messageId, + author: state.author, + text: state.text, + status: MessageReadAloudStatus.playing, + ); + } + } + + Future retry() async { + final messageId = state.messageId; + if (messageId == null) return; + // `state.text` is already flattened — re-running messageTextForSpeech + // would treat code content (e.g. `__init__`) as Markdown markers. + await _speakText( + messageId: messageId, + author: state.author, + text: state.text, + ); + } + + void stop() { + if (_isSupported) unawaited(_channel.invokeMethod('stop')); + state = const MessageReadAloudState(); + } + + Future _handleNativeCallback(MethodCall call) async { + final messageId = call.arguments as String?; + if (messageId == null || state.messageId != messageId) return; + if (call.method == 'finished') { + state = MessageReadAloudState( + messageId: state.messageId, + author: state.author, + text: state.text, + status: MessageReadAloudStatus.finished, + ); + } else if (call.method == 'failed') { + _fail(messageId); + } + } + + void _fail(String messageId) { + if (state.messageId != messageId) return; + state = MessageReadAloudState( + messageId: state.messageId, + author: state.author, + text: state.text, + status: MessageReadAloudStatus.error, + error: "Couldn't play audio. Try again.", + ); + } +} + +final messageReadAloudProvider = + NotifierProvider( + MessageReadAloudNotifier.new, + ); diff --git a/mobile/lib/features/channels/message_read_aloud_bar.dart b/mobile/lib/features/channels/message_read_aloud_bar.dart new file mode 100644 index 00000000000..e600a76d773 --- /dev/null +++ b/mobile/lib/features/channels/message_read_aloud_bar.dart @@ -0,0 +1,122 @@ +import 'dart:async'; + +import 'package:flutter/material.dart'; +import 'package:flutter_hooks/flutter_hooks.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:lucide_icons_flutter/lucide_icons.dart'; + +import '../../shared/theme/theme.dart'; +import 'message_read_aloud.dart'; + +class MessageReadAloudBar extends HookConsumerWidget { + const MessageReadAloudBar({super.key}); + + @override + Widget build(BuildContext context, WidgetRef ref) { + final playback = ref.watch(messageReadAloudProvider); + final showPreparing = useState(false); + useEffect(() { + if (playback.status != MessageReadAloudStatus.preparing) { + showPreparing.value = false; + return null; + } + final timer = Timer(const Duration(milliseconds: 200), () { + showPreparing.value = true; + }); + return timer.cancel; + }, [playback.status]); + if (playback.messageId == null || + playback.status == MessageReadAloudStatus.idle || + playback.status == MessageReadAloudStatus.finished || + (playback.status == MessageReadAloudStatus.preparing && + !showPreparing.value)) { + return const SizedBox.shrink(); + } + + final notifier = ref.read(messageReadAloudProvider.notifier); + final isPaused = playback.status == MessageReadAloudStatus.paused; + final isError = playback.status == MessageReadAloudStatus.error; + final isPreparing = playback.status == MessageReadAloudStatus.preparing; + + return Semantics( + liveRegion: true, + label: isError + ? playback.error + : isPreparing + ? 'Preparing audio' + : 'Reading message from ${playback.author}', + child: Container( + margin: const EdgeInsets.fromLTRB( + Grid.gutter, + Grid.quarter, + Grid.gutter, + Grid.quarter, + ), + padding: const EdgeInsets.symmetric( + horizontal: Grid.half, + vertical: Grid.quarter, + ), + constraints: const BoxConstraints(minHeight: 44), + decoration: BoxDecoration( + color: context.colors.surface, + border: Border.all(color: context.colors.outlineVariant), + borderRadius: BorderRadius.circular(Radii.lg), + ), + child: Row( + children: [ + if (isPreparing) + SizedBox.square( + dimension: 18, + child: CircularProgressIndicator( + strokeWidth: 2, + color: context.colors.primary, + ), + ) + else + Icon( + LucideIcons.volume2, + size: 18, + color: context.colors.primary, + ), + const SizedBox(width: Grid.half), + Expanded( + child: Text( + isError + ? "Couldn't play audio. Try again." + : isPreparing + ? 'Preparing audio…' + : 'Reading message from ${playback.author}', + maxLines: 1, + overflow: TextOverflow.ellipsis, + style: context.textTheme.bodySmall?.copyWith( + color: context.colors.onSurfaceVariant, + ), + ), + ), + if (isError) + TextButton.icon( + onPressed: () => unawaited(notifier.retry()), + icon: const Icon(LucideIcons.rotateCcw, size: 16), + label: const Text('Retry'), + ) + else if (!isPreparing) + IconButton( + tooltip: isPaused ? 'Resume reading' : 'Pause reading', + onPressed: () => + unawaited(isPaused ? notifier.resume() : notifier.pause()), + icon: Icon( + isPaused ? LucideIcons.play : LucideIcons.pause, + size: 18, + ), + ), + IconButton( + tooltip: 'Stop reading', + onPressed: notifier.stop, + icon: const Icon(LucideIcons.x, size: 18), + ), + ], + ), + ), + ); + } +} diff --git a/mobile/lib/features/channels/thread_detail_page.dart b/mobile/lib/features/channels/thread_detail_page.dart index 810861aa00f..e8f34eea15d 100644 --- a/mobile/lib/features/channels/thread_detail_page.dart +++ b/mobile/lib/features/channels/thread_detail_page.dart @@ -25,6 +25,8 @@ import 'day_divider.dart'; import '../profile/user_profile_sheet.dart'; import 'message_actions.dart'; import 'message_content.dart'; +import 'message_read_aloud.dart'; +import 'message_read_aloud_bar.dart'; import 'reaction_row.dart'; import 'read_state/read_state_format.dart'; import 'read_state/read_state_provider.dart'; @@ -67,6 +69,12 @@ class ThreadDetailPage extends HookConsumerWidget { ThreadRepliesArgs(channelId: channelId, rootId: queryRootId), ), ); + final readAloudNotifier = ref.read(messageReadAloudProvider.notifier); + useEffect( + () => + () => readAloudNotifier.stop(), + [channelId, threadHead.id, readAloudNotifier], + ); // The thread query is one-shot and asks only for content kinds, so a // reaction, edit, or deletion that lands while the thread is open never // reaches it — a new pill (and its burst) only showed up after leaving and @@ -390,6 +398,7 @@ class ThreadDetailPage extends HookConsumerWidget { ? const SizedBox.shrink() : ChannelTypingIndicator(entries: threadTyping), ), + const MessageReadAloudBar(), if (isMember && !isArchived) ComposeBar( channelId: channelId, @@ -660,6 +669,7 @@ class _ThreadMessage extends ConsumerWidget { message: message, channelId: channelId, canManageMessage: canManageMessage, + messageAuthor: displayName, allMessages: allMessages, currentPubkey: currentPubkey, isMember: isMember, diff --git a/mobile/test/features/channels/message_read_aloud_test.dart b/mobile/test/features/channels/message_read_aloud_test.dart new file mode 100644 index 00000000000..024defa4a54 --- /dev/null +++ b/mobile/test/features/channels/message_read_aloud_test.dart @@ -0,0 +1,31 @@ +import 'package:buzz/features/channels/message_read_aloud.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('preserves readable words while flattening Markdown', () { + expect( + messageTextForSpeech( + '## **Update**\n\n' + 'Read [the guide](https://example.com), then run `buzz check`.\n\n' + '```sh\nbuzz status\n```', + ), + 'Update\n\nRead the guide, then run buzz check.\n\nbuzz status', + ); + }); + + test('keeps image alt text and list content', () { + expect( + messageTextForSpeech('- First\n- ![Diagram](https://example.com/a.png)'), + 'First\nDiagram', + ); + }); + + test('preserves code identifiers exactly', () { + expect( + messageTextForSpeech( + 'Use `message_read_aloud.dart` and `__init__` next.', + ), + 'Use message_read_aloud.dart and __init__ next.', + ); + }); +}