diff --git a/desktop/playwright.config.ts b/desktop/playwright.config.ts index 7ce7f48389a..ec4c6593475 100644 --- a/desktop/playwright.config.ts +++ b/desktop/playwright.config.ts @@ -11,6 +11,9 @@ export default defineConfig({ ], use: { baseURL: "http://127.0.0.1:4173", + launchOptions: process.env.PLAYWRIGHT_CHROME_EXECUTABLE + ? { executablePath: process.env.PLAYWRIGHT_CHROME_EXECUTABLE } + : undefined, screenshot: "only-on-failure", trace: "on-first-retry", video: "retain-on-failure", @@ -35,6 +38,7 @@ export default defineConfig({ "**/invites-settings-screenshots.spec.ts", "**/messaging.spec.ts", "**/message-feedback-snapshots.spec.ts", + "**/message-read-aloud-screenshots.spec.ts", "**/custom-emoji.spec.ts", "**/profile-custom-emoji-status.spec.ts", "**/custom-emoji-ui.spec.ts", diff --git a/desktop/src-tauri/src/huddle/message_read_aloud.rs b/desktop/src-tauri/src/huddle/message_read_aloud.rs new file mode 100644 index 00000000000..6596d4b5f07 --- /dev/null +++ b/desktop/src-tauri/src/huddle/message_read_aloud.rs @@ -0,0 +1,134 @@ +//! On-demand read-aloud of a single chat message through the local Pocket +//! TTS engine — the same pipeline (and selected Settings voice) the huddle +//! uses for agent speech. +//! +//! Mirrors the `preview_pocket_voice` structure: a short-lived pipeline per +//! request, completion detected by polling the pipeline's active flag. Only +//! one message plays at a time; starting a new one cancels the previous. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use tauri::{AppHandle, Emitter, State}; + +use crate::app_state::AppState; + +use super::models; +use super::tts::TtsPipeline; +use super::tts_settings::{current_settings, pocket_voice_reference}; + +/// Emitted (payload: the caller's `session_id`) when audio playback actually +/// starts, so the UI can move from "preparing" to "playing". +const STARTED_EVENT: &str = "message-read-aloud-started"; + +/// How long synthesis may run before the first audible sample. Matches the +/// voice-preview timeout. +const PLAYBACK_START_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + +/// Poll interval for the playback progress flags. +const POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(25); + +/// Cancel flag for the in-flight message read-aloud playback, if any. +/// Starting a new read-aloud cancels the previous one; stop raises it. +static ACTIVE_CANCEL: Mutex>> = Mutex::new(None); + +/// Swap the shared cancel slot, returning the previous occupant. +fn swap_active_cancel(next: Option>) -> Option> { + let mut slot = ACTIVE_CANCEL + .lock() + .unwrap_or_else(|error| error.into_inner()); + std::mem::replace(&mut *slot, next) +} + +/// Speak `text` through the local Pocket TTS engine with the user's selected +/// Settings voice. +/// +/// Resolves `Ok(true)` once playback finishes, `Ok(false)` when cancelled by +/// [`stop_message_read_aloud`] or a newer read-aloud request. Errors when the +/// voice files are not ready, no Pocket voice is available, or audio never +/// starts. +#[tauri::command] +pub async fn speak_message_read_aloud( + session_id: String, + text: String, + app: AppHandle, + state: State<'_, AppState>, +) -> Result { + if text.trim().is_empty() { + return Err("Nothing to read aloud".to_string()); + } + if !models::is_tts_ready() { + return Err("Voice files are still downloading. Try again shortly.".to_string()); + } + let model_dir = models::tts_model_dir().ok_or("Pocket voice files are unavailable")?; + let settings = current_settings(&state)?; + let voice_name = pocket_voice_reference(&app, &settings.voice_preferences)?; + let output_device = state + .huddle_audio + .output_device + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone(); + + // Only one message plays at a time — cancel whichever was still running. + let cancel = Arc::new(AtomicBool::new(false)); + if let Some(previous) = swap_active_cancel(Some(Arc::clone(&cancel))) { + previous.store(true, Ordering::Release); + } + let cancel_worker = Arc::clone(&cancel); + + let result = tokio::task::spawn_blocking(move || { + let active = Arc::new(AtomicBool::new(false)); + let pipeline = TtsPipeline::new_with_voice( + model_dir, + Arc::clone(&active), + Arc::clone(&cancel_worker), + &voice_name, + output_device, + )?; + pipeline.speak(text)?; + let started = std::time::Instant::now(); + let mut heard_audio = false; + loop { + if cancel_worker.load(Ordering::Acquire) { + return Ok(false); + } + let is_active = active.load(Ordering::Acquire); + if is_active && !heard_audio { + heard_audio = true; + let _ = app.emit(STARTED_EVENT, &session_id); + } + if heard_audio && !is_active { + return Ok(true); + } + if !heard_audio && started.elapsed() > PLAYBACK_START_TIMEOUT { + return Err( + "Playback did not start. Check your audio output and try again.".to_string(), + ); + } + std::thread::sleep(POLL_INTERVAL); + } + }) + .await + .map_err(|error| format!("Read-aloud task failed: {error}"))?; + + // Release the slot if it still belongs to this request so a stale Arc + // doesn't linger after playback ends. + { + let mut slot = ACTIVE_CANCEL + .lock() + .unwrap_or_else(|error| error.into_inner()); + if slot.as_ref().is_some_and(|held| Arc::ptr_eq(held, &cancel)) { + *slot = None; + } + } + result +} + +/// Stop the in-flight message read-aloud, if any. +#[tauri::command] +pub fn stop_message_read_aloud() { + if let Some(cancel) = swap_active_cancel(None) { + cancel.store(true, Ordering::Release); + } +} diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index 03264f80f4c..74520532757 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -27,6 +27,7 @@ mod agent_tts_routing; pub mod agents; pub mod audio_output; pub mod jitter; +pub mod message_read_aloud; pub mod models; pub mod pipeline; pub mod playout; diff --git a/desktop/src-tauri/src/huddle/tts_settings.rs b/desktop/src-tauri/src/huddle/tts_settings.rs index 1b378af8237..1068027aa82 100644 --- a/desktop/src-tauri/src/huddle/tts_settings.rs +++ b/desktop/src-tauri/src/huddle/tts_settings.rs @@ -463,7 +463,7 @@ async fn apply_tts_settings( Ok(voice_change_wait) } -fn current_settings(state: &AppState) -> Result { +pub(crate) fn current_settings(state: &AppState) -> Result { state .huddle_audio .tts diff --git a/desktop/src-tauri/src/initial_window.rs b/desktop/src-tauri/src/initial_window.rs new file mode 100644 index 00000000000..6b9f294e733 --- /dev/null +++ b/desktop/src-tauri/src/initial_window.rs @@ -0,0 +1,72 @@ +//! First-reveal choreography for the main window: keep it hidden (with an +//! opaque backing on macOS) until the initial render is ready and the +//! window-state plugin has settled restored geometry, then show and focus. + +#[cfg(target_os = "macos")] +pub(crate) const INITIAL_RENDER_READY_EVENT: &str = "initial-render-ready"; + +pub(crate) fn reveal_initial_window(window: &tauri::Window) { + if let Err(error) = window.show() { + eprintln!("buzz-desktop: failed to reveal main window: {error}"); + return; + } + if let Err(error) = window.set_focus() { + eprintln!("buzz-desktop: failed to focus main window: {error}"); + } +} + +#[cfg(target_os = "macos")] +pub(crate) fn set_initial_window_backing(window: &tauri::Window) { + // The window remains transparent at runtime for vibrancy. Use an opaque + // native backing only across the first visible frames so the previous app + // cannot show through before WebKit has submitted its first surface. + if let Err(error) = window.set_background_color(Some(tauri::window::Color(17, 21, 24, 255))) { + eprintln!("buzz-desktop: failed to set initial window backing: {error}"); + } +} + +#[cfg(target_os = "macos")] +pub(crate) async fn clear_initial_window_backing(window: &tauri::Window) { + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + if let Err(error) = window.set_background_color(None) { + eprintln!("buzz-desktop: failed to clear initial window backing: {error}"); + } +} + +#[cfg(target_os = "macos")] +pub(crate) async fn wait_for_stable_initial_window_geometry( + window: &tauri::Window, +) { + const MAX_POLLS: usize = 120; + const REQUIRED_STABLE_POLLS: usize = 4; + + let mut previous_bounds = None; + let mut stable_polls = 0; + + for _ in 0..MAX_POLLS { + // Accept whatever geometry the window-state plugin restores — maximized + // or a normal saved size. macOS applies the restore asynchronously, so + // we only need consecutive identical outer bounds to know it settled. + // Gating on `is_maximized()` here would leave `bounds` permanently + // `None` for restored non-maximized windows and stall the reveal until + // the poll timeout. + let bounds = match (window.outer_position(), window.outer_size()) { + (Ok(position), Ok(size)) => Some((position.x, position.y, size.width, size.height)), + _ => None, + }; + + if bounds.is_some() && bounds == previous_bounds { + stable_polls += 1; + if stable_polls >= REQUIRED_STABLE_POLLS { + return; + } + } else { + stable_polls = 0; + } + previous_bounds = bounds; + + tokio::time::sleep(std::time::Duration::from_millis(16)).await; + } + + eprintln!("buzz-desktop: initial window geometry did not settle before reveal timeout"); +} diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index c4b733e3e07..5afda46dbc2 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -9,6 +9,7 @@ mod event_sync; mod events; mod huddle; mod identity_storage; +mod initial_window; mod key_backup; mod linux_media; mod managed_agents; @@ -54,6 +55,7 @@ use huddle::{ join_huddle, leave_huddle, push_audio_pcm, set_huddle_transcription_enabled, set_tts_enabled, set_voice_input_mode, speak_agent_message, start_huddle, start_stt_pipeline, }; +use initial_window::*; use managed_agents::{ backfill_persona_snapshots, ensure_nest, list_managed_agent_runtimes, put_managed_agent_runtime_lifecycle, reconcile_managed_agent_runtimes, @@ -73,73 +75,6 @@ use tauri_plugin_window_state::StateFlags; #[cfg(target_os = "macos")] use tray_menu::show_main_window; -#[cfg(target_os = "macos")] -const INITIAL_RENDER_READY_EVENT: &str = "initial-render-ready"; - -fn reveal_initial_window(window: &tauri::Window) { - if let Err(error) = window.show() { - eprintln!("buzz-desktop: failed to reveal main window: {error}"); - return; - } - if let Err(error) = window.set_focus() { - eprintln!("buzz-desktop: failed to focus main window: {error}"); - } -} - -#[cfg(target_os = "macos")] -fn set_initial_window_backing(window: &tauri::Window) { - // The window remains transparent at runtime for vibrancy. Use an opaque - // native backing only across the first visible frames so the previous app - // cannot show through before WebKit has submitted its first surface. - if let Err(error) = window.set_background_color(Some(tauri::window::Color(17, 21, 24, 255))) { - eprintln!("buzz-desktop: failed to set initial window backing: {error}"); - } -} - -#[cfg(target_os = "macos")] -async fn clear_initial_window_backing(window: &tauri::Window) { - tokio::time::sleep(std::time::Duration::from_millis(250)).await; - if let Err(error) = window.set_background_color(None) { - eprintln!("buzz-desktop: failed to clear initial window backing: {error}"); - } -} - -#[cfg(target_os = "macos")] -async fn wait_for_stable_initial_window_geometry(window: &tauri::Window) { - const MAX_POLLS: usize = 120; - const REQUIRED_STABLE_POLLS: usize = 4; - - let mut previous_bounds = None; - let mut stable_polls = 0; - - for _ in 0..MAX_POLLS { - // Accept whatever geometry the window-state plugin restores — maximized - // or a normal saved size. macOS applies the restore asynchronously, so - // we only need consecutive identical outer bounds to know it settled. - // Gating on `is_maximized()` here would leave `bounds` permanently - // `None` for restored non-maximized windows and stall the reveal until - // the poll timeout. - let bounds = match (window.outer_position(), window.outer_size()) { - (Ok(position), Ok(size)) => Some((position.x, position.y, size.width, size.height)), - _ => None, - }; - - if bounds.is_some() && bounds == previous_bounds { - stable_polls += 1; - if stable_polls >= REQUIRED_STABLE_POLLS { - return; - } - } else { - stable_polls = 0; - } - previous_bounds = bounds; - - tokio::time::sleep(std::time::Duration::from_millis(16)).await; - } - - eprintln!("buzz-desktop: initial window geometry did not settle before reveal timeout"); -} - #[cfg_attr(mobile, tauri::mobile_entry_point)] pub fn run() { // mesh-llm's async chains (model download, node start/join) overflow @@ -898,6 +833,8 @@ pub fn run() { huddle::tts_settings::preview_pocket_voice, huddle::tts_settings::import_pocket_voice, huddle::tts_settings::delete_pocket_voice, + huddle::message_read_aloud::speak_message_read_aloud, + huddle::message_read_aloud::stop_message_read_aloud, speak_agent_message, add_agent_to_huddle, check_pipeline_hotstart, diff --git a/desktop/src/features/channels/ui/ChannelComposerActivityAccessory.tsx b/desktop/src/features/channels/ui/ChannelComposerActivityAccessory.tsx index f99888f0112..529d8c2cf94 100644 --- a/desktop/src/features/channels/ui/ChannelComposerActivityAccessory.tsx +++ b/desktop/src/features/channels/ui/ChannelComposerActivityAccessory.tsx @@ -4,6 +4,7 @@ import { CardMintComposerChip } from "@/features/agents/ui/CardMintComposerChip" import { useCardMintJobs } from "@/features/agents/cardMintStore"; import { BotActivityComposerAction } from "@/features/channels/ui/BotActivityBar"; import { ComposerActivityAccessory } from "@/features/messages/ui/ComposerActivityAccessory"; +import { MessageReadAloudBar } from "@/features/messages/ui/MessageReadAloudBar"; import { TypingIndicatorRow } from "@/features/messages/ui/TypingIndicatorRow"; type ChannelComposerActivityAccessoryProps = { @@ -35,36 +36,41 @@ export function ChannelComposerActivityAccessory({ }: ChannelComposerActivityAccessoryProps) { const cardMintJobs = useCardMintJobs(); return ( - -
- {cardMintJobs.length > 0 ? : null} - {workingBotPubkeys.length > 0 ? ( -
- +
+ +
+ +
+ {cardMintJobs.length > 0 ? : null} + {workingBotPubkeys.length > 0 ? ( +
+ +
+ ) : null} + {typingPubkeys.length > 0 ? ( + -
- ) : null} - {typingPubkeys.length > 0 ? ( - - ) : null} -
- + ) : null} +
+ + ); } diff --git a/desktop/src/features/communities/useCommunityInit.ts b/desktop/src/features/communities/useCommunityInit.ts index afa69f913f6..6564c69dda7 100644 --- a/desktop/src/features/communities/useCommunityInit.ts +++ b/desktop/src/features/communities/useCommunityInit.ts @@ -31,6 +31,7 @@ import { resetAvatarProfileSync } from "@/features/profile/avatarProfileSync"; import { resetSidebarRelayConnectionCardState } from "@/features/sidebar/ui/useSidebarRelayConnectionCard"; import { clearMarkdownNodeCache } from "@/shared/ui/markdown/nodeCache"; import { resetVideoPlayerState } from "@/shared/ui/videoPlayerState"; +import { stopMessageReadAloud } from "@/features/messages/lib/messageReadAloud"; import { initFirstCommunity, @@ -65,6 +66,7 @@ function resetCommunityState({ } resetSidebarRelayConnectionCardState(); resetMediaCaches(); + stopMessageReadAloud(); resetVideoPlayerState(); resetRenderScopedReactionHydration(); clearSearchHitEventCache(); diff --git a/desktop/src/features/home/ui/InboxDetailPane.tsx b/desktop/src/features/home/ui/InboxDetailPane.tsx index 9b6ff57e172..c22aa4a6f63 100644 --- a/desktop/src/features/home/ui/InboxDetailPane.tsx +++ b/desktop/src/features/home/ui/InboxDetailPane.tsx @@ -34,6 +34,7 @@ import { orderMentionPubkeysByText } from "@/features/messages/lib/orderMentionP import { getThreadReference } from "@/features/messages/lib/threading"; import { normalizePubkey } from "@/shared/lib/pubkey"; import { MessageComposer } from "@/features/messages/ui/MessageComposer"; +import { MessageReadAloudBar } from "@/features/messages/ui/MessageReadAloudBar"; import { useAnchoredScroll } from "@/features/messages/ui/useAnchoredScroll"; import { useComposerHeightPadding } from "@/features/messages/ui/useComposerHeightPadding"; import { UpdateIndicator } from "@/features/settings/UpdateIndicator"; @@ -633,6 +634,7 @@ function InboxMessageDetailPane({ }} />
+ { + assert.equal( + messageTextForSpeech( + "## **Update**\n\nRead [the guide](https://example.com), then run `buzz check`.\n\n```sh\nbuzz status\n```", + ), + "Update\n\nRead the guide, then run buzz check.\n\nbuzz status", + ); +}); + +test("messageTextForSpeech keeps image alt text and list content", () => { + assert.equal( + messageTextForSpeech("- First\n- ![Diagram](https://example.com/a.png)"), + "First\nDiagram", + ); +}); + +test("messageTextForSpeech preserves code identifiers exactly", () => { + assert.equal( + messageTextForSpeech("Use `message_read_aloud.dart` and `__init__` next."), + "Use message_read_aloud.dart and __init__ next.", + ); +}); diff --git a/desktop/src/features/messages/lib/messageReadAloud.ts b/desktop/src/features/messages/lib/messageReadAloud.ts new file mode 100644 index 00000000000..8cddc409518 --- /dev/null +++ b/desktop/src/features/messages/lib/messageReadAloud.ts @@ -0,0 +1,375 @@ +import * as React from "react"; + +export type MessageReadAloudStatus = + | "idle" + | "preparing" + | "playing" + | "paused" + | "finished" + | "error"; + +/** + * Which speech backend is driving the current playback. + * + * "native" is the Pocket TTS pipeline in the Tauri backend — the same engine + * and selected Settings voice the huddle uses. "web" is the browser's + * `speechSynthesis`, kept as the fallback for non-Tauri contexts (web dev + * server, e2e mock bridge). Native supports play/stop only; pause/resume is + * web-only. + */ +export type MessageReadAloudEngine = "native" | "web"; + +export type MessageReadAloudState = { + author: string; + engine: MessageReadAloudEngine; + error: string | null; + messageId: string | null; + status: MessageReadAloudStatus; + text: string; +}; + +const IDLE_STATE: MessageReadAloudState = { + author: "", + engine: "web", + error: null, + messageId: null, + status: "idle", + text: "", +}; + +let state = IDLE_STATE; +let activeUtterance: SpeechSynthesisUtterance | null = null; +let nativeSessionCounter = 0; +let currentNativeSessionId: string | null = null; +let nativeStartedListenerReady: Promise | null = null; +const listeners = new Set<() => void>(); + +function publish(next: MessageReadAloudState) { + state = next; + for (const listener of listeners) listener(); +} + +function speechEngine() { + if (typeof window === "undefined") return null; + return window.speechSynthesis ?? null; +} + +function nativeEngineAvailable() { + if (typeof window === "undefined") return false; + const globals = window as unknown as Record; + // The e2e mock bridge defines __TAURI_INTERNALS__ but not the read-aloud + // commands — specs drive the web engine through a speechSynthesis mock. + return ( + "__TAURI_INTERNALS__" in globals && + !("__BUZZ_E2E_INVOKE_MOCK_COMMAND__" in globals) + ); +} + +/** Flatten common Markdown markers while preserving every readable word. */ +export function messageTextForSpeech(markdown: string) { + const codeSpans: string[] = []; + const preserveCode = (text: string) => { + const index = codeSpans.push(text) - 1; + return `${index}`; + }; + + return markdown + .replace(/```(?:[^\n`]*)\n?([\s\S]*?)```/g, (_match, code: string) => + preserveCode(code), + ) + .replace(/`([^`]+)`/g, (_match, code: string) => preserveCode(code)) + .replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1") + .replace(/\[([^\]]+)\]\([^)]*\)/g, "$1") + .replace(/^\s{0,3}(?:#{1,6}|>|[-+*]|\d+[.)])\s+/gm, "") + .replace(/\*\*([^*\n]+)\*\*/g, "$1") + .replace(/(? codeSpans[Number(index)] ?? "", + ) + .replace(/\n{3,}/g, "\n\n") + .trim(); +} + +function readAloudErrorMessage(error: unknown) { + if (typeof error === "string" && error.trim().length > 0) return error; + return "Couldn't play audio. Try again."; +} + +function ensureNativeStartedListener() { + if (!nativeStartedListenerReady) { + nativeStartedListenerReady = import("@tauri-apps/api/event").then( + ({ listen }) => + listen("message-read-aloud-started", (event) => { + if (event.payload !== currentNativeSessionId) return; + if (state.status !== "preparing") return; + publish({ ...state, status: "playing" }); + }).then(() => undefined), + ); + } + return nativeStartedListenerReady; +} + +async function beginNativeSpeech( + messageId: string, + author: string, + text: string, +) { + const sessionId = String(++nativeSessionCounter); + currentNativeSessionId = sessionId; + activeUtterance = null; + speechEngine()?.cancel(); + publish({ + author, + engine: "native", + error: null, + messageId, + status: "preparing", + text, + }); + try { + await ensureNativeStartedListener(); + const { invoke } = await import("@tauri-apps/api/core"); + const finished = await invoke("speak_message_read_aloud", { + sessionId, + text, + }); + if (currentNativeSessionId !== sessionId) return; + currentNativeSessionId = null; + if (finished) { + publish({ + author, + engine: "native", + error: null, + messageId, + status: "finished", + text, + }); + } else if (state.messageId === messageId && state.status !== "idle") { + publish(IDLE_STATE); + } + } catch (error) { + if (currentNativeSessionId !== sessionId) return; + currentNativeSessionId = null; + console.error("message read aloud failed", error); + publish({ + author, + engine: "native", + error: readAloudErrorMessage(error), + messageId, + status: "error", + text, + }); + } +} + +function beginWebSpeech(messageId: string, author: string, text: string) { + const engine = speechEngine(); + if (!engine || typeof SpeechSynthesisUtterance === "undefined") { + publish({ + author, + engine: "web", + error: "Couldn't play audio. Try again.", + messageId, + status: "error", + text, + }); + return; + } + + engine.cancel(); + const utterance = new SpeechSynthesisUtterance(text); + activeUtterance = utterance; + publish({ + author, + engine: "web", + error: null, + messageId, + status: "preparing", + text, + }); + + utterance.onstart = () => { + if (activeUtterance !== utterance) return; + publish({ + author, + engine: "web", + error: null, + messageId, + status: "playing", + text, + }); + }; + utterance.onpause = () => { + if (activeUtterance !== utterance) return; + publish({ + author, + engine: "web", + error: null, + messageId, + status: "paused", + text, + }); + }; + utterance.onresume = () => { + if (activeUtterance !== utterance) return; + publish({ + author, + engine: "web", + error: null, + messageId, + status: "playing", + text, + }); + }; + utterance.onend = () => { + if (activeUtterance !== utterance) return; + activeUtterance = null; + publish({ + author, + engine: "web", + error: null, + messageId, + status: "finished", + text, + }); + }; + utterance.onerror = (event) => { + if (activeUtterance !== utterance || event.error === "canceled") return; + activeUtterance = null; + console.error("message read aloud failed", event.error); + publish({ + author, + engine: "web", + error: "Couldn't play audio. Try again.", + messageId, + status: "error", + text, + }); + }; + + engine.speak(utterance); +} + +function speakText(messageId: string, author: string, text: string) { + if (!text) { + publish({ + author, + engine: "web", + error: "Couldn't play audio. Try again.", + messageId, + status: "error", + text, + }); + return; + } + if (nativeEngineAvailable()) { + void beginNativeSpeech(messageId, author, text); + return; + } + beginWebSpeech(messageId, author, text); +} + +function beginSpeech(messageId: string, author: string, markdown: string) { + speakText(messageId, author, messageTextForSpeech(markdown)); +} + +export function listenToMessage( + messageId: string, + author: string, + markdown: string, +) { + if ( + state.messageId === messageId && + state.status === "paused" && + state.engine === "web" + ) { + speechEngine()?.resume(); + return; + } + if (state.messageId === messageId && state.status === "playing") { + if (state.engine === "native") { + stopMessageReadAloud(); + } else { + speechEngine()?.pause(); + } + return; + } + beginSpeech(messageId, author, markdown); +} + +export function pauseMessageReadAloud() { + if (state.status === "playing" && state.engine === "web") { + speechEngine()?.pause(); + } +} + +export function resumeMessageReadAloud() { + if (state.status === "paused" && state.engine === "web") { + speechEngine()?.resume(); + } +} + +export function retryMessageReadAloud() { + // `state.text` is already flattened — do not re-run messageTextForSpeech, + // which would treat code content (e.g. `__init__`) as Markdown markers. + if (state.messageId) speakText(state.messageId, state.author, state.text); +} + +function stopNativeSpeech() { + currentNativeSessionId = null; + void import("@tauri-apps/api/core") + .then(({ invoke }) => invoke("stop_message_read_aloud")) + .catch((error) => { + console.error("message read aloud stop failed", error); + }); +} + +export function stopMessageReadAloud() { + if ( + state.engine === "native" && + (state.status === "preparing" || state.status === "playing") + ) { + stopNativeSpeech(); + } + activeUtterance = null; + speechEngine()?.cancel(); + publish(IDLE_STATE); +} + +if (typeof window !== "undefined") { + window.addEventListener("hashchange", stopMessageReadAloud); + window.addEventListener("pagehide", stopMessageReadAloud); +} + +export function getMessageReadAloudState() { + return state; +} + +export function subscribeMessageReadAloud(listener: () => void) { + listeners.add(listener); + return () => listeners.delete(listener); +} + +export function useMessageReadAloud() { + return React.useSyncExternalStore( + subscribeMessageReadAloud, + getMessageReadAloudState, + getMessageReadAloudState, + ); +} + +export function useMessageReadAloudForMessage(messageId: string) { + const getSnapshot = React.useCallback( + () => (state.messageId === messageId ? state : IDLE_STATE), + [messageId], + ); + return React.useSyncExternalStore( + subscribeMessageReadAloud, + getSnapshot, + getSnapshot, + ); +} diff --git a/desktop/src/features/messages/ui/MessageActionBar.tsx b/desktop/src/features/messages/ui/MessageActionBar.tsx index 967e50f5d2f..d927357aa47 100644 --- a/desktop/src/features/messages/ui/MessageActionBar.tsx +++ b/desktop/src/features/messages/ui/MessageActionBar.tsx @@ -7,11 +7,17 @@ import { EllipsisVertical, Flag, Link2, + LoaderCircle, MailCheck, MailOpen, + Pause, Pencil, + Play, + RotateCcw, SmilePlus, + Square, Trash2, + Volume2, } from "lucide-react"; import * as React from "react"; @@ -35,6 +41,10 @@ import { copyTextToClipboard } from "@/shared/lib/clipboard"; import { emojiDisplayName } from "@/shared/lib/emojiName"; import { rewriteRelayUrl } from "@/shared/lib/mediaUrl"; import { KIND_HUDDLE_STARTED } from "@/shared/constants/kinds"; +import { + listenToMessage, + useMessageReadAloudForMessage, +} from "@/features/messages/lib/messageReadAloud"; import { Button } from "@/shared/ui/button"; import { DeleteMessageConfirmDialog } from "./DeleteMessageConfirmDialog"; import { @@ -60,6 +70,9 @@ function MoreActionsMenu({ onMarkUnread, onMarkRead, onOpenChange, + onListen, + listenDisabled, + listenLabel, onRemindLater, onUnfollowThread, open, @@ -76,6 +89,9 @@ function MoreActionsMenu({ onMarkUnread?: (message: TimelineMessage) => void; onMarkRead?: (message: TimelineMessage) => void; onOpenChange: (open: boolean) => void; + onListen?: () => void; + listenDisabled?: boolean; + listenLabel?: string; onRemindLater?: (message: TimelineMessage) => void; onUnfollowThread?: (message: TimelineMessage) => void; open: boolean; @@ -149,6 +165,13 @@ function MoreActionsMenu({ ) : null} + {onListen ? ( + + + {listenLabel ?? "Listen to message"} + + ) : null} + {onMarkRead || onMarkUnread ? ( { + if (!isThisMessage || playback.status !== "preparing") { + setShowPreparing(false); + return; + } + const timeout = window.setTimeout(() => setShowPreparing(true), 200); + return () => window.clearTimeout(timeout); + }, [isThisMessage, playback.status]); + const listenLabel = isThisMessage + ? playback.status === "preparing" + ? "Preparing audio…" + : playback.status === "playing" + ? playback.engine === "native" + ? "Stop reading" + : "Pause reading" + : playback.status === "paused" + ? "Resume reading" + : playback.status === "finished" + ? "Replay message" + : "Listen to message" + : "Listen to message"; + const canListen = + !message.pending && + message.kind !== KIND_HUDDLE_STARTED && + message.body.trim().length > 0; + const handleListen = React.useCallback(() => { + listenToMessage(message.id, message.author, message.body); + }, [message.author, message.body, message.id]); const hasMoreMenuActions = Boolean(onEdit) || @@ -398,6 +452,7 @@ export const MessageActionBar = React.memo(function MessageActionBar({ Boolean(onFollowThread) || Boolean(onUnfollowThread) || Boolean(onRemindLater) || + canListen || !message.pending; const wouldAddReaction = React.useCallback( @@ -534,6 +589,51 @@ export const MessageActionBar = React.memo(function MessageActionBar({ ) : null} + {canListen ? ( + + + + + {listenLabel} + + ) : null} + {hasMoreMenuActions ? ( diff --git a/desktop/src/features/messages/ui/MessageReadAloudBar.tsx b/desktop/src/features/messages/ui/MessageReadAloudBar.tsx new file mode 100644 index 00000000000..8bfada33a48 --- /dev/null +++ b/desktop/src/features/messages/ui/MessageReadAloudBar.tsx @@ -0,0 +1,107 @@ +import * as React from "react"; +import { LoaderCircle, Pause, Play, RotateCcw, Volume2, X } from "lucide-react"; + +import { + pauseMessageReadAloud, + resumeMessageReadAloud, + retryMessageReadAloud, + stopMessageReadAloud, + useMessageReadAloud, +} from "@/features/messages/lib/messageReadAloud"; +import { Button } from "@/shared/ui/button"; + +export function MessageReadAloudBar() { + const playback = useMessageReadAloud(); + const [showPreparing, setShowPreparing] = React.useState(false); + React.useEffect(() => { + if (playback.status !== "preparing") { + setShowPreparing(false); + return; + } + const timeout = window.setTimeout(() => setShowPreparing(true), 200); + return () => window.clearTimeout(timeout); + }, [playback.status]); + if ( + playback.status === "idle" || + playback.status === "finished" || + (playback.status === "preparing" && !showPreparing) || + !playback.messageId + ) { + return null; + } + + const isPaused = playback.status === "paused"; + const isError = playback.status === "error"; + const isPreparing = playback.status === "preparing"; + // The native Pocket engine supports play/stop only — the X button stops. + const canPause = playback.engine === "web"; + + return ( +
+ {isPreparing ? ( + + ) : ( + + )} +

+ {isError ? ( + (playback.error ?? "Couldn't play audio. Try again.") + ) : isPreparing ? ( + "Preparing audio…" + ) : ( + <> + Reading message from{" "} + + {playback.author} + + + )} +

+ {isError ? ( + + ) : isPreparing || !canPause ? null : ( + + )} + +
+ ); +} diff --git a/desktop/tests/e2e/message-read-aloud-screenshots.spec.ts b/desktop/tests/e2e/message-read-aloud-screenshots.spec.ts new file mode 100644 index 00000000000..a4b65b2f3a1 --- /dev/null +++ b/desktop/tests/e2e/message-read-aloud-screenshots.spec.ts @@ -0,0 +1,114 @@ +import { expect, test } from "@playwright/test"; + +import { installMockBridge, TEST_IDENTITIES } from "../helpers/bridge"; + +test.beforeEach(async ({ page }) => { + await page.addInitScript(() => { + class MockSpeechSynthesisUtterance extends EventTarget { + error: string | null = null; + onend: (() => void) | null = null; + onerror: ((event: { error: string }) => void) | null = null; + onpause: (() => void) | null = null; + onresume: (() => void) | null = null; + onstart: (() => void) | null = null; + + constructor(public text: string) { + super(); + } + } + + let active: MockSpeechSynthesisUtterance | null = null; + const speechSynthesis = { + cancel() { + active = null; + }, + getVoices() { + return []; + }, + pause() { + active?.onpause?.(); + }, + paused: false, + pending: false, + resume() { + active?.onresume?.(); + }, + speak(utterance: MockSpeechSynthesisUtterance) { + active = utterance; + window.setTimeout(() => { + if (active === utterance) utterance.onstart?.(); + }, 250); + }, + speaking: false, + }; + + Object.defineProperty(window, "SpeechSynthesisUtterance", { + configurable: true, + value: MockSpeechSynthesisUtterance, + }); + Object.defineProperty(window, "speechSynthesis", { + configurable: true, + value: speechSynthesis, + }); + }); + await installMockBridge(page, { + searchProfiles: [ + { + pubkey: TEST_IDENTITIES.alice.pubkey, + displayName: "Honey", + ownerPubkey: TEST_IDENTITIES.tyler.pubkey, + isAgent: true, + }, + ], + }); +}); + +test("messages expose private read-aloud controls", async ({ + page, +}, testInfo) => { + await page.setViewportSize({ width: 1280, height: 820 }); + await page.goto("/"); + await page.getByTestId("channel-general").click(); + + const agentMessage = page + .getByTestId("message-row") + .filter({ hasText: "Hey team — checking in." }); + const humanMessage = page + .getByTestId("message-row") + .filter({ hasText: "Welcome to #general" }); + + await agentMessage.hover(); + const listenButton = agentMessage.getByRole("button", { + name: "Listen to message", + }); + await expect(listenButton).toBeVisible(); + await humanMessage.hover(); + await expect( + humanMessage.getByRole("button", { name: "Listen to message" }), + ).toBeVisible(); + await agentMessage.hover(); + await page.screenshot({ + path: testInfo.outputPath("agent-listen-hover.png"), + }); + + await listenButton.click(); + const readAloudBar = page.getByTestId("message-read-aloud-bar"); + await expect(readAloudBar).toContainText("Reading message from Honey"); + await expect( + readAloudBar.getByRole("button", { name: "Pause reading" }), + ).toBeVisible(); + await page.screenshot({ + path: testInfo.outputPath("agent-now-reading.png"), + }); + + await readAloudBar.getByRole("button", { name: "Pause reading" }).click(); + await expect( + readAloudBar.getByRole("button", { name: "Resume reading" }), + ).toBeVisible(); + await readAloudBar.getByRole("button", { name: "Resume reading" }).click(); + await expect( + readAloudBar.getByRole("button", { name: "Pause reading" }), + ).toBeVisible(); + await readAloudBar.getByRole("button", { name: "Stop reading" }).click(); + await expect(readAloudBar).toHaveCount(0); +}); diff --git a/mobile/ios/Runner/AppDelegate.swift b/mobile/ios/Runner/AppDelegate.swift index f6fe3aaf489..5a1de0b71bf 100644 --- a/mobile/ios/Runner/AppDelegate.swift +++ b/mobile/ios/Runner/AppDelegate.swift @@ -4,11 +4,17 @@ import UIKit import UserNotifications @main -@objc class AppDelegate: FlutterAppDelegate, FlutterImplicitEngineDelegate { +@objc class AppDelegate: FlutterAppDelegate, FlutterImplicitEngineDelegate, + AVSpeechSynthesizerDelegate +{ private var mediaUploadChannel: FlutterMethodChannel? private var qrScannerChannel: FlutterMethodChannel? private var inlinePhotoPickerSupportChannel: FlutterMethodChannel? private var nativeAttachmentPopoverCoordinator: NativeAttachmentPopoverCoordinator? + private var messageReadAloudChannel: FlutterMethodChannel? + private let messageSpeechSynthesizer = AVSpeechSynthesizer() + private var speakingMessageId: String? + private var speakingUtterance: AVSpeechUtterance? override func application( _ application: UIApplication, @@ -50,6 +56,14 @@ import UserNotifications result(false) } } + messageReadAloudChannel = FlutterMethodChannel( + name: "buzz/message_read_aloud", + binaryMessenger: messenger + ) + messageSpeechSynthesizer.delegate = self + messageReadAloudChannel?.setMethodCallHandler { [weak self] call, result in + self?.handleMessageReadAloudMethodCall(call, result: result) + } if let inlinePhotoPickerRegistrar = engineBridge.pluginRegistry.registrar( forPlugin: "BuzzInlinePhotoPicker" @@ -72,6 +86,98 @@ import UserNotifications ) } + private func handleMessageReadAloudMethodCall( + _ call: FlutterMethodCall, + result: @escaping FlutterResult + ) { + switch call.method { + case "speak": + guard + let arguments = call.arguments as? [String: Any], + let messageId = arguments["messageId"] as? String, + let author = arguments["author"] as? String, + let text = arguments["text"] as? String, + !text.isEmpty + else { + result( + FlutterError( + code: "invalid_arguments", + message: "Expected a message id, author, and non-empty text.", + details: nil + ) + ) + return + } + + if messageSpeechSynthesizer.isSpeaking || messageSpeechSynthesizer.isPaused { + speakingMessageId = nil + speakingUtterance = nil + messageSpeechSynthesizer.stopSpeaking(at: .immediate) + } + do { + let audioSession = AVAudioSession.sharedInstance() + try audioSession.setCategory( + .playback, + mode: .spokenAudio, + options: [.duckOthers] + ) + try audioSession.setActive(true) + } catch { + result( + FlutterError( + code: "audio_session_failed", + message: "Unable to start message audio.", + details: error.localizedDescription + ) + ) + return + } + let utterance = AVSpeechUtterance(string: text) + utterance.rate = AVSpeechUtteranceDefaultSpeechRate + speakingMessageId = messageId + speakingUtterance = utterance + messageSpeechSynthesizer.speak(utterance) + UIAccessibility.post( + notification: .announcement, + argument: "Reading message from \(author)" + ) + result(nil) + case "pause": + result(messageSpeechSynthesizer.pauseSpeaking(at: .word)) + case "resume": + result(messageSpeechSynthesizer.continueSpeaking()) + case "stop": + speakingMessageId = nil + speakingUtterance = nil + if messageSpeechSynthesizer.isSpeaking || messageSpeechSynthesizer.isPaused { + messageSpeechSynthesizer.stopSpeaking(at: .immediate) + } + try? AVAudioSession.sharedInstance().setActive( + false, + options: [.notifyOthersOnDeactivation] + ) + result(nil) + default: + result(FlutterMethodNotImplemented) + } + } + + func speechSynthesizer( + _ synthesizer: AVSpeechSynthesizer, + didFinish utterance: AVSpeechUtterance + ) { + guard speakingUtterance === utterance, let messageId = speakingMessageId else { + return + } + speakingMessageId = nil + speakingUtterance = nil + try? AVAudioSession.sharedInstance().setActive( + false, + options: [.notifyOthersOnDeactivation] + ) + messageReadAloudChannel?.invokeMethod("finished", arguments: messageId) + } + private static func handleQrScannerMethodCall( _ call: FlutterMethodCall, result: @escaping FlutterResult diff --git a/mobile/lib/features/channels/channel_detail_page.dart b/mobile/lib/features/channels/channel_detail_page.dart index 044342d101f..ef60284134a 100644 --- a/mobile/lib/features/channels/channel_detail_page.dart +++ b/mobile/lib/features/channels/channel_detail_page.dart @@ -40,6 +40,8 @@ import 'manage_channel_sheet.dart'; import 'members_sheet.dart'; import 'message_actions.dart'; import 'message_content.dart'; +import 'message_read_aloud.dart'; +import 'message_read_aloud_bar.dart'; import 'read_state/deferred_read_state_update.dart'; import 'read_state/read_state_provider.dart'; import 'read_state/read_state_time.dart'; @@ -194,6 +196,13 @@ class ChannelDetailPage extends HookConsumerWidget { return null; }, [channel.id]); + final readAloudNotifier = ref.read(messageReadAloudProvider.notifier); + useEffect( + () => + () => readAloudNotifier.stop(), + [channel.id, readAloudNotifier], + ); + useEffect(() { final messageId = initialMessageId; if (messageId == null || channel.isForum) return null; @@ -381,6 +390,7 @@ class ChannelDetailPage extends HookConsumerWidget { ? const SizedBox.shrink() : ChannelTypingIndicator(entries: typingEntries), ), + const MessageReadAloudBar(), if (!resolvedChannel.isForum && resolvedChannel.isMember && !resolvedChannel.isArchived) diff --git a/mobile/lib/features/channels/channel_detail_page/message_bubble.dart b/mobile/lib/features/channels/channel_detail_page/message_bubble.dart index cb6ab670659..88a626a0afc 100644 --- a/mobile/lib/features/channels/channel_detail_page/message_bubble.dart +++ b/mobile/lib/features/channels/channel_detail_page/message_bubble.dart @@ -99,6 +99,7 @@ class _MessageBubble extends ConsumerWidget { message: message, channelId: currentChannelId, canManageMessage: canManageMessage, + messageAuthor: displayName, allMessages: allMessages, currentPubkey: currentPubkey, isMember: isMember, diff --git a/mobile/lib/features/channels/message_actions.dart b/mobile/lib/features/channels/message_actions.dart index 519d25898c0..402e1a54dc7 100644 --- a/mobile/lib/features/channels/message_actions.dart +++ b/mobile/lib/features/channels/message_actions.dart @@ -22,6 +22,7 @@ import '../../shared/reminders/remind_me_later_sheet.dart'; import '../../shared/reminders/reminder_service.dart'; import 'channel_management_provider.dart'; import 'emoji_picker.dart'; +import 'message_read_aloud.dart'; import 'reaction_row.dart'; import 'recent_emoji_provider.dart'; import 'read_state/message_read_state.dart'; @@ -41,6 +42,7 @@ void showMessageActions({ required TimelineMessage message, required String channelId, required bool canManageMessage, + String? messageAuthor, List? allMessages, String? currentPubkey, bool isMember = false, @@ -50,97 +52,152 @@ void showMessageActions({ context: context, isScrollControlled: true, showDragHandle: true, - builder: (sheetContext) => SafeArea( - child: ConstrainedBox( - constraints: BoxConstraints( - maxHeight: MediaQuery.sizeOf(sheetContext).height * 0.7, - ), - child: SingleChildScrollView( - child: Padding( - padding: const EdgeInsets.fromLTRB( - Grid.gutter, - 0, - Grid.gutter, - Grid.xs, + builder: (sheetContext) => Consumer( + builder: (sheetContext, sheetRef, _) { + final playback = sheetRef.watch(messageReadAloudProvider); + final isThisMessage = playback.messageId == message.id; + final listenLabel = isThisMessage + ? switch (playback.status) { + MessageReadAloudStatus.preparing => 'Preparing audio…', + MessageReadAloudStatus.playing => 'Pause reading', + MessageReadAloudStatus.paused => 'Resume reading', + MessageReadAloudStatus.finished => 'Replay message', + _ => 'Listen to message', + } + : 'Listen to message'; + return SafeArea( + child: ConstrainedBox( + constraints: BoxConstraints( + maxHeight: MediaQuery.sizeOf(sheetContext).height * 0.7, ), - child: Column( - mainAxisSize: MainAxisSize.min, - children: [ - _QuickReactionRow( - message: message, - sheetContext: sheetContext, - pageContext: context, - pageRef: ref, + child: SingleChildScrollView( + child: Padding( + padding: const EdgeInsets.fromLTRB( + Grid.gutter, + 0, + Grid.gutter, + Grid.xs, ), - const SizedBox(height: Grid.xs), - if (!message.isSystem) ...[ - // Fast actions: respond now, hand off context, defer. - _FastActionsRow( - message: message, - channelId: channelId, - allMessages: allMessages, - currentPubkey: currentPubkey, - isMember: isMember, - isArchived: isArchived, - pageContext: context, - ), - const SizedBox(height: Grid.xs), - // Triage: come back to this message later. - _MarkReadUnreadTile(message: message, channelId: channelId), - _FollowThreadTile(message: message), - const SheetDivider(), - // Export: take the content out of the conversation. - ListTile( - leading: const Icon(LucideIcons.copy), - title: const Text('Copy text'), - onTap: () { - Navigator.of(sheetContext).pop(); - // Copy to clipboard - final data = ClipboardData(text: message.content); - Clipboard.setData(data); - }, - ), - ], - if (canManageMessage) ...[ - if (!message.isSystem) const SheetDivider(), - ListTile( - leading: const Icon(LucideIcons.pencil), - title: const Text('Edit message'), - onTap: () { - Navigator.of(sheetContext).pop(); - _showEditSheet( - context: context, - ref: ref, + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + _QuickReactionRow( + message: message, + sheetContext: sheetContext, + pageContext: context, + pageRef: ref, + ), + const SizedBox(height: Grid.xs), + if (!message.isSystem) ...[ + // Fast actions: respond now, hand off context, defer. + _FastActionsRow( message: message, channelId: channelId, - ); - }, - ), - ListTile( - leading: Icon( - LucideIcons.trash2, - color: sheetContext.colors.error, - ), - title: Text( - 'Delete message', - style: TextStyle(color: sheetContext.colors.error), - ), - onTap: () { - Navigator.of(sheetContext).pop(); - _confirmDelete( - context: context, - ref: ref, + allMessages: allMessages, + currentPubkey: currentPubkey, + isMember: isMember, + isArchived: isArchived, + pageContext: context, + ), + const SizedBox(height: Grid.xs), + if (defaultTargetPlatform == TargetPlatform.iOS && + message.content.trim().isNotEmpty) + ListTile( + leading: Icon( + isThisMessage && + playback.status == + MessageReadAloudStatus.playing + ? LucideIcons.pause + : isThisMessage && + playback.status == + MessageReadAloudStatus.paused + ? LucideIcons.play + : isThisMessage && + playback.status == + MessageReadAloudStatus.finished + ? LucideIcons.rotateCcw + : LucideIcons.volume2, + ), + title: Text(listenLabel), + onTap: + isThisMessage && + playback.status == + MessageReadAloudStatus.preparing + ? null + : () { + Navigator.of(sheetContext).pop(); + unawaited( + sheetRef + .read(messageReadAloudProvider.notifier) + .listen( + messageId: message.id, + author: messageAuthor ?? 'Agent', + markdown: message.content, + ), + ); + }, + ), + // Triage: come back to this message later. + _MarkReadUnreadTile( + message: message, channelId: channelId, - messageId: message.id, - ); - }, - ), - ], - ], + ), + _FollowThreadTile(message: message), + const SheetDivider(), + // Export: take the content out of the conversation. + ListTile( + leading: const Icon(LucideIcons.copy), + title: const Text('Copy text'), + onTap: () { + Navigator.of(sheetContext).pop(); + // Copy to clipboard + final data = ClipboardData(text: message.content); + Clipboard.setData(data); + }, + ), + ], + if (canManageMessage) ...[ + if (!message.isSystem) const SheetDivider(), + ListTile( + leading: const Icon(LucideIcons.pencil), + title: const Text('Edit message'), + onTap: () { + Navigator.of(sheetContext).pop(); + _showEditSheet( + context: context, + ref: ref, + message: message, + channelId: channelId, + ); + }, + ), + ListTile( + leading: Icon( + LucideIcons.trash2, + color: sheetContext.colors.error, + ), + title: Text( + 'Delete message', + style: TextStyle(color: sheetContext.colors.error), + ), + onTap: () { + Navigator.of(sheetContext).pop(); + _confirmDelete( + context: context, + ref: ref, + channelId: channelId, + messageId: message.id, + ); + }, + ), + ], + ], + ), + ), ), ), - ), - ), + ); + }, ), ); } diff --git a/mobile/lib/features/channels/message_read_aloud.dart b/mobile/lib/features/channels/message_read_aloud.dart new file mode 100644 index 00000000000..2b55be5e4bb --- /dev/null +++ b/mobile/lib/features/channels/message_read_aloud.dart @@ -0,0 +1,245 @@ +import 'dart:async'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter/widgets.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; + +enum MessageReadAloudStatus { + idle, + preparing, + playing, + paused, + finished, + error, +} + +@immutable +class MessageReadAloudState { + final String? messageId; + final String author; + final String text; + final MessageReadAloudStatus status; + final String? error; + + const MessageReadAloudState({ + this.messageId, + this.author = '', + this.text = '', + this.status = MessageReadAloudStatus.idle, + this.error, + }); +} + +String messageTextForSpeech(String markdown) { + final codeSpans = []; + String preserveCode(String text) { + final index = codeSpans.length; + codeSpans.add(text); + return '\uE000$index\uE001'; + } + + return markdown + .replaceAllMapped( + RegExp(r'```(?:[^\n`]*)\n?([\s\S]*?)```'), + (match) => preserveCode(match.group(1) ?? ''), + ) + .replaceAllMapped( + RegExp(r'`([^`]+)`'), + (match) => preserveCode(match.group(1) ?? ''), + ) + .replaceAllMapped( + RegExp(r'!\[([^\]]*)\]\([^)]*\)'), + (match) => match.group(1) ?? '', + ) + .replaceAllMapped( + RegExp(r'\[([^\]]+)\]\([^)]*\)'), + (match) => match.group(1) ?? '', + ) + .replaceAll( + RegExp(r'^\s{0,3}(?:#{1,6}|>|[-+*]|\d+[.)])\s+', multiLine: true), + '', + ) + .replaceAllMapped( + RegExp(r'\*\*([^*\n]+)\*\*'), + (match) => match.group(1) ?? '', + ) + .replaceAllMapped( + RegExp(r'(? match.group(1) ?? '', + ) + .replaceAllMapped( + RegExp(r'(? match.group(1) ?? '', + ) + .replaceAllMapped( + RegExp(r'(? match.group(1) ?? '', + ) + .replaceAllMapped( + RegExp(r'~~([^~\n]+)~~'), + (match) => match.group(1) ?? '', + ) + .replaceAllMapped( + RegExp(r'\uE000(\d+)\uE001'), + (match) => codeSpans[int.parse(match.group(1)!)], + ) + .replaceAll(RegExp(r'\n{3,}'), '\n\n') + .trim(); +} + +class MessageReadAloudNotifier extends Notifier { + static const _channel = MethodChannel('buzz/message_read_aloud'); + AppLifecycleListener? _lifecycleListener; + bool get _isSupported => defaultTargetPlatform == TargetPlatform.iOS; + + @override + MessageReadAloudState build() { + _channel.setMethodCallHandler(_handleNativeCallback); + _lifecycleListener?.dispose(); + _lifecycleListener = AppLifecycleListener(onPause: stop, onDetach: stop); + ref.onDispose(() { + _lifecycleListener?.dispose(); + _lifecycleListener = null; + _channel.setMethodCallHandler(null); + if (_isSupported) unawaited(_channel.invokeMethod('stop')); + }); + return const MessageReadAloudState(); + } + + Future listen({ + required String messageId, + required String author, + required String markdown, + }) async { + if (state.messageId == messageId) { + if (state.status == MessageReadAloudStatus.playing || + state.status == MessageReadAloudStatus.preparing) { + await pause(); + return; + } + if (state.status == MessageReadAloudStatus.paused) { + await resume(); + return; + } + } + + await _speakText( + messageId: messageId, + author: author, + text: messageTextForSpeech(markdown), + ); + } + + Future _speakText({ + required String messageId, + required String author, + required String text, + }) async { + state = MessageReadAloudState( + messageId: messageId, + author: author, + text: text, + status: MessageReadAloudStatus.preparing, + ); + + if (!_isSupported || text.isEmpty) { + _fail(messageId); + return; + } + + try { + await _channel.invokeMethod('speak', { + 'messageId': messageId, + 'author': author, + 'text': text, + }); + if (state.messageId == messageId && + state.status == MessageReadAloudStatus.preparing) { + state = MessageReadAloudState( + messageId: messageId, + author: author, + text: text, + status: MessageReadAloudStatus.playing, + ); + } + } catch (error) { + debugPrint('message read aloud failed: $error'); + _fail(messageId); + } + } + + Future pause() async { + if (state.status != MessageReadAloudStatus.playing) return; + final paused = await _channel.invokeMethod('pause') ?? false; + if (paused) { + state = MessageReadAloudState( + messageId: state.messageId, + author: state.author, + text: state.text, + status: MessageReadAloudStatus.paused, + ); + } + } + + Future resume() async { + if (state.status != MessageReadAloudStatus.paused) return; + final resumed = await _channel.invokeMethod('resume') ?? false; + if (resumed) { + state = MessageReadAloudState( + messageId: state.messageId, + author: state.author, + text: state.text, + status: MessageReadAloudStatus.playing, + ); + } + } + + Future retry() async { + final messageId = state.messageId; + if (messageId == null) return; + // `state.text` is already flattened — re-running messageTextForSpeech + // would treat code content (e.g. `__init__`) as Markdown markers. + await _speakText( + messageId: messageId, + author: state.author, + text: state.text, + ); + } + + void stop() { + if (_isSupported) unawaited(_channel.invokeMethod('stop')); + state = const MessageReadAloudState(); + } + + Future _handleNativeCallback(MethodCall call) async { + final messageId = call.arguments as String?; + if (messageId == null || state.messageId != messageId) return; + if (call.method == 'finished') { + state = MessageReadAloudState( + messageId: state.messageId, + author: state.author, + text: state.text, + status: MessageReadAloudStatus.finished, + ); + } else if (call.method == 'failed') { + _fail(messageId); + } + } + + void _fail(String messageId) { + if (state.messageId != messageId) return; + state = MessageReadAloudState( + messageId: state.messageId, + author: state.author, + text: state.text, + status: MessageReadAloudStatus.error, + error: "Couldn't play audio. Try again.", + ); + } +} + +final messageReadAloudProvider = + NotifierProvider( + MessageReadAloudNotifier.new, + ); diff --git a/mobile/lib/features/channels/message_read_aloud_bar.dart b/mobile/lib/features/channels/message_read_aloud_bar.dart new file mode 100644 index 00000000000..e600a76d773 --- /dev/null +++ b/mobile/lib/features/channels/message_read_aloud_bar.dart @@ -0,0 +1,122 @@ +import 'dart:async'; + +import 'package:flutter/material.dart'; +import 'package:flutter_hooks/flutter_hooks.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:lucide_icons_flutter/lucide_icons.dart'; + +import '../../shared/theme/theme.dart'; +import 'message_read_aloud.dart'; + +class MessageReadAloudBar extends HookConsumerWidget { + const MessageReadAloudBar({super.key}); + + @override + Widget build(BuildContext context, WidgetRef ref) { + final playback = ref.watch(messageReadAloudProvider); + final showPreparing = useState(false); + useEffect(() { + if (playback.status != MessageReadAloudStatus.preparing) { + showPreparing.value = false; + return null; + } + final timer = Timer(const Duration(milliseconds: 200), () { + showPreparing.value = true; + }); + return timer.cancel; + }, [playback.status]); + if (playback.messageId == null || + playback.status == MessageReadAloudStatus.idle || + playback.status == MessageReadAloudStatus.finished || + (playback.status == MessageReadAloudStatus.preparing && + !showPreparing.value)) { + return const SizedBox.shrink(); + } + + final notifier = ref.read(messageReadAloudProvider.notifier); + final isPaused = playback.status == MessageReadAloudStatus.paused; + final isError = playback.status == MessageReadAloudStatus.error; + final isPreparing = playback.status == MessageReadAloudStatus.preparing; + + return Semantics( + liveRegion: true, + label: isError + ? playback.error + : isPreparing + ? 'Preparing audio' + : 'Reading message from ${playback.author}', + child: Container( + margin: const EdgeInsets.fromLTRB( + Grid.gutter, + Grid.quarter, + Grid.gutter, + Grid.quarter, + ), + padding: const EdgeInsets.symmetric( + horizontal: Grid.half, + vertical: Grid.quarter, + ), + constraints: const BoxConstraints(minHeight: 44), + decoration: BoxDecoration( + color: context.colors.surface, + border: Border.all(color: context.colors.outlineVariant), + borderRadius: BorderRadius.circular(Radii.lg), + ), + child: Row( + children: [ + if (isPreparing) + SizedBox.square( + dimension: 18, + child: CircularProgressIndicator( + strokeWidth: 2, + color: context.colors.primary, + ), + ) + else + Icon( + LucideIcons.volume2, + size: 18, + color: context.colors.primary, + ), + const SizedBox(width: Grid.half), + Expanded( + child: Text( + isError + ? "Couldn't play audio. Try again." + : isPreparing + ? 'Preparing audio…' + : 'Reading message from ${playback.author}', + maxLines: 1, + overflow: TextOverflow.ellipsis, + style: context.textTheme.bodySmall?.copyWith( + color: context.colors.onSurfaceVariant, + ), + ), + ), + if (isError) + TextButton.icon( + onPressed: () => unawaited(notifier.retry()), + icon: const Icon(LucideIcons.rotateCcw, size: 16), + label: const Text('Retry'), + ) + else if (!isPreparing) + IconButton( + tooltip: isPaused ? 'Resume reading' : 'Pause reading', + onPressed: () => + unawaited(isPaused ? notifier.resume() : notifier.pause()), + icon: Icon( + isPaused ? LucideIcons.play : LucideIcons.pause, + size: 18, + ), + ), + IconButton( + tooltip: 'Stop reading', + onPressed: notifier.stop, + icon: const Icon(LucideIcons.x, size: 18), + ), + ], + ), + ), + ); + } +} diff --git a/mobile/lib/features/channels/thread_detail_page.dart b/mobile/lib/features/channels/thread_detail_page.dart index 810861aa00f..e8f34eea15d 100644 --- a/mobile/lib/features/channels/thread_detail_page.dart +++ b/mobile/lib/features/channels/thread_detail_page.dart @@ -25,6 +25,8 @@ import 'day_divider.dart'; import '../profile/user_profile_sheet.dart'; import 'message_actions.dart'; import 'message_content.dart'; +import 'message_read_aloud.dart'; +import 'message_read_aloud_bar.dart'; import 'reaction_row.dart'; import 'read_state/read_state_format.dart'; import 'read_state/read_state_provider.dart'; @@ -67,6 +69,12 @@ class ThreadDetailPage extends HookConsumerWidget { ThreadRepliesArgs(channelId: channelId, rootId: queryRootId), ), ); + final readAloudNotifier = ref.read(messageReadAloudProvider.notifier); + useEffect( + () => + () => readAloudNotifier.stop(), + [channelId, threadHead.id, readAloudNotifier], + ); // The thread query is one-shot and asks only for content kinds, so a // reaction, edit, or deletion that lands while the thread is open never // reaches it — a new pill (and its burst) only showed up after leaving and @@ -390,6 +398,7 @@ class ThreadDetailPage extends HookConsumerWidget { ? const SizedBox.shrink() : ChannelTypingIndicator(entries: threadTyping), ), + const MessageReadAloudBar(), if (isMember && !isArchived) ComposeBar( channelId: channelId, @@ -660,6 +669,7 @@ class _ThreadMessage extends ConsumerWidget { message: message, channelId: channelId, canManageMessage: canManageMessage, + messageAuthor: displayName, allMessages: allMessages, currentPubkey: currentPubkey, isMember: isMember, diff --git a/mobile/test/features/channels/message_read_aloud_test.dart b/mobile/test/features/channels/message_read_aloud_test.dart new file mode 100644 index 00000000000..024defa4a54 --- /dev/null +++ b/mobile/test/features/channels/message_read_aloud_test.dart @@ -0,0 +1,31 @@ +import 'package:buzz/features/channels/message_read_aloud.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('preserves readable words while flattening Markdown', () { + expect( + messageTextForSpeech( + '## **Update**\n\n' + 'Read [the guide](https://example.com), then run `buzz check`.\n\n' + '```sh\nbuzz status\n```', + ), + 'Update\n\nRead the guide, then run buzz check.\n\nbuzz status', + ); + }); + + test('keeps image alt text and list content', () { + expect( + messageTextForSpeech('- First\n- ![Diagram](https://example.com/a.png)'), + 'First\nDiagram', + ); + }); + + test('preserves code identifiers exactly', () { + expect( + messageTextForSpeech( + 'Use `message_read_aloud.dart` and `__init__` next.', + ), + 'Use message_read_aloud.dart and __init__ next.', + ); + }); +}