diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e2043786fd..43e3af0dd63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,7 @@ jobs: build-essential \ curl \ file \ + libasound2-dev \ libayatana-appindicator3-dev \ libgtk-3-dev \ librsvg2-dev \ diff --git a/Cargo.lock b/Cargo.lock index 6a191b60d7d..0fa7e765f3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3904,6 +3904,7 @@ dependencies = [ "sprout-auth", "sprout-core", "sprout-db", + "sprout-huddle", "sprout-media", "sprout-pubsub", "sprout-search", diff --git a/crates/sprout-core/src/kind.rs b/crates/sprout-core/src/kind.rs index 164b48ae2d0..ef2a777f83f 100644 --- a/crates/sprout-core/src/kind.rs +++ b/crates/sprout-core/src/kind.rs @@ -219,6 +219,8 @@ pub const KIND_HUDDLE_ENDED: u32 = 48103; pub const KIND_HUDDLE_TRACK_PUBLISHED: u32 = 48104; /// A huddle recording became available. pub const KIND_HUDDLE_RECORDING_AVAILABLE: u32 = 48105; +/// Huddle channel guidelines/rules document. +pub const KIND_HUDDLE_GUIDELINES: u32 = 48106; // Media (49000–49999) /// Internal kind for media upload audit entries. Not a relay event kind. diff --git a/crates/sprout-huddle/Cargo.toml b/crates/sprout-huddle/Cargo.toml index 07356cb51a7..3f430a89343 100644 --- a/crates/sprout-huddle/Cargo.toml +++ b/crates/sprout-huddle/Cargo.toml @@ -18,6 +18,10 @@ chrono = { workspace = true } tracing = { workspace = true } thiserror = { workspace = true } jsonwebtoken = { workspace = true } -hmac = { workspace = true } -sha2 = { workspace = true } -hex = { workspace = true } +hmac = { workspace = true, optional = true } +sha2 = { workspace = true, optional = true } +hex = { workspace = true, optional = true } + +[features] +default = [] +webhook = ["hmac", "sha2", "hex"] diff --git a/crates/sprout-huddle/src/lib.rs b/crates/sprout-huddle/src/lib.rs index 1eb3a2c6aa0..2d4492cc7d1 100644 --- a/crates/sprout-huddle/src/lib.rs +++ b/crates/sprout-huddle/src/lib.rs @@ -9,19 +9,24 @@ /// Error types for the huddle layer. pub mod error; /// In-memory huddle session and participant tracking. +#[cfg(feature = "webhook")] pub mod session; /// LiveKit access token generation. pub mod token; /// LiveKit webhook signature verification and event parsing. +#[cfg(feature = "webhook")] pub mod webhook; pub use error::HuddleError; +#[cfg(feature = "webhook")] pub use session::{HuddleParticipant, HuddleSession, TrackInfo, TrackKind}; pub use token::LiveKitToken; +#[cfg(feature = "webhook")] pub use webhook::WebhookEvent; use uuid::Uuid; +#[cfg(feature = "webhook")] pub use sprout_core::kind::{ KIND_HUDDLE_ENDED, KIND_HUDDLE_PARTICIPANT_JOINED, KIND_HUDDLE_PARTICIPANT_LEFT, KIND_HUDDLE_RECORDING_AVAILABLE, KIND_HUDDLE_STARTED, KIND_HUDDLE_TRACK_PUBLISHED, @@ -74,6 +79,7 @@ impl HuddleService { } /// Verify the webhook signature and parse the LiveKit event payload. + #[cfg(feature = "webhook")] pub fn parse_webhook( &self, body: &[u8], @@ -146,6 +152,7 @@ mod tests { } #[test] + #[cfg(feature = "webhook")] fn session_lifecycle() { let channel_id = Uuid::new_v4(); let room_name = HuddleService::create_room_name(channel_id); diff --git a/crates/sprout-huddle/src/token.rs b/crates/sprout-huddle/src/token.rs index 611a62cda0c..ac286f87f89 100644 --- a/crates/sprout-huddle/src/token.rs +++ b/crates/sprout-huddle/src/token.rs @@ -50,7 +50,7 @@ pub fn generate_token( ttl: Option, ) -> Result { let now = Utc::now(); - let ttl = ttl.unwrap_or_else(|| Duration::hours(6)); + let ttl = ttl.unwrap_or_else(|| Duration::hours(1)); let expires_at = now + ttl; let claims = LiveKitClaims { diff --git a/crates/sprout-relay/Cargo.toml b/crates/sprout-relay/Cargo.toml index ab56a601c38..09da25aecd8 100644 --- a/crates/sprout-relay/Cargo.toml +++ b/crates/sprout-relay/Cargo.toml @@ -38,6 +38,7 @@ deadpool-redis = { workspace = true } redis = { workspace = true } sqlx = { workspace = true } base64 = "0.22" +sprout-huddle = { workspace = true, features = ["webhook"] } sprout-workflow = { workspace = true, features = ["reqwest"] } sprout-media = { workspace = true } bytes = "1" diff --git a/crates/sprout-relay/src/api/huddles.rs b/crates/sprout-relay/src/api/huddles.rs new file mode 100644 index 00000000000..d83ca2c0005 --- /dev/null +++ b/crates/sprout-relay/src/api/huddles.rs @@ -0,0 +1,128 @@ +//! LiveKit huddle token endpoint. +//! +//! ## Routes +//! - `POST /api/huddles/{channel_id}/token` — generate a LiveKit access token +//! for the authenticated user to join the channel's huddle room. + +use std::sync::Arc; + +use axum::{ + extract::{Path, Query, State}, + http::{HeaderMap, StatusCode}, + response::Json, +}; +use uuid::Uuid; + +use super::{ + check_channel_membership, check_token_channel_access, extract_auth_context, internal_error, + scope_error, +}; +use crate::state::AppState; + +/// Query parameters for [`huddle_token`]. +#[derive(serde::Deserialize)] +pub struct HuddleTokenQuery { + /// The parent (non-ephemeral) channel this huddle belongs to. + /// + /// When provided and the caller is a member of the parent channel, the relay + /// will auto-add them to the private ephemeral huddle channel so they can + /// obtain a token without requiring an explicit invite. + pub parent_channel_id: Option, +} + +/// `POST /api/huddles/{channel_id}/token` — generate a LiveKit access token. +/// +/// Returns `{ "token": "", "url": "", "room": "sprout-" }`. +/// Returns 501 if LiveKit is not configured on this relay. +/// Returns 403 if the caller is not a member of the channel. +pub async fn huddle_token( + State(state): State>, + headers: HeaderMap, + Path(channel_id): Path, + Query(query): Query, +) -> Result, (StatusCode, Json)> { + let ctx = extract_auth_context(&headers, &state).await?; + sprout_auth::require_scope(&ctx.scopes, sprout_auth::Scope::MessagesRead) + .map_err(scope_error)?; + check_token_channel_access(&ctx, &channel_id)?; + + // Require LiveKit to be configured. + let huddle_service = state.huddle_service.as_ref().ok_or_else(|| { + ( + StatusCode::NOT_IMPLEMENTED, + Json(serde_json::json!({ + "error": "huddles_not_configured", + "message": "LiveKit is not configured on this relay" + })), + ) + })?; + + // Verify the caller is a member of the channel (or it's an open channel). + // If they're not a member, attempt relay-side auto-add when: + // 1. parent_channel_id was provided + // 2. The target channel is private + ephemeral (ttl_seconds IS NOT NULL) + // 3. The caller IS a member of the parent channel + let membership_result = check_channel_membership(&state, channel_id, &ctx.pubkey_bytes).await; + + if let Err(ref membership_err) = membership_result { + if let Some(parent_id) = query.parent_channel_id { + // Gate 1: caller must be a member of the parent channel. + check_channel_membership(&state, parent_id, &ctx.pubkey_bytes).await?; + + // Gate 2: target channel must be private + ephemeral. + let channel = state + .db + .get_channel(channel_id) + .await + .map_err(|e| internal_error(&format!("db error: {e}")))?; + + if channel.visibility == "private" && channel.ttl_seconds.is_some() { + // Auto-add: use the channel creator as invited_by (truthful attribution). + // The creator is always an active owner, satisfying add_member's invite check. + state + .db + .add_member( + channel_id, + &ctx.pubkey_bytes, + sprout_db::channel::MemberRole::Member, + Some(&channel.created_by), + ) + .await + .map_err(|e| internal_error(&format!("auto-add failed: {e}")))?; + + tracing::info!( + "Huddle auto-add: added {} to ephemeral channel {} (parent: {})", + ctx.pubkey.to_hex(), + channel_id, + parent_id + ); + // Fall through to token generation. + } else { + // Not a private ephemeral channel — return the original 403. + return Err(membership_err.clone()); + } + } else { + // No parent_channel_id provided — return the original 403. + membership_result?; + } + } + + // Generate the LiveKit token. + let room = sprout_huddle::HuddleService::create_room_name(channel_id); + let identity = ctx.pubkey.to_hex(); + let lk_token = huddle_service + .generate_token(&room, &identity, &identity) + .map_err(|e| internal_error(&format!("token generation failed: {e}")))?; + + let livekit_url = state.livekit_url.as_deref().unwrap_or_default(); + + if livekit_url.is_empty() { + return Err(internal_error("livekit_url is not configured")); + } + + Ok(Json(serde_json::json!({ + "token": lk_token.token, + "url": livekit_url, + "room": room, + }))) +} diff --git a/crates/sprout-relay/src/api/mod.rs b/crates/sprout-relay/src/api/mod.rs index 8cddece093c..ffac082907a 100644 --- a/crates/sprout-relay/src/api/mod.rs +++ b/crates/sprout-relay/src/api/mod.rs @@ -26,6 +26,8 @@ pub mod dms; pub mod events; /// Personalized home feed endpoint. pub mod feed; +/// LiveKit huddle token endpoint. +pub mod huddles; /// Blossom-compatible media upload, retrieval, and existence check endpoints. pub mod media; /// Channel membership endpoints. @@ -44,6 +46,8 @@ pub mod search; pub mod tokens; /// User profile endpoints. pub mod users; +/// LiveKit webhook handler for server-side presence tracking. +pub mod webhooks; /// Shared helpers for workflow API handlers. pub mod workflow_helpers; /// Workflow CRUD, trigger, and webhook endpoints. @@ -58,6 +62,7 @@ pub use channels_metadata::get_channel_handler; pub use dms::{add_dm_member_handler, hide_dm_handler, list_dms_handler, open_dm_handler}; pub use events::get_event; pub use feed::feed_handler; +pub use huddles::huddle_token; pub use members::list_members; pub use messages::{get_thread, list_messages, validate_imeta_tags, verify_imeta_blobs}; pub use presence::{presence_handler, set_presence_handler}; @@ -67,6 +72,7 @@ pub use users::{ get_contact_list, get_profile, get_user_notes, get_user_profile, get_users_batch, put_channel_add_policy, search_users, }; +pub use webhooks::handle_livekit_webhook; pub use workflows::{ create_workflow, delete_workflow, get_workflow, list_channel_workflows, list_run_approvals, list_workflow_runs, trigger_workflow, update_workflow, workflow_webhook, diff --git a/crates/sprout-relay/src/api/webhooks.rs b/crates/sprout-relay/src/api/webhooks.rs new file mode 100644 index 00000000000..57c6da4f39e --- /dev/null +++ b/crates/sprout-relay/src/api/webhooks.rs @@ -0,0 +1,120 @@ +//! LiveKit webhook handler — server-side huddle presence tracking. +//! +//! Receives webhook events from LiveKit and emits corresponding Nostr +//! huddle lifecycle events (kinds 48100–48103). This provides authoritative +//! presence tracking that survives client crashes — LiveKit fires +//! `participant_left` when the WebRTC connection drops, regardless of +//! whether the client performed a graceful shutdown. +//! +//! ## Route +//! `POST /internal/livekit/webhook` — internal only, not exposed through +//! the public API gateway. + +use std::sync::Arc; + +use axum::{ + extract::State, + http::{HeaderMap, StatusCode}, +}; +use sprout_huddle::WebhookEvent; + +use crate::state::AppState; + +/// Handle a LiveKit webhook event. +/// +/// Verifies the webhook signature, parses the event, and emits the +/// corresponding Nostr huddle lifecycle event to the parent channel. +/// +/// Returns 200 on success (even if the event type is unrecognized — +/// LiveKit expects 2xx for all webhook deliveries). +/// Returns 401 if signature verification fails. +/// Returns 501 if huddles are not configured. +pub async fn handle_livekit_webhook( + State(state): State>, + headers: HeaderMap, + body: String, +) -> StatusCode { + let huddle_service = match state.huddle_service.as_ref() { + Some(svc) => svc, + None => return StatusCode::NOT_IMPLEMENTED, + }; + + // Verify signature and parse the event. + let auth_header = headers + .get("Authorization") + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + + let event = match huddle_service.parse_webhook(body.as_bytes(), auth_header) { + Ok(e) => e, + Err(sprout_huddle::HuddleError::InvalidWebhookSignature) => { + tracing::warn!("LiveKit webhook: signature verification failed"); + return StatusCode::UNAUTHORIZED; + } + Err(e) => { + // Signed but malformed/unknown — log and return 200 so LiveKit + // doesn't retry. Parse failures are not auth failures. + tracing::warn!("LiveKit webhook: parse error (signed OK): {e}"); + return StatusCode::OK; + } + }; + + // Dispatch on the parsed enum variant. + // Room names follow the format "sprout-{channel_uuid}". + match event { + WebhookEvent::RoomStarted { room } => { + tracing::info!("LiveKit: room started {room}"); + // TODO: Emit kind:48100 signed by relay keypair + } + WebhookEvent::RoomFinished { room } => { + tracing::info!("LiveKit: room finished {room}"); + // TODO: Emit kind:48103 + archive ephemeral channel + } + WebhookEvent::ParticipantJoined { room, identity } => { + let Some(channel_id) = parse_channel_id(&room) else { + tracing::debug!("LiveKit webhook for non-sprout room or invalid UUID: {room}"); + return StatusCode::OK; + }; + tracing::info!( + "LiveKit: participant joined room {room} (channel {channel_id}), identity={identity}" + ); + // TODO: Emit kind:48101 signed by relay keypair. + // The client also emits this event; the server-side copy provides + // crash-recovery redundancy. + } + WebhookEvent::ParticipantLeft { room, identity } => { + let Some(channel_id) = parse_channel_id(&room) else { + tracing::debug!("LiveKit webhook for non-sprout room or invalid UUID: {room}"); + return StatusCode::OK; + }; + tracing::info!( + "LiveKit: participant left room {room} (channel {channel_id}), identity={identity}" + ); + // TODO: Emit kind:48102 signed by relay keypair. + // This is the key crash-recovery path — fires even if the client crashed. + } + WebhookEvent::TrackPublished { + room, + identity, + kind, + } => { + tracing::debug!("LiveKit: track published in {room} by {identity} (kind={kind})"); + // TODO: Emit kind:48104 (track published) if/when that event kind is defined. + } + } + + StatusCode::OK +} + +/// Extract the channel UUID from a LiveKit room name. +/// +/// Room names follow the format `sprout-{uuid}`. Returns `None` if the name +/// does not match the expected prefix or contains an invalid UUID. +fn parse_channel_id(room_name: &str) -> Option { + let channel_id = room_name.strip_prefix("sprout-")?; + uuid::Uuid::parse_str(channel_id) + .map_err(|_| { + tracing::warn!("LiveKit webhook: invalid channel UUID in room name: {room_name}"); + }) + .ok() +} diff --git a/crates/sprout-relay/src/handlers/ingest.rs b/crates/sprout-relay/src/handlers/ingest.rs index c06f03b370d..22429062131 100644 --- a/crates/sprout-relay/src/handlers/ingest.rs +++ b/crates/sprout-relay/src/handlers/ingest.rs @@ -14,13 +14,15 @@ use sprout_auth::Scope; use sprout_core::kind::{ event_kind_u32, is_parameterized_replaceable, KIND_AUTH, KIND_CANVAS, KIND_CONTACT_LIST, KIND_DELETION, KIND_FORUM_COMMENT, KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_GIFT_WRAP, - KIND_LONG_FORM, KIND_MEMBER_ADDED_NOTIFICATION, KIND_MEMBER_REMOVED_NOTIFICATION, - KIND_NIP29_CREATE_GROUP, KIND_NIP29_DELETE_EVENT, KIND_NIP29_DELETE_GROUP, - KIND_NIP29_EDIT_METADATA, KIND_NIP29_JOIN_REQUEST, KIND_NIP29_LEAVE_REQUEST, - KIND_NIP29_PUT_USER, KIND_NIP29_REMOVE_USER, KIND_PRESENCE_UPDATE, KIND_PROFILE, KIND_REACTION, - KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_BOOKMARKED, KIND_STREAM_MESSAGE_DIFF, - KIND_STREAM_MESSAGE_EDIT, KIND_STREAM_MESSAGE_PINNED, KIND_STREAM_MESSAGE_SCHEDULED, - KIND_STREAM_MESSAGE_V2, KIND_STREAM_REMINDER, KIND_TEXT_NOTE, + KIND_HUDDLE_ENDED, KIND_HUDDLE_GUIDELINES, KIND_HUDDLE_PARTICIPANT_JOINED, + KIND_HUDDLE_PARTICIPANT_LEFT, KIND_HUDDLE_RECORDING_AVAILABLE, KIND_HUDDLE_STARTED, + KIND_HUDDLE_TRACK_PUBLISHED, KIND_LONG_FORM, KIND_MEMBER_ADDED_NOTIFICATION, + KIND_MEMBER_REMOVED_NOTIFICATION, KIND_NIP29_CREATE_GROUP, KIND_NIP29_DELETE_EVENT, + KIND_NIP29_DELETE_GROUP, KIND_NIP29_EDIT_METADATA, KIND_NIP29_JOIN_REQUEST, + KIND_NIP29_LEAVE_REQUEST, KIND_NIP29_PUT_USER, KIND_NIP29_REMOVE_USER, KIND_PRESENCE_UPDATE, + KIND_PROFILE, KIND_REACTION, KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_BOOKMARKED, + KIND_STREAM_MESSAGE_DIFF, KIND_STREAM_MESSAGE_EDIT, KIND_STREAM_MESSAGE_PINNED, + KIND_STREAM_MESSAGE_SCHEDULED, KIND_STREAM_MESSAGE_V2, KIND_STREAM_REMINDER, KIND_TEXT_NOTE, }; use sprout_core::verification::verify_event; @@ -176,6 +178,14 @@ fn required_scope_for_kind(kind: u32, event: &Event) -> Result Ok(Scope::ChannelsWrite), KIND_NIP29_JOIN_REQUEST | KIND_NIP29_LEAVE_REQUEST => Ok(Scope::ChannelsRead), + // Huddle lifecycle events + guidelines + KIND_HUDDLE_STARTED + | KIND_HUDDLE_PARTICIPANT_JOINED + | KIND_HUDDLE_PARTICIPANT_LEFT + | KIND_HUDDLE_ENDED + | KIND_HUDDLE_TRACK_PUBLISHED + | KIND_HUDDLE_RECORDING_AVAILABLE + | KIND_HUDDLE_GUIDELINES => Ok(Scope::ChannelsWrite), _ => Err("restricted: unknown event kind"), } } @@ -283,6 +293,12 @@ pub(crate) fn requires_h_channel_scope(kind: u32) -> bool { | KIND_NIP29_DELETE_EVENT | KIND_NIP29_DELETE_GROUP | KIND_NIP29_LEAVE_REQUEST + // Huddle lifecycle events + guidelines + | KIND_HUDDLE_STARTED + | KIND_HUDDLE_PARTICIPANT_JOINED + | KIND_HUDDLE_PARTICIPANT_LEFT + | KIND_HUDDLE_ENDED + | KIND_HUDDLE_GUIDELINES ) } diff --git a/crates/sprout-relay/src/main.rs b/crates/sprout-relay/src/main.rs index 3ced6651bc9..a93cab0ef02 100644 --- a/crates/sprout-relay/src/main.rs +++ b/crates/sprout-relay/src/main.rs @@ -10,6 +10,7 @@ use sprout_db::{Db, DbConfig}; use sprout_pubsub::PubSubManager; use sprout_search::{SearchConfig, SearchService}; +use sprout_huddle::{HuddleConfig, HuddleService}; use sprout_relay::config::Config; use sprout_relay::metrics as relay_metrics; use sprout_relay::router::{build_health_router, build_router}; @@ -127,6 +128,28 @@ async fn main() -> anyhow::Result<()> { .map_err(|e| anyhow::anyhow!("failed to initialize media storage: {e}"))?; info!("Media storage connected"); + // Huddles are enabled by default with dev credentials. + // Set SPROUT_HUDDLES_DISABLED=true to turn them off. + // Override LIVEKIT_URL / LIVEKIT_API_KEY / LIVEKIT_API_SECRET for production. + let huddle_service = if std::env::var("SPROUT_HUDDLES_DISABLED") + .map(|v| v == "true" || v == "1") + .unwrap_or(false) + { + info!("Huddles explicitly disabled via SPROUT_HUDDLES_DISABLED"); + None + } else { + let url = std::env::var("LIVEKIT_URL").unwrap_or_else(|_| "ws://localhost:7880".into()); + let key = std::env::var("LIVEKIT_API_KEY").unwrap_or_else(|_| "devkey".into()); + let secret = std::env::var("LIVEKIT_API_SECRET").unwrap_or_else(|_| "secret".into()); + info!("Huddles enabled (LiveKit URL: {url})"); + let svc = HuddleService::new(HuddleConfig { + livekit_url: url.clone(), + livekit_api_key: key, + livekit_api_secret: secret, + }); + Some((svc, url)) + }; + let state = Arc::new(AppState::new( config.clone(), db, @@ -138,6 +161,7 @@ async fn main() -> anyhow::Result<()> { Arc::clone(&workflow_engine), relay_keypair, media_storage, + huddle_service, )); // Wire the action sink — must happen after AppState (which creates diff --git a/crates/sprout-relay/src/router.rs b/crates/sprout-relay/src/router.rs index e449b66d5b5..921213cc16d 100644 --- a/crates/sprout-relay/src/router.rs +++ b/crates/sprout-relay/src/router.rs @@ -51,6 +51,13 @@ pub fn build_router(state: Arc) -> Router { // ── All other routes: 1 MB body limit ──────────────────────────────────── let api_router = Router::new() + // ── Internal routes (not exposed through the public API gateway) ───── + // LiveKit fires this on participant_joined/left (including crashes), + // providing authoritative presence tracking for crash-orphan recovery. + .route( + "/internal/livekit/webhook", + post(api::handle_livekit_webhook), + ) .route("/", get(nip11_or_ws_handler)) .route("/info", get(relay_info_handler)) .route("/.well-known/nostr.json", get(api::nip05::nostr_nip05)) @@ -104,6 +111,8 @@ pub fn build_router(state: Arc) -> Router { "/api/approvals/by-hash/{hash}/deny", post(api::deny_approval_by_hash), ) + // Huddle routes + .route("/api/huddles/{channel_id}/token", post(api::huddle_token)) // Membership routes .route("/api/channels/{channel_id}/members", get(api::list_members)) // Channel detail + metadata routes diff --git a/crates/sprout-relay/src/state.rs b/crates/sprout-relay/src/state.rs index 6db07313290..1c19168e59d 100644 --- a/crates/sprout-relay/src/state.rs +++ b/crates/sprout-relay/src/state.rs @@ -15,6 +15,7 @@ use sprout_audit::AuditService; use sprout_auth::AuthService; use sprout_core::event::StoredEvent; use sprout_db::Db; +use sprout_huddle::HuddleService; use sprout_media::MediaStorage; use sprout_pubsub::PubSubManager; use sprout_search::SearchService; @@ -202,6 +203,10 @@ pub struct AppState { pub search_index_tx: mpsc::Sender, /// Media storage client (S3/MinIO). pub media_storage: Arc, + /// LiveKit huddle service — `None` when LiveKit is not configured. + pub huddle_service: Option>, + /// LiveKit server URL — populated when `huddle_service` is `Some`. + pub livekit_url: Option, /// Set to `true` on SIGTERM — readiness probe returns 503. pub shutting_down: Arc, /// Process start time — used by `/_status` endpoint. @@ -222,6 +227,7 @@ impl AppState { workflow_engine: Arc, relay_keypair: nostr::Keys, media_storage: MediaStorage, + huddle_service: Option<(HuddleService, String)>, ) -> Self { let max_connections = config.max_connections; let max_concurrent_handlers = config.max_concurrent_handlers; @@ -280,6 +286,8 @@ impl AppState { search_index_tx, media_storage: Arc::new(media_storage), + livekit_url: huddle_service.as_ref().map(|(_, url)| url.clone()), + huddle_service: huddle_service.map(|(svc, _)| Arc::new(svc)), shutting_down: Arc::new(AtomicBool::new(false)), started_at: Instant::now(), } diff --git a/crates/sprout-sdk/src/builders.rs b/crates/sprout-sdk/src/builders.rs index a000f6a9c08..a3d5e68e76f 100644 --- a/crates/sprout-sdk/src/builders.rs +++ b/crates/sprout-sdk/src/builders.rs @@ -563,6 +563,116 @@ pub fn build_contact_list( Ok(EventBuilder::new(Kind::Custom(3), "", tags)) } +// ── Huddle shared helper ────────────────────────────────────────────────────── + +/// Shared builder for huddle lifecycle events (kinds 48100–48103). +/// +/// All huddle events share: an `["h", parent_channel_id]` tag, JSON content +/// with `ephemeral_channel_id`, optional extra content fields, and optional +/// p-tags for participant identity. +fn build_huddle_event_sdk( + kind: u16, + parent_channel_id: Uuid, + ephemeral_channel_id: Uuid, + extra_fields: &[(&str, &str)], + participant_pubkey: Option<&str>, +) -> Result { + let mut tags = vec![tag(&["h", &parent_channel_id.to_string()])?]; + if let Some(pk) = participant_pubkey { + tags.push(tag(&["p", pk])?); + } + let mut map = serde_json::Map::new(); + map.insert( + "ephemeral_channel_id".into(), + serde_json::Value::String(ephemeral_channel_id.to_string()), + ); + for (k, v) in extra_fields { + map.insert((*k).into(), serde_json::Value::String(v.to_string())); + } + let content = serde_json::Value::Object(map).to_string(); + Ok(EventBuilder::new(Kind::Custom(kind), content, tags)) +} + +// ── Builder 26: build_huddle_started ───────────────────────────────────────── + +/// Build a huddle-started event (kind 48100). +/// +/// Posted to the parent channel as an advisory UI hint that a huddle has begun. +/// - `parent_channel_id`: the channel the huddle belongs to (h-tag) +/// - `ephemeral_channel_id`: the short-lived channel UUID for the huddle session +/// - `livekit_room`: LiveKit room name participants should join +pub fn build_huddle_started( + parent_channel_id: Uuid, + ephemeral_channel_id: Uuid, + livekit_room: &str, +) -> Result { + build_huddle_event_sdk( + 48100, + parent_channel_id, + ephemeral_channel_id, + &[("livekit_room", livekit_room)], + None, + ) +} + +// ── Builder 27: build_huddle_participant_joined ─────────────────────────────── + +/// Build a huddle-participant-joined event (kind 48101). +/// +/// Posted to the parent channel when a participant enters the huddle. +/// - `parent_channel_id`: the channel the huddle belongs to (h-tag) +/// - `ephemeral_channel_id`: the short-lived channel UUID for the huddle session +/// - `participant_pubkey`: hex pubkey of the joining participant (p-tag) +pub fn build_huddle_participant_joined( + parent_channel_id: Uuid, + ephemeral_channel_id: Uuid, + participant_pubkey: &str, +) -> Result { + build_huddle_event_sdk( + 48101, + parent_channel_id, + ephemeral_channel_id, + &[], + Some(participant_pubkey), + ) +} + +// ── Builder 28: build_huddle_participant_left ───────────────────────────────── + +/// Build a huddle-participant-left event (kind 48102). +/// +/// Posted to the parent channel when a participant exits the huddle. +/// - `parent_channel_id`: the channel the huddle belongs to (h-tag) +/// - `ephemeral_channel_id`: the short-lived channel UUID for the huddle session +/// - `participant_pubkey`: hex pubkey of the departing participant (p-tag) +pub fn build_huddle_participant_left( + parent_channel_id: Uuid, + ephemeral_channel_id: Uuid, + participant_pubkey: &str, +) -> Result { + build_huddle_event_sdk( + 48102, + parent_channel_id, + ephemeral_channel_id, + &[], + Some(participant_pubkey), + ) +} + +// ── Builder 29: build_huddle_ended ─────────────────────────────────────────── + +/// Build a huddle-ended event (kind 48103). +/// +/// Posted to the parent channel when the huddle session concludes. +/// - `parent_channel_id`: the channel the huddle belongs to (h-tag) +/// - `ephemeral_channel_id`: the short-lived channel UUID for the huddle session +pub fn build_huddle_ended( + parent_channel_id: Uuid, + ephemeral_channel_id: Uuid, +) -> Result { + build_huddle_event_sdk(48103, parent_channel_id, ephemeral_channel_id, &[], None) +} + // ── Helper: extract_channel_id ─────────────────────────────────────────────── /// Extract the channel UUID from an event's `h` tag. @@ -1393,4 +1503,99 @@ mod tests { let err = build_contact_list(&contacts).unwrap_err(); assert!(matches!(err, SdkError::InvalidInput(_))); } + + // ── build_huddle_started ────────────────────────────────────────────────── + + #[test] + fn huddle_started_happy_path() { + let parent = uuid(); + let ephemeral = uuid(); + let ev = sign(build_huddle_started(parent, ephemeral, "my-room").unwrap()); + assert_eq!(ev.kind.as_u16(), 48100); + assert!(has_tag(&ev, "h", &parent.to_string())); + let v: serde_json::Value = serde_json::from_str(&ev.content).unwrap(); + assert_eq!(v["ephemeral_channel_id"], ephemeral.to_string()); + assert_eq!(v["livekit_room"], "my-room"); + } + + #[test] + fn huddle_started_h_tag_is_parent_not_ephemeral() { + let parent = uuid(); + let ephemeral = uuid(); + let ev = sign(build_huddle_started(parent, ephemeral, "room").unwrap()); + assert!(has_tag(&ev, "h", &parent.to_string())); + assert!(!has_tag(&ev, "h", &ephemeral.to_string())); + } + + // ── build_huddle_participant_joined ─────────────────────────────────────── + + #[test] + fn huddle_participant_joined_happy_path() { + let parent = uuid(); + let ephemeral = uuid(); + let pubkey = "a".repeat(64); + let ev = sign(build_huddle_participant_joined(parent, ephemeral, &pubkey).unwrap()); + assert_eq!(ev.kind.as_u16(), 48101); + assert!(has_tag(&ev, "h", &parent.to_string())); + assert!(has_tag(&ev, "p", &pubkey)); + let v: serde_json::Value = serde_json::from_str(&ev.content).unwrap(); + assert_eq!(v["ephemeral_channel_id"], ephemeral.to_string()); + } + + #[test] + fn huddle_participant_joined_h_tag_is_parent_not_ephemeral() { + let parent = uuid(); + let ephemeral = uuid(); + let pubkey = "b".repeat(64); + let ev = sign(build_huddle_participant_joined(parent, ephemeral, &pubkey).unwrap()); + assert!(has_tag(&ev, "h", &parent.to_string())); + assert!(!has_tag(&ev, "h", &ephemeral.to_string())); + } + + // ── build_huddle_participant_left ───────────────────────────────────────── + + #[test] + fn huddle_participant_left_happy_path() { + let parent = uuid(); + let ephemeral = uuid(); + let pubkey = "c".repeat(64); + let ev = sign(build_huddle_participant_left(parent, ephemeral, &pubkey).unwrap()); + assert_eq!(ev.kind.as_u16(), 48102); + assert!(has_tag(&ev, "h", &parent.to_string())); + assert!(has_tag(&ev, "p", &pubkey)); + let v: serde_json::Value = serde_json::from_str(&ev.content).unwrap(); + assert_eq!(v["ephemeral_channel_id"], ephemeral.to_string()); + } + + #[test] + fn huddle_participant_left_h_tag_is_parent_not_ephemeral() { + let parent = uuid(); + let ephemeral = uuid(); + let pubkey = "d".repeat(64); + let ev = sign(build_huddle_participant_left(parent, ephemeral, &pubkey).unwrap()); + assert!(has_tag(&ev, "h", &parent.to_string())); + assert!(!has_tag(&ev, "h", &ephemeral.to_string())); + } + + // ── build_huddle_ended ──────────────────────────────────────────────────── + + #[test] + fn huddle_ended_happy_path() { + let parent = uuid(); + let ephemeral = uuid(); + let ev = sign(build_huddle_ended(parent, ephemeral).unwrap()); + assert_eq!(ev.kind.as_u16(), 48103); + assert!(has_tag(&ev, "h", &parent.to_string())); + let v: serde_json::Value = serde_json::from_str(&ev.content).unwrap(); + assert_eq!(v["ephemeral_channel_id"], ephemeral.to_string()); + } + + #[test] + fn huddle_ended_h_tag_is_parent_not_ephemeral() { + let parent = uuid(); + let ephemeral = uuid(); + let ev = sign(build_huddle_ended(parent, ephemeral).unwrap()); + assert!(has_tag(&ev, "h", &parent.to_string())); + assert!(!has_tag(&ev, "h", &ephemeral.to_string())); + } } diff --git a/desktop/package.json b/desktop/package.json index 25848ae1b24..4822b866014 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -46,6 +46,7 @@ "clsx": "^2.1.1", "emoji-mart": "^5.6.0", "jdenticon": "^3.3.0", + "livekit-client": "^2.18.1", "lucide-react": "^0.577.0", "react": "^19.1.0", "react-diff-view": "^3.3.2", diff --git a/desktop/pnpm-lock.yaml b/desktop/pnpm-lock.yaml index 51b5e5d478a..83a60e73436 100644 --- a/desktop/pnpm-lock.yaml +++ b/desktop/pnpm-lock.yaml @@ -74,6 +74,9 @@ importers: jdenticon: specifier: ^3.3.0 version: 3.3.0 + livekit-client: + specifier: ^2.18.1 + version: 2.18.1(@types/dom-mediacapture-record@1.0.22) lucide-react: specifier: ^0.577.0 version: 0.577.0(react@19.2.5) @@ -312,6 +315,9 @@ packages: cpu: [x64] os: [win32] + '@bufbuild/protobuf@1.10.1': + resolution: {integrity: sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ==, tarball: https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-1.10.1.tgz} + '@emoji-mart/data@1.2.1': resolution: {integrity: sha512-no2pQMWiBy6gpBEiqGeU77/bFejDqUTRY7KX+0+iur13op3bqUsXdnwoZs6Xb1zbv0gAj5VvS1PWoUUckSr5Dw==} @@ -508,6 +514,12 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@livekit/mutex@1.1.1': + resolution: {integrity: sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==, tarball: https://registry.npmjs.org/@livekit/mutex/-/mutex-1.1.1.tgz} + + '@livekit/protocol@1.44.0': + resolution: {integrity: sha512-/vfhDUGcUKO8Q43r6i+5FrDhl5oZjm/X3U4x2Iciqvgn5C8qbj+57YPcWSJ1kyIZm5Cm6AV2nAPjMm3ETD/iyg==, tarball: https://registry.npmjs.org/@livekit/protocol/-/protocol-1.44.0.tgz} + '@noble/ciphers@2.1.1': resolution: {integrity: sha512-bysYuiVfhxNJuldNXlFEitTVdNnYUc+XNJZd7Qm2a5j1vZHgY+fazadNFWFaMK/2vye0JVlxV3gHmC0WDfAOQw==} engines: {node: '>= 20.19.0'} @@ -1293,6 +1305,9 @@ packages: '@types/debug@4.1.12': resolution: {integrity: sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==} + '@types/dom-mediacapture-record@1.0.22': + resolution: {integrity: sha512-mUMZLK3NvwRLcAAT9qmcK+9p7tpU2FHdDsntR3YI4+GY88XrgG4XiE7u1Q2LAN2/FZOz/tdMDC3GQCR4T8nFuw==} + '@types/estree-jsx@1.0.5': resolution: {integrity: sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==} @@ -1514,6 +1529,10 @@ packages: estree-util-is-identifier-name@3.0.0: resolution: {integrity: sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==} + events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==, tarball: https://registry.npmjs.org/events/-/events-3.3.0.tgz} + engines: {node: '>=0.8.x'} + extend@3.0.2: resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} @@ -1646,6 +1665,9 @@ packages: resolution: {integrity: sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==} hasBin: true + jose@6.2.2: + resolution: {integrity: sha512-d7kPDd34KO/YnzaDOlikGpOurfF0ByC2sEV4cANCtdqLlTfBlw2p14O/5d/zv40gJPbIQxfES3nSx1/oYNyuZQ==, tarball: https://registry.npmjs.org/jose/-/jose-6.2.2.tgz} + js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -1666,9 +1688,18 @@ packages: lines-and-columns@1.2.4: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} + livekit-client@2.18.1: + resolution: {integrity: sha512-nGjuEEV1mVN01EcAMwGIwG3J1gpBMqwn2V4R6W/8zz9Rah1CaAohIm6AMLG7BdctQpyeh34dfAOLfDodIsWyYA==, tarball: https://registry.npmjs.org/livekit-client/-/livekit-client-2.18.1.tgz} + peerDependencies: + '@types/dom-mediacapture-record': ^1 + lodash@4.17.23: resolution: {integrity: sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==} + loglevel@1.9.2: + resolution: {integrity: sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==, tarball: https://registry.npmjs.org/loglevel/-/loglevel-1.9.2.tgz} + engines: {node: '>= 0.6.0'} + longest-streak@3.1.0: resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==} @@ -2075,9 +2106,19 @@ packages: run-parallel@1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} + rxjs@7.8.2: + resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} + scheduler@0.27.0: resolution: {integrity: sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==} + sdp-transform@2.15.0: + resolution: {integrity: sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw==, tarball: https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.15.0.tgz} + hasBin: true + + sdp@3.2.2: + resolution: {integrity: sha512-xZocWwfyp4hkbN4hLWxMjmv2Q8aNa9MhmOZ7L9aCZPT+dZsgRr6wZRrSYE3HTdyk/2pZKPSgqI7ns7Een1xMSA==, tarball: https://registry.npmjs.org/sdp/-/sdp-3.2.2.tgz} + semver@6.3.1: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true @@ -2173,13 +2214,16 @@ packages: resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} tslib@2.8.1: - resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==, tarball: https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz} tsx@4.21.0: resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==} engines: {node: '>=18.0.0'} hasBin: true + typed-emitter@2.1.0: + resolution: {integrity: sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==, tarball: https://registry.npmjs.org/typed-emitter/-/typed-emitter-2.1.0.tgz} + typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} @@ -2296,6 +2340,10 @@ packages: webpack-virtual-modules@0.6.2: resolution: {integrity: sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ==} + webrtc-adapter@9.0.4: + resolution: {integrity: sha512-5ZZY1+lGq8LEKuDlg9M2RPJHlH3R7OVwyHqMcUsLKCgd9Wvf+QrFTCItkXXYPmrJn8H6gRLXbSgxLLdexiqHxw==, tarball: https://registry.npmjs.org/webrtc-adapter/-/webrtc-adapter-9.0.4.tgz} + engines: {node: '>=6.0.0', npm: '>=3.10.0'} + yallist@3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} @@ -2471,6 +2519,8 @@ snapshots: '@biomejs/cli-win32-x64@2.4.6': optional: true + '@bufbuild/protobuf@1.10.1': {} + '@emoji-mart/data@1.2.1': {} '@emoji-mart/react@1.1.1(emoji-mart@5.6.0)(react@19.2.5)': @@ -2592,6 +2642,12 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@livekit/mutex@1.1.1': {} + + '@livekit/protocol@1.44.0': + dependencies: + '@bufbuild/protobuf': 1.10.1 + '@noble/ciphers@2.1.1': {} '@noble/curves@2.0.1': @@ -3306,6 +3362,8 @@ snapshots: dependencies: '@types/ms': 2.1.0 + '@types/dom-mediacapture-record@1.0.22': {} + '@types/estree-jsx@1.0.5': dependencies: '@types/estree': 1.0.8 @@ -3524,6 +3582,8 @@ snapshots: estree-util-is-identifier-name@3.0.0: {} + events@3.3.0: {} + extend@3.0.2: {} fast-glob@3.3.3: @@ -3659,6 +3719,8 @@ snapshots: jiti@1.21.7: {} + jose@6.2.2: {} + js-tokens@4.0.0: {} jsesc@3.1.0: {} @@ -3669,8 +3731,23 @@ snapshots: lines-and-columns@1.2.4: {} + livekit-client@2.18.1(@types/dom-mediacapture-record@1.0.22): + dependencies: + '@livekit/mutex': 1.1.1 + '@livekit/protocol': 1.44.0 + '@types/dom-mediacapture-record': 1.0.22 + events: 3.3.0 + jose: 6.2.2 + loglevel: 1.9.2 + sdp-transform: 2.15.0 + tslib: 2.8.1 + typed-emitter: 2.1.0 + webrtc-adapter: 9.0.4 + lodash@4.17.23: {} + loglevel@1.9.2: {} + longest-streak@3.1.0: {} loose-envify@1.4.0: @@ -4335,8 +4412,17 @@ snapshots: dependencies: queue-microtask: 1.2.3 + rxjs@7.8.2: + dependencies: + tslib: 2.8.1 + optional: true + scheduler@0.27.0: {} + sdp-transform@2.15.0: {} + + sdp@3.2.2: {} + semver@6.3.1: {} seroval-plugins@1.5.2(seroval@1.5.2): @@ -4459,6 +4545,10 @@ snapshots: optionalDependencies: fsevents: 2.3.3 + typed-emitter@2.1.0: + optionalDependencies: + rxjs: 7.8.2 + typescript@5.9.3: {} undici-types@7.18.2: {} @@ -4561,6 +4651,10 @@ snapshots: webpack-virtual-modules@0.6.2: {} + webrtc-adapter@9.0.4: + dependencies: + sdp: 3.2.2 + yallist@3.1.1: {} yaml@2.8.3: {} diff --git a/desktop/public/worklet.js b/desktop/public/worklet.js new file mode 100644 index 00000000000..60c22cfaa24 --- /dev/null +++ b/desktop/public/worklet.js @@ -0,0 +1,66 @@ +// AudioWorklet processor — runs in the AudioWorklet thread. +// Accumulates PCM Float32 samples and sends 100ms batches to the main thread. +// +// Supports push-to-talk (PTT) gating: when `this.transmitting` is false, +// incoming audio frames are discarded and the buffer is reset. The main thread +// sends `{ type: 'ptt', active: boolean }` messages to toggle transmission. +// Default: transmitting=true (open mic for VAD mode compatibility). +// +// Note: when the worklet is disconnected, any partial buffer (< 4800 samples) +// is silently dropped. The last ~100ms of speech may be lost on huddle leave. +// This is acceptable for voice — losing a partial syllable at disconnect is +// imperceptible compared to the natural end-of-conversation flow. +class SttTapProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.buffer = new Float32Array(4800); // ~100ms at 48kHz + this.offset = 0; + this.transmitting = true; // default: open (VAD mode). PTT mode sets false on init. + + // Listen for PTT state changes from main thread. + // Direction: main→worklet (receives). The worklet→main direction uses + // this.port.postMessage for PCM data — these don't conflict. + this.port.onmessage = (e) => { + if (e.data && e.data.type === "ptt") { + this.transmitting = e.data.active; + } + }; + } + + process(inputs) { + const input = inputs[0]?.[0]; // mono channel + if (!input) return true; + + // PTT gating: discard frames when not transmitting. + // Reset buffer offset so we don't send stale audio when PTT activates. + if (!this.transmitting) { + this.offset = 0; + return true; + } + + // Accumulate samples + const remaining = this.buffer.length - this.offset; + const toCopy = Math.min(input.length, remaining); + this.buffer.set(input.subarray(0, toCopy), this.offset); + this.offset += toCopy; + + // Flush when buffer is full + if (this.offset >= this.buffer.length) { + // Transfer ownership for zero-copy + this.port.postMessage(this.buffer, [this.buffer.buffer]); + this.buffer = new Float32Array(4800); + this.offset = 0; + + // Handle leftover samples + if (toCopy < input.length) { + const leftover = input.subarray(toCopy); + this.buffer.set(leftover); + this.offset = leftover.length; + } + } + + return true; + } +} + +registerProcessor("stt-tap-processor", SttTapProcessor); diff --git a/desktop/scripts/check-file-sizes.mjs b/desktop/scripts/check-file-sizes.mjs index 6b5dd5b4eac..3e70e22ecd8 100644 --- a/desktop/scripts/check-file-sizes.mjs +++ b/desktop/scripts/check-file-sizes.mjs @@ -40,13 +40,14 @@ const overrides = new Map([ ["src/features/settings/ui/SettingsView.tsx", 600], ["src/features/sidebar/ui/AppSidebar.tsx", 860], // channels + forums creation forms + Pulse nav ["src/features/tokens/ui/TokenSettingsCard.tsx", 800], - ["src/shared/api/relayClientSession.ts", 792], // durable websocket session manager with reconnect/replay/recovery state + sendTypingIndicator + fetchChannelHistoryBefore; temporarily raised to cover subscription replay hardening on human-reply + ["src/shared/api/relayClientSession.ts", 835], // durable websocket session manager with reconnect/replay/recovery state + sendTypingIndicator + fetchChannelHistoryBefore + subscribeToChannelLive (huddle TTS) + subscribeToHuddleEvents (huddle indicator) ["src/shared/api/tauri.ts", 1100], // remote agent provider API bindings + canvas API functions - ["src-tauri/src/lib.rs", 570], // sprout-media:// proxy + Range headers + Sprout nest init (ensure_nest) in setup() + persona pack commands + ["src-tauri/src/lib.rs", 710], // sprout-media:// proxy + Range headers + Sprout nest init (ensure_nest) in setup() + huddle command registration + PTT global shortcut handler + persona pack commands + app_handle storage for event emission ["src-tauri/src/commands/media.rs", 720], // ffmpeg video transcode + poster frame extraction + run_ffmpeg_with_timeout (find_ffmpeg, is_video_file, transcode_to_mp4, extract_poster_frame, transcode_and_extract_poster) + spawn_blocking wrappers + tests ["src-tauri/src/commands/agents.rs", 880], // remote agent lifecycle routing (local + provider branches) + scope enforcement + persona pack metadata wiring + mcp_toolsets field ["src-tauri/src/managed_agents/runtime.rs", 690], // KNOWN_AGENT_BINARIES const + process_belongs_to_us FFI (macOS proc_name + Linux /proc/comm) + terminate_process + start/stop/sync lifecycle + pack persona live-read ["src-tauri/src/managed_agents/backend.rs", 530], // provider IPC, validation, discovery, binary resolution + tests + ["src/features/huddle/HuddleContext.tsx", 650], // huddle lifecycle context + joinHuddle + connectAndSetupMedia shared helper + activeSpeakers/isReconnecting state + PTT (reusable AudioContext) + TTS subscription + mic level analyser (10fps throttle) + agent pubkey refresh ["src/features/agents/hooks.ts", 540], // agent query/mutation surface now includes built-in persona library activation + useUpdateManagedAgentMutation ["src/features/agents/ui/AgentsView.tsx", 880], // remote agent lifecycle controls + persona/team management + persona import-update dialog wiring + built-in catalog/library state orchestration ["src/features/agents/ui/ManagedAgentRow.tsx", 530], // EditAgentDialog integration + provider/local branching @@ -56,6 +57,12 @@ const overrides = new Map([ ["src/features/agents/ui/CreateAgentDialog.tsx", 685], // provider selector + config form + schema-typed config coercion + required field validation + locked scopes ["src/features/channels/ui/AddChannelBotDialog.tsx", 640], // provider mode: Run on selector, trust warning, probe effect, single-agent enforcement, provider warnings display ["src/shared/api/types.ts", 550], // persona provider/model fields + forum types + workflow type re-exports + ephemeral channel TTL fields + mcpToolsets + sourcePack + UpdateManagedAgentInput edit fields + ["src-tauri/src/events.rs", 555], // event builders + build_huddle_guidelines (kind:48106) + post_event_raw transport helper + participant p-tag on join/leave + ["src-tauri/src/huddle/kokoro.rs", 980], // Kokoro ONNX TTS engine + three-tier G2P + ARPAbet→IPA + CoreML + synth_chunk() public API + style validation + hyphenated compound splitting + 23 unit tests + ["src-tauri/src/huddle/mod.rs", 1020], // huddle state machine + Tauri commands + sync protocol doc; state/relay/pipeline extracted + emit_huddle_state_changed wiring + ["src-tauri/src/huddle/models.rs", 850], // model download manager for Moonshine STT + Kokoro TTS with streaming downloads + SHA-256 verification + Rust-native tar extraction + version manifest + atomic swap + hot-start signaling + ["src-tauri/src/huddle/stt.rs", 580], // STT pipeline + PTT edge-detection flush + PTT gating (is_speech AND ptt_active) + barge-in for VAD mode + rubato resampler + earshot VAD + sherpa-onnx transcription + ["src-tauri/src/huddle/preprocessing.rs", 670], // TTS text preprocessing pipeline + unified split_sentences + int_to_words 0-999999 + URL trailing punctuation preservation + 23 unit tests ]); async function walkFiles(directory) { diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 67120130159..3daefb6b1ba 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -53,6 +53,28 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "alsa" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812947049edcd670a82cd5c73c3661d2e58468577ba8489de58e1a73c04cbd5d" +dependencies = [ + "alsa-sys", + "bitflags 2.11.1", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad7569085a265dd3f607ebecce7458eaab2132a84393534c95b18dcbc3f31e04" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -250,7 +272,44 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84790c55b5704b0d35130bf16a4ce22a8e70eb0ea773522557524d9a4852663d" dependencies = [ "nix", - "rand 0.9.2", + "rand 0.9.4", +] + +[[package]] +name = "audio-core" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ebbf82d06013f4c41fe71303feb980cddd78496d904d06be627972de51a24" + +[[package]] +name = "audioadapter" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91f87b70b051c5866680ad79f6743a42ccab264c009d1a71f4d33a3872ae60c8" +dependencies = [ + "audio-core", + "num-traits", +] + +[[package]] +name = "audioadapter-buffers" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9097d67933fb083d382ce980430afdb758aada60846010aee6be068c06cef0ca" +dependencies = [ + "audioadapter", + "audioadapter-sample", + "num-traits", +] + +[[package]] +name = "audioadapter-sample" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ab94f2bc04a14e1f49ee5f222f66460e8a1b51627bdfedf34eed394d747938" +dependencies = [ + "audio-core", + "num-traits", ] [[package]] @@ -403,9 +462,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" dependencies = [ "serde_core", ] @@ -507,6 +566,16 @@ dependencies = [ "serde", ] +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + [[package]] name = "bzip2" version = "0.5.2" @@ -532,7 +601,7 @@ version = "0.18.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ca26ef0159422fb77631dc9d17b102f253b876fe1586b03b803e63a309b4ee2" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cairo-sys-rs", "glib", "libc", @@ -604,9 +673,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.59" +version = "1.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283" +checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" dependencies = [ "find-msvc-tools", "jobserver", @@ -664,6 +733,17 @@ dependencies = [ "cpufeatures 0.2.17", ] +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chacha20poly1305" version = "0.10.1" @@ -671,7 +751,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ "aead", - "chacha20", + "chacha20 0.9.1", "cipher", "poly1305", "zeroize", @@ -790,10 +870,10 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "064badf302c3194842cf2c5d61f56cc88e54a759313879cdf03abdd27d0c3b97" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "core-foundation 0.10.1", "core-graphics-types", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -803,11 +883,55 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "core-foundation 0.10.1", "libc", ] +[[package]] +name = "coreaudio-rs" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16dd574a72a021b90c7656c474ea31d11a2f0366a8eff574186e761e0b9e3586" +dependencies = [ + "bitflags 2.11.1", + "libc", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "cpal" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8942da362c0f0d895d7cac616263f2f9424edc5687364dfd1d25ef7eba506d7" +dependencies = [ + "alsa", + "coreaudio-rs", + "dasp_sample", + "jni", + "js-sys", + "libc", + "mach2", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "objc2", + "objc2-audio-toolbox", + "objc2-avf-audio", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows 0.62.2", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -859,6 +983,25 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -969,6 +1112,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-encoding" version = "2.10.0" @@ -981,6 +1130,16 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2" +[[package]] +name = "der" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fd89660b2dc699704064e59e9dba0147b903e85319429e131620d022be411b" +dependencies = [ + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.5.8" @@ -1085,7 +1244,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "block2", "libc", "objc2", @@ -1176,6 +1335,21 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "earshot" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d393a8f23619412e0502df1b94cd148e34855a50a4143d365cc6336cc338b4f4" +dependencies = [ + "libm", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "embed-resource" version = "3.0.8" @@ -1280,6 +1454,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fastrand" version = "2.4.1" @@ -1350,6 +1530,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared 0.1.1", +] + [[package]] name = "foreign-types" version = "0.5.0" @@ -1357,7 +1546,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" dependencies = [ "foreign-types-macros", - "foreign-types-shared", + "foreign-types-shared 0.3.1", ] [[package]] @@ -1371,6 +1560,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "foreign-types-shared" version = "0.3.1" @@ -1604,6 +1799,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "gethostname" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" +dependencies = [ + "rustix", + "windows-link 0.2.1", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -1651,6 +1856,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -1693,7 +1899,7 @@ version = "0.18.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "233daaf6e83ae6a12a52055f568f9d7cf4671dabb78ff9560ab6da230ce00ee5" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "futures-channel", "futures-core", "futures-executor", @@ -1740,6 +1946,24 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "global-hotkey" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9247516746aa8e53411a0db9b62b0e24efbcf6a76e0ba73e5a91b512ddabed7" +dependencies = [ + "crossbeam-channel", + "keyboard-types", + "objc2", + "objc2-app-kit", + "once_cell", + "serde", + "thiserror 2.0.18", + "windows-sys 0.59.0", + "x11rb", + "xkeysym", +] + [[package]] name = "gobject-sys" version = "0.18.0" @@ -1815,7 +2039,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.13.1", + "indexmap 2.14.0", "slab", "tokio", "tokio-util", @@ -1839,9 +2063,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" [[package]] name = "heck" @@ -1891,6 +2115,12 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "hmac-sha256" +version = "1.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f" + [[package]] name = "html5ever" version = "0.29.1" @@ -1984,15 +2214,14 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.7" +version = "0.27.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "c2b52f86d1d4bc0d6b4e6826d960b1b333217e07d36b882dca570a5e1c48895b" dependencies = [ "http", "hyper", "hyper-util", "rustls", - "rustls-pki-types", "tokio", "tokio-rustls", "tower-service", @@ -2186,12 +2415,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "serde", "serde_core", ] @@ -2347,9 +2576,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.94" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" dependencies = [ "cfg-if", "futures-util", @@ -2385,7 +2614,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b750dcadc39a09dbadd74e118f6dd6598df77fa01df0cfcdc52c28dece74528a" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "serde", "unicode-segmentation", ] @@ -2398,10 +2627,16 @@ checksum = "02cb977175687f33fa4afa0c95c112b987ea1443e5a51c8f8ff27dc618270cc2" dependencies = [ "cssparser 0.29.6", "html5ever 0.29.1", - "indexmap 2.13.1", + "indexmap 2.14.0", "selectors 0.24.0", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "leb128fmt" version = "0.1.0" @@ -2434,9 +2669,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.184" +version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" +checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" [[package]] name = "libloading" @@ -2448,16 +2683,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "libredox" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" +checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "libc", "plain", - "redox_syscall 0.7.3", + "redox_syscall 0.7.4", ] [[package]] @@ -2503,6 +2744,12 @@ dependencies = [ "crc", ] +[[package]] +name = "lzma-rust2" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1670343e58806300d87950e3401e820b519b9384281bbabfb15e3636689ffd69" + [[package]] name = "lzma-sys" version = "0.1.20" @@ -2532,6 +2779,15 @@ dependencies = [ "time", ] +[[package]] +name = "mach2" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1b95cd5421ec55b445b5ae102f5ea0e768de1f82bd3001e11f426c269c3aea" +dependencies = [ + "libc", +] + [[package]] name = "markup5ever" version = "0.14.1" @@ -2574,6 +2830,16 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "memchr" version = "2.8.0" @@ -2643,13 +2909,46 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", + "rayon", +] + [[package]] name = "ndk" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "jni-sys 0.3.1", "log", "ndk-sys", @@ -2697,7 +2996,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cfg-if", "cfg_aliases", "libc", @@ -2722,7 +3021,7 @@ dependencies = [ "bip39", "bitcoin", "cbc", - "chacha20", + "chacha20 0.9.1", "chacha20poly1305", "getrandom 0.2.17", "instant", @@ -2753,7 +3052,7 @@ dependencies = [ "bip39", "bitcoin", "cbc", - "chacha20", + "chacha20 0.9.1", "chacha20poly1305", "getrandom 0.2.17", "instant", @@ -2781,12 +3080,62 @@ dependencies = [ "zbus", ] +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2794,6 +3143,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -2834,21 +3184,71 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "block2", "objc2", "objc2-core-foundation", "objc2-foundation", ] +[[package]] +name = "objc2-audio-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" +dependencies = [ + "bitflags 2.11.1", + "libc", + "objc2", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-avf-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" +dependencies = [ + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.11.1", + "objc2", +] + [[package]] name = "objc2-core-foundation" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", + "block2", "dispatch2", + "libc", "objc2", ] @@ -2858,7 +3258,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "dispatch2", "objc2", "objc2-core-foundation", @@ -2886,7 +3286,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "block2", "libc", "objc2", @@ -2899,7 +3299,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "objc2", "objc2-core-foundation", ] @@ -2910,7 +3310,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f112d1746737b0da274ef79a23aac283376f335f4095a083a267a082f21db0c0" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "objc2", "objc2-app-kit", "objc2-foundation", @@ -2922,7 +3322,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96c1358452b371bf9f104e21ec536d37a650eb10f7ee379fff67d2e08d537f1f" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "objc2", "objc2-core-foundation", "objc2-foundation", @@ -2934,7 +3334,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d87d638e33c06f577498cbcc50491496a3ed4246998a7fbba7ccb98b1e7eab22" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "objc2", "objc2-core-foundation", "objc2-foundation", @@ -2946,7 +3346,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2e5aaab980c433cf470df9d7af96a7b46a9d892d521a2cbbb2f8a4c16751e7f" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "block2", "objc2", "objc2-app-kit", @@ -2978,12 +3378,50 @@ dependencies = [ "pathdiff", ] +[[package]] +name = "openssl" +version = "0.10.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfe4646e360ec77dff7dde40ed3d6c5fee52d156ef4a62f53973d38294dad87f" +dependencies = [ + "bitflags 2.11.1", + "cfg-if", + "foreign-types 0.3.2", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-sys" +version = "0.9.113" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad2f2c0eba47118757e4c6d2bff2838f3e0523380021356e7875e858372ce644" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -3000,6 +3438,30 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "ort" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" +dependencies = [ + "ndarray", + "ort-sys", + "smallvec", + "tracing", + "ureq 3.3.0", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" +dependencies = [ + "hmac-sha256", + "lzma-rust2", + "ureq 3.3.0", +] + [[package]] name = "osakit" version = "0.3.1" @@ -3095,6 +3557,15 @@ dependencies = [ "hmac", ] +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -3307,9 +3778,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "plain" @@ -3324,7 +3795,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "740ebea15c5d1428f910cd1a5f52cebf8d25006245ed8ade92702f4943d91e07" dependencies = [ "base64 0.22.1", - "indexmap 2.13.1", + "indexmap 2.14.0", "quick-xml 0.38.4", "serde", "time", @@ -3349,7 +3820,7 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "crc32fast", "fdeflate", "flate2", @@ -3381,6 +3852,21 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -3421,6 +3907,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -3537,7 +4032,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -3611,14 +4106,25 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20 0.10.0", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.2.2" @@ -3676,6 +4182,22 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_distr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d431c2703ccf129de4d45253c03f49ebb22b97d6ad79ee3ecfc7e3f4862c1d8" +dependencies = [ + "num-traits", + "rand 0.10.1", +] + [[package]] name = "rand_hc" version = "0.2.0" @@ -3700,22 +4222,57 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "realfft" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677" +dependencies = [ + "rustfft", +] + [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", ] [[package]] name = "redox_syscall" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" +checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", ] [[package]] @@ -3885,17 +4442,55 @@ dependencies = [ ] [[package]] -name = "ring" -version = "0.17.14" +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rodio" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a536bb79db59098ef71a4dd4246c02eb87b316deceb1b68e0cde7167ec01eb" +dependencies = [ + "cpal", + "dasp_sample", + "num-rational", + "rand 0.10.1", + "rand_distr", + "rtrb", + "symphonia", + "thiserror 2.0.18", +] + +[[package]] +name = "rtrb" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +checksum = "7204ed6420f698836b76d4d5c2ec5dec7585fd5c3a788fd1cde855d1de598239" + +[[package]] +name = "rubato" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce96ead1a91f7895704a9f08ea5947dfc8bd7c1f2936a22295b655ec67e5c6ef" dependencies = [ - "cc", - "cfg-if", - "getrandom 0.2.17", - "libc", - "untrusted", - "windows-sys 0.52.0", + "audioadapter", + "audioadapter-buffers", + "num-complex", + "num-integer", + "num-traits", + "realfft", + "visibility", + "windowfunctions", ] [[package]] @@ -3913,13 +4508,27 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rustix" version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys", @@ -3928,11 +4537,12 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.37" +version = "0.23.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" dependencies = [ "aws-lc-rs", + "log", "once_cell", "ring", "rustls-pki-types", @@ -3992,9 +4602,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" dependencies = [ "aws-lc-rs", "ring", @@ -4137,7 +4747,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -4178,7 +4788,7 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5d9c0c92a92d33f08817311cf3f2c29a3538a8240e94a6a3c622ce652d7e00c" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cssparser 0.36.0", "derive_more 2.1.1", "log", @@ -4260,7 +4870,7 @@ version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ - "indexmap 2.13.1", + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -4319,7 +4929,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.13.1", + "indexmap 2.14.0", "schemars 0.9.0", "schemars 1.2.1", "serde_core", @@ -4346,7 +4956,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.13.1", + "indexmap 2.14.0", "itoa", "ryu", "serde", @@ -4427,6 +5037,28 @@ dependencies = [ "digest 0.11.2", ] +[[package]] +name = "sherpa-onnx" +version = "1.12.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72788b30c5d1ec4a38fbedd57b5c82c429d187b14ccb7e029dc18e5ec7aec69c" +dependencies = [ + "serde", + "serde_json", + "sherpa-onnx-sys", +] + +[[package]] +name = "sherpa-onnx-sys" +version = "1.12.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b71e9586b28493902080ed89e9b4457d2c684521d4c10df62c381b6fdcb51" +dependencies = [ + "bzip2 0.4.4", + "tar", + "ureq 2.12.1", +] + [[package]] name = "shlex" version = "1.3.0" @@ -4483,6 +5115,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + [[package]] name = "softbuffer" version = "0.4.8" @@ -4536,23 +5179,34 @@ name = "sprout" version = "0.1.0" dependencies = [ "atomic-write-file", + "audioadapter-buffers", "base64 0.22.1", + "bzip2 0.5.2", "chrono", "dirs", + "earshot", "hex", "infer", "libc", + "ndarray", "nostr 0.37.0", + "ort", "png 0.18.1", + "regex", "reqwest 0.13.2", + "rodio", + "rubato", "serde", "serde_json", "sha2 0.11.0", + "sherpa-onnx", "sprout-core", "sprout-persona", + "tar", "tauri", "tauri-build", "tauri-plugin-dialog", + "tauri-plugin-global-shortcut", "tauri-plugin-notification", "tauri-plugin-opener", "tauri-plugin-process", @@ -4596,6 +5250,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "string_cache" version = "0.8.9" @@ -4668,6 +5328,153 @@ dependencies = [ "serde_json", ] +[[package]] +name = "symphonia" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-pcm", + "symphonia-codec-vorbis", + "symphonia-core", + "symphonia-format-isomp4", + "symphonia-format-ogg", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91565e180aea25d9b80a910c546802526ffd0072d0b8974e3ebe59b686c9976" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4872dd6bb56bf5eac799e3e957aa1981086c3e613b27e0ac23b176054f7c57ed" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c263845aa86881416849c1729a54c7f55164f8b96111dba59de46849e73a790" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e89d716c01541ad3ebe7c91ce4c8d38a7cf266a3f7b2f090b108fb0cb031d95" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-vorbis" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f025837c309cd69ffef572750b4a2257b59552c5399a5e49707cc5b1b85d1c73" +dependencies = [ + "log", + "symphonia-core", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-core" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea00cc4f79b7f6bb7ff87eddc065a1066f3a43fe1875979056672c9ef948c2af" +dependencies = [ + "arrayvec", + "bitflags 1.3.2", + "bytemuck", + "lazy_static", + "log", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "243739585d11f81daf8dac8d9f3d18cc7898f6c09a259675fc364b382c30e0a5" +dependencies = [ + "encoding_rs", + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b4955c67c1ed3aa8ae8428d04ca8397fbef6a19b2b051e73b5da8b1435639cb" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d7c3df0e7d94efb68401d81906eae73c02b40d5ec1a141962c592d0f11a96f" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36306ff42b9ffe6e5afc99d49e121e0bd62fe79b9db7b9681d48e29fa19e6b16" +dependencies = [ + "encoding_rs", + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-utils-xiph" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27c85ab799a338446b68eec77abf42e1a6f1bb490656e121c6e27bfbab9f16" +dependencies = [ + "symphonia-core", + "symphonia-metadata", +] + [[package]] name = "syn" version = "1.0.109" @@ -4716,7 +5523,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -4750,7 +5557,7 @@ version = "0.34.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9103edf55f2da3c82aea4c7fab7c4241032bfeea0e71fa557d98e00e7ce7cc20" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "block2", "core-foundation 0.10.1", "core-graphics", @@ -4776,7 +5583,7 @@ dependencies = [ "tao-macros", "unicode-segmentation", "url", - "windows", + "windows 0.61.3", "windows-core 0.61.2", "windows-version", "x11-dl", @@ -4858,7 +5665,7 @@ dependencies = [ "webkit2gtk", "webview2-com", "window-vibrancy", - "windows", + "windows 0.61.3", ] [[package]] @@ -4983,6 +5790,21 @@ dependencies = [ "url", ] +[[package]] +name = "tauri-plugin-global-shortcut" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "424af23c7e88d05e4a1a6fc2c7be077912f8c76bd7900fd50aa2b7cbf5a2c405" +dependencies = [ + "global-hotkey", + "log", + "serde", + "serde_json", + "tauri", + "tauri-plugin", + "thiserror 2.0.18", +] + [[package]] name = "tauri-plugin-notification" version = "2.3.3" @@ -4991,7 +5813,7 @@ checksum = "01fc2c5ff41105bd1f7242d8201fdf3efd70749b82fa013a17f2126357d194cc" dependencies = [ "log", "notify-rust", - "rand 0.9.2", + "rand 0.9.4", "serde", "serde_json", "serde_repr", @@ -5020,7 +5842,7 @@ dependencies = [ "tauri-plugin", "thiserror 2.0.18", "url", - "windows", + "windows 0.61.3", "zbus", ] @@ -5076,7 +5898,7 @@ dependencies = [ "futures-util", "http", "log", - "rand 0.9.2", + "rand 0.9.4", "rustls", "serde", "serde_json", @@ -5093,7 +5915,7 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73736611e14142408d15353e21e3cca2f12a3cfb523ad0ce85999b6d2ef1a704" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "log", "serde", "serde_json", @@ -5124,7 +5946,7 @@ dependencies = [ "url", "webkit2gtk", "webview2-com", - "windows", + "windows 0.61.3", ] [[package]] @@ -5149,7 +5971,7 @@ dependencies = [ "url", "webkit2gtk", "webview2-com", - "windows", + "windows 0.61.3", "wry", ] @@ -5210,7 +6032,7 @@ checksum = "0b1e66e07de489fe43a46678dd0b8df65e0c973909df1b60ba33874e297ba9b9" dependencies = [ "quick-xml 0.37.5", "thiserror 2.0.18", - "windows", + "windows 0.61.3", "windows-version", ] @@ -5415,7 +6237,7 @@ version = "0.9.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ - "indexmap 2.13.1", + "indexmap 2.14.0", "serde_core", "serde_spanned 1.1.1", "toml_datetime 0.7.5+spec-1.1.0", @@ -5457,7 +6279,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.13.1", + "indexmap 2.14.0", "toml_datetime 0.6.3", "winnow 0.5.40", ] @@ -5468,7 +6290,7 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338" dependencies = [ - "indexmap 2.13.1", + "indexmap 2.14.0", "serde", "serde_spanned 0.6.9", "toml_datetime 0.6.3", @@ -5481,7 +6303,7 @@ version = "0.25.11+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" dependencies = [ - "indexmap 2.13.1", + "indexmap 2.14.0", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "winnow 1.0.1", @@ -5523,7 +6345,7 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "bytes", "futures-util", "http", @@ -5578,6 +6400,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "tray-icon" version = "0.21.3" @@ -5617,7 +6449,7 @@ dependencies = [ "http", "httparse", "log", - "rand 0.9.2", + "rand 0.9.4", "rustls", "rustls-pki-types", "sha1", @@ -5738,6 +6570,52 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64 0.22.1", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "url", + "webpki-roots 0.26.11", +] + +[[package]] +name = "ureq" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +dependencies = [ + "base64 0.22.1", + "der", + "log", + "native-tls", + "percent-encoding", + "rustls-pki-types", + "socks", + "ureq-proto", + "utf8-zero", + "webpki-root-certs", +] + +[[package]] +name = "ureq-proto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +dependencies = [ + "base64 0.22.1", + "http", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -5769,6 +6647,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -5787,6 +6671,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version-compare" version = "0.2.1" @@ -5799,6 +6689,17 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "visibility" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "vswhom" version = "0.1.0" @@ -5870,9 +6771,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.117" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" dependencies = [ "cfg-if", "once_cell", @@ -5883,9 +6784,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.67" +version = "0.4.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e" +checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" dependencies = [ "js-sys", "wasm-bindgen", @@ -5893,9 +6794,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.117" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5903,9 +6804,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.117" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" dependencies = [ "bumpalo", "proc-macro2", @@ -5916,9 +6817,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.117" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" dependencies = [ "unicode-ident", ] @@ -5940,7 +6841,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap 2.13.1", + "indexmap 2.14.0", "wasm-encoder", "wasmparser", ] @@ -5964,17 +6865,17 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "hashbrown 0.15.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "semver", ] [[package]] name = "web-sys" -version = "0.3.94" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd70027e39b12f0849461e08ffc50b9cd7688d942c1c8e3c7b22273236b4dd0a" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" dependencies = [ "js-sys", "wasm-bindgen", @@ -6081,7 +6982,7 @@ checksum = "7130243a7a5b33c54a444e54842e6a9e133de08b5ad7b5861cd8ed9a6a5bc96a" dependencies = [ "webview2-com-macros", "webview2-com-sys", - "windows", + "windows 0.61.3", "windows-core 0.61.2", "windows-implement", "windows-interface", @@ -6105,7 +7006,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "381336cfffd772377d291702245447a5251a2ffa5bad679c99e61bc48bacbf9c" dependencies = [ "thiserror 2.0.18", - "windows", + "windows 0.61.3", "windows-core 0.61.2", ] @@ -6155,17 +7056,38 @@ dependencies = [ "windows-version", ] +[[package]] +name = "windowfunctions" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90628d739333b7c5d2ee0b70210b97b8cddc38440c682c96fd9e2c24c2db5f3a" +dependencies = [ + "num-traits", +] + [[package]] name = "windows" version = "0.61.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ - "windows-collections", + "windows-collections 0.2.0", "windows-core 0.61.2", - "windows-future", + "windows-future 0.2.1", "windows-link 0.1.3", - "windows-numerics", + "windows-numerics 0.2.0", +] + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections 0.3.2", + "windows-core 0.62.2", + "windows-future 0.3.2", + "windows-numerics 0.3.1", ] [[package]] @@ -6177,6 +7099,15 @@ dependencies = [ "windows-core 0.61.2", ] +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core 0.62.2", +] + [[package]] name = "windows-core" version = "0.61.2" @@ -6211,7 +7142,18 @@ checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ "windows-core 0.61.2", "windows-link 0.1.3", - "windows-threading", + "windows-threading 0.1.0", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core 0.62.2", + "windows-link 0.2.1", + "windows-threading 0.2.1", ] [[package]] @@ -6258,6 +7200,16 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core 0.62.2", + "windows-link 0.2.1", +] + [[package]] name = "windows-registry" version = "0.6.1" @@ -6407,6 +7359,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-version" version = "0.1.7" @@ -6619,7 +7580,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck 0.5.0", - "indexmap 2.13.1", + "indexmap 2.14.0", "prettyplease", "syn 2.0.117", "wasm-metadata", @@ -6649,8 +7610,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.11.0", - "indexmap 2.13.1", + "bitflags 2.11.1", + "indexmap 2.14.0", "log", "serde", "serde_derive", @@ -6669,7 +7630,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap 2.13.1", + "indexmap 2.14.0", "log", "semver", "serde", @@ -6723,7 +7684,7 @@ dependencies = [ "webkit2gtk", "webkit2gtk-sys", "webview2-com", - "windows", + "windows 0.61.3", "windows-core 0.61.2", "windows-version", "x11-dl", @@ -6750,6 +7711,23 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "x11rb" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" +dependencies = [ + "gethostname", + "rustix", + "x11rb-protocol", +] + +[[package]] +name = "x11rb-protocol" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" + [[package]] name = "xattr" version = "1.6.1" @@ -6760,6 +7738,12 @@ dependencies = [ "rustix", ] +[[package]] +name = "xkeysym" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" + [[package]] name = "xz2" version = "0.1.7" @@ -6955,7 +7939,7 @@ checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" dependencies = [ "aes", "arbitrary", - "bzip2", + "bzip2 0.5.2", "constant_time_eq", "crc32fast", "crossbeam-utils", @@ -6964,7 +7948,7 @@ dependencies = [ "flate2", "getrandom 0.3.4", "hmac", - "indexmap 2.13.1", + "indexmap 2.14.0", "lzma-rs", "memchr", "pbkdf2", @@ -6985,7 +7969,7 @@ checksum = "caa8cd6af31c3b31c6631b8f483848b91589021b28fffe50adada48d4f4d2ed1" dependencies = [ "arbitrary", "crc32fast", - "indexmap 2.13.1", + "indexmap 2.14.0", "memchr", ] diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index 77427ba119d..31f1b83e625 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -35,7 +35,7 @@ tauri-plugin-updater = "2" tauri-plugin-process = "2" infer = "0.19" hex = "0.4" -tokio = { version = "1", features = ["fs", "sync"] } +tokio = { version = "1", features = ["fs", "sync", "rt"] } serde = { version = "1", features = ["derive"] } serde_json = "1" nostr = "0.37" @@ -45,11 +45,22 @@ sprout-core = { path = "../../crates/sprout-core" } sprout-persona = { path = "../../crates/sprout-persona" } base64 = "0.22" sha2 = "0.11" +tar = "0.4" +bzip2 = "0.5" chrono = { version = "0.4", features = ["serde"] } +tauri-plugin-global-shortcut = "2" tauri-plugin-notification = "2.3.3" uuid = { version = "1", features = ["v4"] } png = "0.18" zip = "2" +sherpa-onnx = "1.12" +ort = { version = "=2.0.0-rc.12", default-features = false, features = ["std", "ndarray", "tracing", "download-binaries", "tls-native", "copy-dylibs", "api-23", "coreml"] } +ndarray = { version = "0.17", features = ["rayon"] } +regex = "1" +rodio = "0.22" +earshot = "1.0" +rubato = "2.0" +audioadapter-buffers = "3.0" tempfile = "3" [dev-dependencies] diff --git a/desktop/src-tauri/Entitlements.plist b/desktop/src-tauri/Entitlements.plist new file mode 100644 index 00000000000..d459cb2ca5f --- /dev/null +++ b/desktop/src-tauri/Entitlements.plist @@ -0,0 +1,8 @@ + + + + + com.apple.security.device.audio-input + + + diff --git a/desktop/src-tauri/Info.plist b/desktop/src-tauri/Info.plist index 41531bc3ade..b46d11a2632 100644 --- a/desktop/src-tauri/Info.plist +++ b/desktop/src-tauri/Info.plist @@ -6,5 +6,7 @@ Sprout CFBundleName Sprout + NSMicrophoneUsageDescription + Sprout needs microphone access for voice huddles. diff --git a/desktop/src-tauri/capabilities/default.json b/desktop/src-tauri/capabilities/default.json index 4e3d55d987f..fc4eae76c71 100644 --- a/desktop/src-tauri/capabilities/default.json +++ b/desktop/src-tauri/capabilities/default.json @@ -21,6 +21,9 @@ "updater:default", "updater:allow-check", "updater:allow-download-and-install", - "process:allow-restart" + "process:allow-restart", + "global-shortcut:allow-register", + "global-shortcut:allow-unregister", + "global-shortcut:allow-is-registered" ] } diff --git a/desktop/src-tauri/src/app_state.rs b/desktop/src-tauri/src/app_state.rs index 7af53fd053c..a1d8ca6ddeb 100644 --- a/desktop/src-tauri/src/app_state.rs +++ b/desktop/src-tauri/src/app_state.rs @@ -3,6 +3,7 @@ use std::{collections::HashMap, io::Write, sync::Mutex}; use nostr::{Keys, ToBech32}; use tauri::{AppHandle, Manager}; +use crate::huddle::HuddleState; use crate::managed_agents::ManagedAgentProcess; pub struct AppState { @@ -12,6 +13,13 @@ pub struct AppState { pub session_token: Mutex>, pub managed_agents_store_lock: Mutex<()>, pub managed_agent_processes: Mutex>, + pub huddle_state: Mutex, + /// Tauri app handle — stored after setup so huddle commands can emit + /// `huddle-state-changed` events without needing the handle threaded + /// through every call site. + /// + /// Set once during `setup()` in `lib.rs`; never cleared. + pub app_handle: Mutex>, } pub fn build_app_state() -> AppState { @@ -55,6 +63,37 @@ pub fn build_app_state() -> AppState { session_token: Mutex::new(None), managed_agents_store_lock: Mutex::new(()), managed_agent_processes: Mutex::new(HashMap::new()), + huddle_state: Mutex::new(HuddleState::default()), + app_handle: Mutex::new(None), + } +} + +impl AppState { + /// Lock the huddle state mutex, converting a poisoned-lock error to a String. + /// + /// Convenience wrapper — replaces 15+ instances of + /// `state.huddle_state.lock().map_err(|e| e.to_string())?` throughout the + /// huddle module. + pub fn huddle(&self) -> Result, String> { + self.huddle_state.lock().map_err(|e| e.to_string()) + } + + /// Emit the current huddle state to the frontend via Tauri event. + /// + /// Acquires both locks (app_handle + huddle_state), clones a snapshot, + /// releases both, then emits. Best-effort — no-op if either lock is + /// poisoned or the app_handle hasn't been set yet. + pub fn emit_huddle_state_changed(&self) { + let app = match self.app_handle.lock() { + Ok(guard) => guard.clone(), + Err(_) => return, + }; + let Some(app) = app else { return }; + let snapshot = match self.huddle_state.lock() { + Ok(hs) => hs.clone(), + Err(_) => return, + }; + crate::huddle::state::emit_huddle_state(&app, &snapshot); } } diff --git a/desktop/src-tauri/src/events.rs b/desktop/src-tauri/src/events.rs index efa18ebaa90..4c4dedf1ff0 100644 --- a/desktop/src-tauri/src/events.rs +++ b/desktop/src-tauri/src/events.rs @@ -357,6 +357,114 @@ pub fn build_profile( Ok(EventBuilder::new(Kind::Custom(0), content)) } +// ── Huddles ────────────────────────────────────────────────────────────────── + +/// Validate that a string is a valid UUID (defense-in-depth for `&str` channel IDs). +fn validate_channel_id(id: &str) -> Result<(), String> { + uuid::Uuid::parse_str(id).map_err(|_| format!("invalid channel UUID: {id}"))?; + Ok(()) +} + +/// Shared builder for huddle lifecycle events (kinds 48100–48103). +/// All huddle events share: validate two channel IDs, JSON content with +/// `ephemeral_channel_id`, an `["h", parent_channel_id]` tag, and an +/// optional `["p", participant_pubkey]` tag for join/leave identity. +fn build_huddle_event( + kind: u16, + parent_channel_id: &str, + ephemeral_channel_id: &str, + extra_fields: &[(&str, &str)], + participant_pubkey: Option<&str>, +) -> Result { + validate_channel_id(parent_channel_id)?; + validate_channel_id(ephemeral_channel_id)?; + let mut content = serde_json::json!({ + "ephemeral_channel_id": ephemeral_channel_id, + }); + for (k, v) in extra_fields { + content[*k] = serde_json::Value::String(v.to_string()); + } + let mut tags = vec![tag(vec!["h", parent_channel_id])?]; + if let Some(pk) = participant_pubkey { + tags.push(tag(vec!["p", pk])?); + } + Ok(EventBuilder::new(Kind::Custom(kind), content.to_string()).tags(tags)) +} + +/// Kind 48100 — huddle started advisory posted to the parent channel. +pub fn build_huddle_started( + parent_channel_id: &str, + ephemeral_channel_id: &str, + livekit_room: &str, +) -> Result { + build_huddle_event( + 48100, + parent_channel_id, + ephemeral_channel_id, + &[("livekit_room", livekit_room)], + None, + ) +} + +/// Kind 48101 — participant joined a huddle, posted to the parent channel. +/// +/// `participant_pubkey`: when provided, adds a `["p", pubkey]` tag so +/// consumers can identify who joined without parsing the event's author. +pub fn build_huddle_participant_joined( + parent_channel_id: &str, + ephemeral_channel_id: &str, + participant_pubkey: Option<&str>, +) -> Result { + build_huddle_event( + 48101, + parent_channel_id, + ephemeral_channel_id, + &[], + participant_pubkey, + ) +} + +/// Kind 48102 — participant left a huddle, posted to the parent channel. +/// +/// `participant_pubkey`: when provided, adds a `["p", pubkey]` tag so +/// consumers can identify who left without parsing the event's author. +pub fn build_huddle_participant_left( + parent_channel_id: &str, + ephemeral_channel_id: &str, + participant_pubkey: Option<&str>, +) -> Result { + build_huddle_event( + 48102, + parent_channel_id, + ephemeral_channel_id, + &[], + participant_pubkey, + ) +} + +/// Kind 48103 — huddle ended, posted to the parent channel. +pub fn build_huddle_ended( + parent_channel_id: &str, + ephemeral_channel_id: &str, +) -> Result { + build_huddle_event(48103, parent_channel_id, ephemeral_channel_id, &[], None) +} + +/// Kind 48106 — voice-mode guidelines for agents in a huddle. +/// +/// Posted to the **ephemeral** channel (not the parent) so agents see it +/// via EOSE replay when they subscribe. Uses a dedicated kind so the TTS +/// pipeline can filter it out without fragile content-prefix matching. +pub fn build_huddle_guidelines( + ephemeral_channel_id: &str, + guidelines_text: &str, +) -> Result { + validate_channel_id(ephemeral_channel_id)?; + check_content(guidelines_text)?; + let tags = vec![tag(vec!["h", ephemeral_channel_id])?]; + Ok(EventBuilder::new(Kind::Custom(48106), guidelines_text).tags(tags)) +} + // ── Social notes ──────────────────────────────────────────────────────────── /// Kind 1 — NIP-01 short text note (global, no channel scope). @@ -401,3 +509,42 @@ pub fn build_contact_list( } Ok(EventBuilder::new(Kind::ContactList, "").tags(tags)) } + +// ── Transport ──────────────────────────────────────────────────────────────── + +/// Post a pre-signed event to the relay. +/// +/// Standalone helper for async tasks that don't have access to `&AppState`. +/// The caller pre-captures `http_client`, `api_token`, and `pubkey_hex` at +/// spawn time and passes them here. +/// +/// Returns `Err` on transport failure OR non-2xx HTTP status. +pub async fn post_event_raw( + http_client: &reqwest::Client, + api_token: Option<&str>, + pubkey_hex: &str, + event_json: String, +) -> Result<(), String> { + let url = format!("{}/api/events", crate::relay::relay_api_base_url()); + let req = match api_token { + Some(token) => http_client + .post(&url) + .header("Authorization", format!("Bearer {token}")), + None => http_client.post(&url).header("X-Pubkey", pubkey_hex), + }; + let response = req + .header("Content-Type", "application/json") + .body(event_json) + .send() + .await + .map_err(|e| format!("event POST failed: {e}"))?; + + if !response.status().is_success() { + return Err(format!( + "event POST HTTP {}: {}", + response.status().as_u16(), + response.status().canonical_reason().unwrap_or("unknown"), + )); + } + Ok(()) +} diff --git a/desktop/src-tauri/src/huddle/agents.rs b/desktop/src-tauri/src/huddle/agents.rs new file mode 100644 index 00000000000..76708f7585e --- /dev/null +++ b/desktop/src-tauri/src/huddle/agents.rs @@ -0,0 +1,99 @@ +//! Agent enrollment for huddles. +//! +//! Mental model: +//! add_agent_to_huddle → kind:9000 to ephemeral channel +//! → kind:9000 to parent channel (best-effort) +//! +//! ACP spawning is NOT needed here: the running agent process auto-subscribes +//! when it receives the kind:9000 membership notification. Huddle-specific +//! env vars (interrupt mode, custom system prompt) are a post-MVP enhancement. + +use serde::Serialize; +use uuid::Uuid; + +use crate::{app_state::AppState, events, relay::submit_event}; + +// ── Constants ───────────────────────────────────────────────────────────────── + +/// Voice-mode guidelines posted as kind:48106 (huddle guidelines) to the +/// ephemeral channel at huddle start. Agents see them via EOSE replay. +/// Instructs agents on voice-mode etiquette: TTS constraints, brevity, self-selection. +/// Build voice-mode guidelines with the parent channel ID so agents know +/// where "the main channel" is. +pub fn voice_mode_guidelines(parent_channel_id: &str) -> String { + format!( + "\ +You are in a live voice huddle attached to channel {parent_channel_id}. +Your text is read aloud via TTS. You will be interrupted when humans speak — this is normal. + +- If not addressed or relevant: do nothing. Do not respond. +- One or two short sentences max. Start with the answer, no preamble. +- No markdown, code blocks, lists, or structured data — say it naturally. +- To share code or detailed data: say \"I'll post that in the main channel\" and do so. +- When tool results are long, summarize the key finding verbally. +- If interrupted, continue naturally. No apologies. +- In multi-agent huddles, identify yourself only when needed. +- Use your Sprout tools proactively when asked." + ) +} + +// ── Agent enrollment ────────────────────────────────────────────────────────── + +/// Result of adding an agent to a huddle. +/// +/// **Invariant:** `ephemeral_added` is always `true` on success — the function +/// returns `Err` before constructing this struct if the ephemeral add fails. +/// The field exists for forward compatibility with future batch-add operations +/// where partial success may be meaningful. +/// +/// `parent_added` reflects whether the parent-channel add succeeded; +/// `parent_error` carries the error string when it didn't. +#[derive(Debug, Serialize)] +pub struct AgentAddResult { + /// Always `true` — invariant guaranteed by [`add_agent_to_huddle`]. + pub ephemeral_added: bool, + /// Whether the agent was also added to the parent channel (best-effort). + pub parent_added: bool, + /// Error from the parent-channel add, if it failed. + pub parent_error: Option, +} + +/// Add an agent to both the ephemeral and parent huddle channels. +/// +/// Returns `Err` only if the ephemeral-channel add fails (policy rejection or +/// network error). The parent-channel add is best-effort: failure is captured +/// in `AgentAddResult::parent_error` rather than propagated. +/// +/// The running ACP process for this agent will auto-subscribe to the new +/// channel when it receives the kind:9000 membership notification. +pub async fn add_agent_to_huddle( + ephemeral_channel_id: Uuid, + parent_channel_id: Uuid, + agent_pubkey: &str, + state: &AppState, +) -> Result { + // 1. Add agent to ephemeral channel (required — fail hard on rejection). + let add_eph = events::build_add_member(ephemeral_channel_id, agent_pubkey, Some("bot"))?; + submit_event(add_eph, state).await?; + + // 2. Add agent to parent channel — so agent has full context. + // Best-effort: capture the error but don't propagate it. + let (parent_added, parent_error) = { + let add_parent = events::build_add_member(parent_channel_id, agent_pubkey, Some("bot"))?; + match submit_event(add_parent, state).await { + Ok(_) => (true, None), + Err(e) => { + eprintln!( + "sprout-desktop: add agent to parent channel failed (may already be member): {e}" + ); + (false, Some(e)) + } + } + }; + + Ok(AgentAddResult { + ephemeral_added: true, + parent_added, + parent_error, + }) +} diff --git a/desktop/src-tauri/src/huddle/kokoro.rs b/desktop/src-tauri/src/huddle/kokoro.rs new file mode 100644 index 00000000000..9f1c138c11b --- /dev/null +++ b/desktop/src-tauri/src/huddle/kokoro.rs @@ -0,0 +1,950 @@ +//! Kokoro-82M ONNX TTS engine — single-session inference with IPA G2P. +//! +//! Mental model: +//! +//! load_text_to_speech(model_dir) → KokoroTTS +//! load_voice_style(path) → VoiceStyle +//! tts.synth_chunk(text, lang, &style, steps, speed) → Vec @ 24 kHz +//! +//! ┌──────────┐ G2P ┌──────────┐ tokenize ┌──────────┐ +//! │ raw text │ ──────→ │ IPA str │ ─────────→ │ int64[] │ +//! └──────────┘ lexicon └──────────┘ 115-char └────┬─────┘ +//! │ +//! ┌──────────┐ style ┌──────────┐ ONNX ┌────▼─────┐ +//! │ .bin file│ ──────→ │ [1, 256] │ ─────────→ │ Vec │ +//! └──────────┘ indexed └──────────┘ session └──────────┘ +//! by token count 24 kHz PCM +//! +//! G2P strategy: dictionary lookup (us_gold.json, Apache-2.0 via misaki). +//! OOV words are spelled letter-by-letter using a static IPA table. +//! No espeak dependency — fully GPL-free. + +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use ndarray::{Array1, Array2}; +use ort::{session::Session, value::Value}; + +// ── Public constants ────────────────────────────────────────────────────────── + +pub const SAMPLE_RATE: u32 = 24_000; +pub const DEFAULT_VOICE: &str = "af_heart"; + +// Maximum phoneme tokens before padding (model context = 512, minus 2 pad tokens). +const MAX_PHONEME_TOKENS: usize = 510; + +// ── VoiceStyle ──────────────────────────────────────────────────────────────── + +/// Raw f32 voice embedding loaded from a `.bin` file. +/// +/// The binary is a flat array of shape `[-1, 256]` in row-major order. +/// Row `i` is the style vector for an utterance with `i` phoneme tokens. +/// This encodes both speaker identity and sequence-length-dependent prosody. +#[derive(Debug)] +pub struct VoiceStyle { + data: Vec, // flat: row i = data[i*256 .. (i+1)*256] +} + +impl VoiceStyle { + /// Return the 256-dim style vector for a given phoneme token count. + /// Clamps to the last available row if `token_count` is out of range. + fn get(&self, token_count: usize) -> &[f32] { + let max_rows = self.data.len() / 256; + let idx = token_count.min(max_rows.saturating_sub(1)); + &self.data[idx * 256..(idx + 1) * 256] + } +} + +/// Load a voice style from a raw little-endian f32 binary file. +pub fn load_voice_style(path: &Path) -> Result { + let bytes = fs::read(path).map_err(|e| format!("read voice {}: {e}", path.display()))?; + if bytes.len() % 4 != 0 { + return Err(format!( + "voice file {} has non-multiple-of-4 byte count ({})", + path.display(), + bytes.len() + )); + } + let data: Vec = bytes + .chunks_exact(4) + .map(|b| f32::from_le_bytes(b.try_into().unwrap())) + .collect(); + if data.len() < 256 { + return Err(format!( + "voice file {} too small ({} floats, need at least 256)", + path.display(), + data.len() + )); + } + if data.len() % 256 != 0 { + return Err(format!( + "voice style has {} floats — expected a multiple of 256 (got {} remainder)", + data.len(), + data.len() % 256, + )); + } + Ok(VoiceStyle { data }) +} + +// ── Tokenizer ───────────────────────────────────────────────────────────────── + +/// Static 115-entry IPA char → int64 lookup table. +/// IDs are non-contiguous (0–177); unknown chars are silently dropped. +/// Pad token '$' = 0 is prepended and appended to every sequence. +fn build_vocab() -> HashMap { + // Source: onnx-community/Kokoro-82M-v1.0-ONNX tokenizer.json + #[rustfmt::skip] + let entries: &[(char, i64)] = &[ + ('$', 0), + (';', 1), (':', 2), (',', 3), ('.', 4), ('!', 5), ('?', 6), + ('—', 9), ('…', 10), ('"', 11), ('(', 12), (')', 13), ('\u{201c}', 14), ('\u{201d}', 15), + (' ', 16), ('\u{0303}', 17), + ('ʣ', 18), ('ʥ', 19), ('ʦ', 20), ('ʨ', 21), ('ᵝ', 22), ('ꭧ', 23), + ('A', 24), ('I', 25), ('O', 31), ('Q', 33), ('S', 35), ('T', 36), + ('W', 39), ('Y', 41), ('ᵊ', 42), + ('a', 43), ('b', 44), ('c', 45), ('d', 46), ('e', 47), ('f', 48), + ('h', 50), ('i', 51), ('j', 52), ('k', 53), ('l', 54), ('m', 55), + ('n', 56), ('o', 57), ('p', 58), ('q', 59), ('r', 60), ('s', 61), + ('t', 62), ('u', 63), ('v', 64), ('w', 65), ('x', 66), ('y', 67), ('z', 68), + ('ɑ', 69), ('ɐ', 70), ('ɒ', 71), ('æ', 72), ('β', 75), ('ɔ', 76), + ('ɕ', 77), ('ç', 78), ('ɖ', 80), ('ð', 81), ('ʤ', 82), ('ə', 83), + ('ɚ', 85), ('ɛ', 86), ('ɜ', 87), ('ɟ', 90), ('ɡ', 92), ('ɥ', 99), + ('ɨ', 101), ('ɪ', 102), ('ʝ', 103), ('ɯ', 110), ('ɰ', 111), + ('ŋ', 112), ('ɳ', 113), ('ɲ', 114), ('ɴ', 115), ('ø', 116), + ('ɸ', 118), ('θ', 119), ('œ', 120), ('ɹ', 123), ('ɾ', 125), + ('ɻ', 126), ('ʁ', 128), ('ɽ', 129), ('ʂ', 130), ('ʃ', 131), + ('ʈ', 132), ('ʧ', 133), ('ʊ', 135), ('ʋ', 136), ('ʌ', 138), + ('ɣ', 139), ('ɤ', 140), ('χ', 142), ('ʎ', 143), ('ʒ', 147), + ('ʔ', 148), ('ˈ', 156), ('ˌ', 157), ('ː', 158), ('ʰ', 162), + ('ʲ', 164), ('↓', 169), ('→', 171), ('↗', 172), ('↘', 173), ('ᵻ', 177), + ]; + entries.iter().copied().collect() +} + +/// Convert an IPA phoneme string to a padded int64 token sequence. +/// Returns `[0, id1, id2, ..., idN, 0]` clamped to MAX_PHONEME_TOKENS+2. +/// The pre-pad token count (ids.len() - 2) is used to index the style vector. +fn tokenize(phonemes: &str, vocab: &HashMap) -> Vec { + let mut ids: Vec = vec![0]; // BOS pad + for id in phonemes + .chars() + .filter_map(|c| vocab.get(&c).copied()) + .take(MAX_PHONEME_TOKENS) + { + ids.push(id); + } + ids.push(0); // EOS pad + ids +} + +// ── G2P Lexicon ─────────────────────────────────────────────────────────────── + +/// Grapheme-to-phoneme engine with a four-tier fallback chain: +/// +/// 1. Misaki gold+silver dicts (183K words, Kokoro-native IPA) +/// 2. CMUdict (135K words, ARPAbet→Kokoro IPA) — covers inflected forms +/// 3. Morphological suffix stripping (-s/-ed/-ing) + retry tiers 1-2 +/// 4. Letter-by-letter spelling +/// +/// All dictionaries are Apache-2.0 or BSD licensed. No GPL. +struct Lexicon { + /// Misaki gold+silver merged dictionary (Kokoro-native IPA). + misaki: HashMap, + /// CMU Pronouncing Dictionary (ARPAbet converted to Kokoro IPA at load time). + cmudict: HashMap, +} + +/// IPA pronunciations for individual letter names (used for OOV words). +fn letter_ipa(c: char) -> &'static str { + match c { + 'a' => "ˈeɪ", + 'b' => "bˈiː", + 'c' => "sˈiː", + 'd' => "dˈiː", + 'e' => "ˈiː", + 'f' => "ˈɛf", + 'g' => "dʒˈiː", + 'h' => "ˈeɪtʃ", + 'i' => "ˈaɪ", + 'j' => "dʒˈeɪ", + 'k' => "kˈeɪ", + 'l' => "ˈɛl", + 'm' => "ˈɛm", + 'n' => "ˈɛn", + 'o' => "ˈoʊ", + 'p' => "pˈiː", + 'q' => "kjˈuː", + 'r' => "ˈɑːɹ", + 's' => "ˈɛs", + 't' => "tˈiː", + 'u' => "jˈuː", + 'v' => "vˈiː", + 'w' => "dˈʌbəljˌuː", + 'x' => "ˈɛks", + 'y' => "wˈaɪ", + 'z' => "zˈiː", + _ => "", + } +} + +/// Punctuation chars that are valid Kokoro vocab tokens and should pass through. +fn is_passthrough_punct(c: char) -> bool { + matches!(c, ';' | ':' | ',' | '.' | '!' | '?' | '—' | '…' | ' ') +} + +/// Vowels that trigger US English /t/→/ɾ/ flapping (misaki's US_TAUS). +const US_TAUS: &str = "AIOWYiuæɑəɛɪɹʊʌ"; + +/// ARPAbet → Kokoro IPA conversion. Stress digit is stripped before lookup. +fn arpabet_to_ipa(phoneme: &str) -> &'static str { + match phoneme { + "AA" => "ɑ", + "AE" => "æ", + "AH" => "ʌ", + "AO" => "ɔ", + "AW" => "W", + "AY" => "I", + "EH" => "ɛ", + "ER" => "ɜɹ", + "EY" => "A", + "IH" => "ɪ", + "IY" => "i", + "OW" => "O", + "OY" => "Y", + "UH" => "ʊ", + "UW" => "u", + "B" => "b", + "CH" => "ʧ", + "D" => "d", + "DH" => "ð", + "F" => "f", + "G" => "ɡ", + "HH" => "h", + "JH" => "ʤ", + "K" => "k", + "L" => "l", + "M" => "m", + "N" => "n", + "NG" => "ŋ", + "P" => "p", + "R" => "ɹ", + "S" => "s", + "SH" => "ʃ", + "T" => "t", + "TH" => "θ", + "V" => "v", + "W" => "w", + "Y" => "j", + "Z" => "z", + "ZH" => "ʒ", + _ => "", + } +} + +/// Convert a CMUdict ARPAbet pronunciation line to Kokoro IPA. +/// Input: "K R IY0 EY1 T AH0 D" → Output: "kɹiˈAtəd" +fn arpabet_line_to_ipa(arpabet: &str) -> String { + let mut out = String::new(); + for token in arpabet.split_whitespace() { + // Split phoneme from stress digit (e.g., "EY1" → "EY", Some('1')) + let (base, stress) = if token.ends_with(|c: char| c.is_ascii_digit()) { + (&token[..token.len() - 1], token.as_bytes().last().copied()) + } else { + (token, None) + }; + // Stress marker goes BEFORE the vowel's IPA + match stress { + Some(b'1') => out.push('ˈ'), // primary + Some(b'2') => out.push('ˌ'), // secondary + _ => {} + } + // AH with stress=0 is schwa (ə), not ʌ + if base == "AH" && stress == Some(b'0') { + out.push('ə'); + } else if base == "ER" && stress == Some(b'0') { + // Unstressed ER is just əɹ + out.push_str("əɹ"); + } else { + out.push_str(arpabet_to_ipa(base)); + } + } + out +} + +impl Lexicon { + /// Load misaki gold+silver dicts and CMUdict. + fn load(gold_path: &Path, silver_path: &Path, cmudict_path: &Path) -> Result { + let mut misaki = Self::load_json(silver_path)?; + let gold = Self::load_json(gold_path)?; + misaki.extend(gold); + + let cmudict = if cmudict_path.exists() { + Self::load_cmudict(cmudict_path)? + } else { + eprintln!( + "sprout-desktop: CMUdict not found at {} — inflected forms may be spelled out", + cmudict_path.display() + ); + HashMap::new() + }; + + eprintln!( + "sprout-desktop: G2P loaded — misaki: {} words, cmudict: {} words", + misaki.len(), + cmudict.len() + ); + Ok(Lexicon { misaki, cmudict }) + } + + fn load_json(path: &Path) -> Result, String> { + let content = + fs::read_to_string(path).map_err(|e| format!("read {}: {e}", path.display()))?; + let raw: serde_json::Value = + serde_json::from_str(&content).map_err(|e| format!("parse {}: {e}", path.display()))?; + let obj = raw + .as_object() + .ok_or_else(|| format!("{}: expected JSON object", path.display()))?; + let mut dict = HashMap::with_capacity(obj.len()); + for (word, val) in obj { + let ipa = match val { + serde_json::Value::String(s) => s.clone(), + serde_json::Value::Object(m) => m + .get("DEFAULT") + .or_else(|| m.values().next()) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(), + _ => continue, + }; + if !ipa.is_empty() { + dict.insert(word.to_lowercase(), ipa); + } + } + Ok(dict) + } + + /// Load CMUdict and convert ARPAbet → Kokoro IPA at load time. + /// Format: "WORD PH1 PH2 PH3\n" (single space between word and phonemes). + /// Variant pronunciations like "WORD(2)" are skipped — we take the first. + fn load_cmudict(path: &Path) -> Result, String> { + let content = fs::read_to_string(path).map_err(|e| format!("read cmudict: {e}"))?; + let mut dict = HashMap::with_capacity(140_000); + for line in content.lines() { + // Skip comments and blank lines + if line.starts_with(";;;") || line.is_empty() { + continue; + } + // Split on first space + let (word, phonemes) = match line.find(' ') { + Some(i) => (&line[..i], line[i + 1..].trim()), + None => continue, + }; + // Skip variant pronunciations like "WORD(2)" + if word.contains('(') { + continue; + } + let key = word.to_lowercase(); + let ipa = arpabet_line_to_ipa(phonemes); + if !ipa.is_empty() { + dict.entry(key).or_insert(ipa); + } + } + Ok(dict) + } + + /// Look up a word across all tiers. Returns None if not found anywhere. + fn lookup(&self, word: &str) -> Option { + self.misaki + .get(word) + .cloned() + .or_else(|| self.cmudict.get(word).cloned()) + } + + /// Apply English -s/-es/-ies suffix phoneme rules (misaki's `_s`). + fn apply_s(stem_ipa: &str) -> String { + let last = stem_ipa.chars().last().unwrap_or(' '); + if "ptkfθ".contains(last) { + format!("{stem_ipa}s") + } else if "szʃʒʧʤ".contains(last) { + format!("{stem_ipa}ᵻz") + } else { + format!("{stem_ipa}z") + } + } + + /// Apply English -ed suffix phoneme rules (misaki's `_ed`). + fn apply_ed(stem_ipa: &str) -> String { + let chars: Vec = stem_ipa.chars().collect(); + let last = *chars.last().unwrap_or(&' '); + if "pkfθʃsʧ".contains(last) { + format!("{stem_ipa}t") + } else if last == 'd' { + format!("{stem_ipa}ᵻd") + } else if last != 't' { + format!("{stem_ipa}d") + } else if chars.len() >= 2 && US_TAUS.contains(chars[chars.len() - 2]) { + // US flap: "created" → kɹiˈAɾᵻd + let mut out: String = chars[..chars.len() - 1].iter().collect(); + out.push_str("ɾᵻd"); + out + } else { + format!("{stem_ipa}ᵻd") + } + } + + /// Apply English -ing suffix phoneme rules (misaki's `_ing`). + fn apply_ing(stem_ipa: &str) -> String { + let chars: Vec = stem_ipa.chars().collect(); + let last = *chars.last().unwrap_or(&' '); + if last == 't' && chars.len() >= 2 && US_TAUS.contains(chars[chars.len() - 2]) { + // US flap: "creating" → kɹiˈAɾɪŋ + let mut out: String = chars[..chars.len() - 1].iter().collect(); + out.push_str("ɾɪŋ"); + out + } else { + format!("{stem_ipa}ɪŋ") + } + } + + /// Try stripping -s/-ed/-ing suffix, look up the base, and re-apply phonetically. + fn try_morphological(&self, word: &str) -> Option { + // Try -s / -es / -ies + if word.len() >= 3 && word.ends_with('s') { + // -ies → base + y + if word.len() > 4 && word.ends_with("ies") { + if let Some(stem) = self.lookup(&format!("{}y", &word[..word.len() - 3])) { + return Some(Self::apply_s(&stem)); + } + } + // -es → base + if word.len() > 4 && word.ends_with("es") && !word.ends_with("ies") { + if let Some(stem) = self.lookup(&word[..word.len() - 2]) { + return Some(Self::apply_s(&stem)); + } + } + // -s → base + if !word.ends_with("ss") { + if let Some(stem) = self.lookup(&word[..word.len() - 1]) { + return Some(Self::apply_s(&stem)); + } + } + } + // Try -ed / -d + if word.len() >= 4 && word.ends_with('d') { + // -ed → base (not -eed) + if word.len() > 4 && word.ends_with("ed") && !word.ends_with("eed") { + if let Some(stem) = self.lookup(&word[..word.len() - 2]) { + return Some(Self::apply_ed(&stem)); + } + // -ed where base ends in e: "created" → "create" + if let Some(stem) = self.lookup(&format!("{}e", &word[..word.len() - 2])) { + return Some(Self::apply_ed(&stem)); + } + } + // -d → base (e.g., "discovered" → strip "d" → "discovere" fails, + // but "configured" → strip "d" → "configure" works) + if !word.ends_with("dd") { + if let Some(stem) = self.lookup(&word[..word.len() - 1]) { + return Some(Self::apply_ed(&stem)); + } + } + } + // Try -ing + if word.len() >= 5 && word.ends_with("ing") { + let base = &word[..word.len() - 3]; + // -ing → base (e.g., "running" base = "runn" — won't match, need double-consonant) + if let Some(stem) = self.lookup(base) { + return Some(Self::apply_ing(&stem)); + } + // -ing + e → base+e (e.g., "creating" → "creat" + "e" = "create") + if let Some(stem) = self.lookup(&format!("{base}e")) { + return Some(Self::apply_ing(&stem)); + } + // Double consonant: "running" → "run" + if base.len() >= 2 { + let bytes = base.as_bytes(); + if bytes[bytes.len() - 1] == bytes[bytes.len() - 2] { + if let Some(stem) = self.lookup(&base[..base.len() - 1]) { + return Some(Self::apply_ing(&stem)); + } + } + } + } + None + } + + /// Convert a single word to IPA using the full fallback chain. + fn word_to_ipa(&self, word: &str) -> String { + // Compound words: split on hyphens and underscores, process each part + // independently. "short-and-natural" → "short" + "and" + "natural", + // "parent_event_id" → "parent" + "event" + "id". + // Each part gets full dict lookup. Joined with a space (brief TTS pause). + if word.contains('-') || word.contains('_') { + let parts: Vec = word + .split(|c: char| c == '-' || c == '_') + .filter(|p| !p.is_empty()) + .map(|p| self.word_to_ipa(p)) + .collect(); + return parts.join(" "); + } + + // Normalize curly quotes to straight apostrophes. + let normalized = word.replace('\u{2019}', "'").replace('\u{2018}', "'"); + let stripped: String = normalized + .chars() + .filter(|c| c.is_alphabetic() || *c == '\'') + .collect::() + .to_lowercase(); + + // Tier 1+2: misaki + CMUdict direct lookup + if let Some(ipa) = self.lookup(&stripped) { + return ipa; + } + + // Contractions: "don't" → "don" + "'t" + if let Some(apos_idx) = stripped.find('\'') { + let base = &stripped[..apos_idx]; + let suffix = &stripped[apos_idx..]; + if let Some(base_ipa) = self.lookup(base) { + let suffix_ipa = self.lookup(suffix).unwrap_or_else(|| match suffix { + "'ve" => "v".to_string(), + "'re" => "ɹ".to_string(), + _ => String::new(), + }); + if !suffix_ipa.is_empty() { + return format!("{base_ipa}{suffix_ipa}"); + } + } + } + + // Tier 3: morphological suffix stripping + if let Some(ipa) = self.try_morphological(&stripped) { + return ipa; + } + + // Tier 4: letter-by-letter spelling + stripped + .chars() + .filter(|c| c.is_alphabetic()) + .map(letter_ipa) + .collect() + } + + /// Convert a full text chunk to an IPA phoneme string. + fn text_to_ipa(&self, text: &str) -> String { + let mut out = String::new(); + for token in text.split_whitespace() { + if !out.is_empty() { + out.push(' '); + } + let leading: String = token + .chars() + .take_while(|c| is_passthrough_punct(*c)) + .collect(); + let trailing: String = token + .chars() + .rev() + .take_while(|c| is_passthrough_punct(*c)) + .collect::() + .chars() + .rev() + .collect(); + let word = &token[leading.len()..token.len() - trailing.len()]; + out.push_str(&leading); + if !word.is_empty() { + out.push_str(&self.word_to_ipa(word)); + } + out.push_str(&trailing); + } + out + } +} + +// ── KokoroTTS ───────────────────────────────────────────────────────────────── + +pub struct KokoroTTS { + session: Session, + vocab: HashMap, + lexicon: Lexicon, + // Retained for potential future use (e.g., hot-reloading voices by path). + #[allow(dead_code)] + model_dir: PathBuf, +} + +/// Load the Kokoro TTS engine from a model directory. +/// +/// Expects: +/// `/model.onnx` (or model_quantized.onnx — tries both) +/// `/us_gold.json` (G2P dictionary) +/// +/// CoreML execution provider is registered with auto-fallback to CPU. +/// The compiled CoreML model is cached in `/.coreml_cache/`. +pub fn load_text_to_speech(model_dir: &str) -> Result { + let model_dir_path = PathBuf::from(model_dir); + + // Try quantized model first for speed, fall back to full-precision. + let model_path = ["model_quantized.onnx", "model_q8f16.onnx", "model.onnx"] + .iter() + .map(|name| model_dir_path.join(name)) + .find(|p| p.exists()) + .ok_or_else(|| format!("no model.onnx found in {model_dir}"))?; + + // Try CoreML first (zero binary cost — macOS system framework). + // If the model has ops CoreML can't handle (common with quantized models), + // the EP registers fine but commit_from_file fails. Catch that and retry + // with CPU-only. This is the expected path for model_q8f16.onnx. + let session = { + let mut builder_with_coreml = Session::builder() + .map_err(|e| format!("session builder: {e}"))? + .with_execution_providers([ort::ep::CoreML::default() + .with_compute_units(ort::ep::coreml::ComputeUnits::All) + .with_model_format(ort::ep::coreml::ModelFormat::MLProgram) + .with_model_cache_dir(model_dir_path.join(".coreml_cache").to_string_lossy()) + .build()]) + .map_err(|e| format!("execution provider: {e}"))?; + + match builder_with_coreml.commit_from_file(&model_path) { + Ok(s) => { + eprintln!("sprout-desktop: Kokoro loaded with CoreML acceleration"); + s + } + Err(coreml_err) => { + eprintln!( + "sprout-desktop: CoreML failed for {}, falling back to CPU: {coreml_err}", + model_path.display() + ); + // Retry without any execution providers — pure CPU. + Session::builder() + .map_err(|e| format!("session builder (CPU fallback): {e}"))? + .commit_from_file(&model_path) + .map_err(|e| format!("load model {} (CPU): {e}", model_path.display()))? + } + } + }; + + let gold_path = model_dir_path.join("us_gold.json"); + let silver_path = model_dir_path.join("us_silver.json"); + let cmudict_path = model_dir_path.join("cmudict.dict"); + let lexicon = Lexicon::load(&gold_path, &silver_path, &cmudict_path)?; + + Ok(KokoroTTS { + session, + vocab: build_vocab(), + lexicon, + model_dir: model_dir_path, + }) +} + +impl KokoroTTS { + /// Synthesize a single pre-split text chunk. Caller is responsible for sentence splitting. + /// This avoids double-splitting when the TTS pipeline has already split the text. + /// + /// - `_lang` is accepted for API compatibility but currently unused + /// (Kokoro v1.0 language is selected by voice name prefix, e.g. `af_*`). + /// - `_steps` is accepted for API compatibility but currently unused + /// (Kokoro is not diffusion-based). + pub fn synth_chunk( + &mut self, + text: &str, + _lang: &str, + style: &VoiceStyle, + _steps: usize, + speed: f32, + ) -> Result, String> { + // G2P: text → IPA phoneme string + let ipa = self.lexicon.text_to_ipa(text); + + // Tokenize: IPA → int64 ids with BOS/EOS pad tokens + let token_ids = tokenize(&ipa, &self.vocab); + + // Style vector is indexed by phoneme count (excluding the 2 pad tokens). + // Shape expected by model: [1, 256] (kokoro-js uses [1, 256], not [1, 1, 256]). + let phoneme_count = token_ids.len() - 2; + let style_slice = style.get(phoneme_count); + + // Build ONNX input tensors. + let seq_len = token_ids.len(); + let input_ids_arr = Array2::from_shape_vec((1, seq_len), token_ids) + .map_err(|e| format!("input_ids shape: {e}"))?; + let input_ids_val = + Value::from_array(input_ids_arr).map_err(|e| format!("input_ids Value: {e}"))?; + + // Style: [1, 256]. Research notes [1, 1, 256] but kokoro-js uses [1, 256]. + let style_arr = Array2::from_shape_vec((1, 256), style_slice.to_vec()) + .map_err(|e| format!("style shape: {e}"))?; + let style_val = Value::from_array(style_arr).map_err(|e| format!("style Value: {e}"))?; + + let speed_arr = Array1::from_vec(vec![speed]); + let speed_val = Value::from_array(speed_arr).map_err(|e| format!("speed Value: {e}"))?; + + // Run inference. Output[0] = waveform float32[1, N_samples]. + let outputs = self + .session + .run(ort::inputs! { + "input_ids" => &input_ids_val, + "style" => &style_val, + "speed" => &speed_val, + }) + .map_err(|e| format!("onnx run: {e}"))?; + + let (_, waveform) = outputs[0] + .try_extract_tensor::() + .map_err(|e| format!("extract waveform: {e}"))?; + + Ok(waveform.to_vec()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // ── Tokenizer ───────────────────────────────────────────────────────── + + #[test] + fn tokenize_empty_produces_bos_eos() { + let vocab = build_vocab(); + let ids = tokenize("", &vocab); + assert_eq!(ids, vec![0, 0]); // BOS + EOS only + } + + #[test] + fn tokenize_known_chars() { + let vocab = build_vocab(); + let ids = tokenize("a", &vocab); + // 'a' maps to 43 in the vocab + assert_eq!(ids, vec![0, 43, 0]); + } + + #[test] + fn tokenize_unknown_chars_dropped() { + let vocab = build_vocab(); + let ids = tokenize("🎉", &vocab); + // Emoji not in vocab — should be dropped, leaving only BOS+EOS + assert_eq!(ids, vec![0, 0]); + } + + #[test] + fn tokenize_respects_max_length() { + let vocab = build_vocab(); + let long_input: String = "a".repeat(600); // exceeds MAX_PHONEME_TOKENS (510) + let ids = tokenize(&long_input, &vocab); + // Should be clamped: BOS + 510 tokens + EOS = 512 + assert_eq!(ids.len(), 512); + assert_eq!(ids[0], 0); // BOS + assert_eq!(*ids.last().unwrap(), 0); // EOS + } + + // ── ARPAbet conversion ──────────────────────────────────────────────── + + #[test] + fn arpabet_simple_word() { + // "HH AH0 L OW1" = hello + let ipa = arpabet_line_to_ipa("HH AH0 L OW1"); + assert_eq!(ipa, "həlˈO"); + } + + #[test] + fn arpabet_stress_markers() { + // Primary stress before vowel, secondary stress before vowel + let ipa = arpabet_line_to_ipa("K R IY0 EY1 T"); + // IY0 = unstressed 'i', EY1 = primary 'A' + assert!(ipa.contains('ˈ'), "should contain primary stress: {ipa}"); + } + + #[test] + fn arpabet_schwa() { + // AH0 should produce schwa (ə), not ʌ + let ipa = arpabet_line_to_ipa("AH0"); + assert_eq!(ipa, "ə"); + } + + #[test] + fn arpabet_unstressed_er() { + // ER0 should produce əɹ + let ipa = arpabet_line_to_ipa("ER0"); + assert_eq!(ipa, "əɹ"); + } + + // ── Letter IPA ──────────────────────────────────────────────────────── + + #[test] + fn letter_ipa_covers_alphabet() { + for c in 'a'..='z' { + let ipa = letter_ipa(c); + assert!(!ipa.is_empty(), "letter_ipa('{c}') returned empty"); + } + } + + #[test] + fn letter_ipa_non_alpha_empty() { + assert_eq!(letter_ipa('1'), ""); + assert_eq!(letter_ipa('!'), ""); + } + + // ── Punctuation passthrough ─────────────────────────────────────────── + + #[test] + fn passthrough_punct_includes_expected() { + assert!(is_passthrough_punct('.')); + assert!(is_passthrough_punct('!')); + assert!(is_passthrough_punct('?')); + assert!(is_passthrough_punct(' ')); + assert!(is_passthrough_punct(',')); + } + + #[test] + fn passthrough_punct_excludes_alpha() { + assert!(!is_passthrough_punct('a')); + assert!(!is_passthrough_punct('Z')); + } + + // ── VoiceStyle ──────────────────────────────────────────────────────── + + #[test] + fn voice_style_get_clamps_to_last_row() { + // 2 rows of 256 floats + let data: Vec = (0..512).map(|i| i as f32).collect(); + let style = VoiceStyle { data }; + // Row 0 + assert_eq!(style.get(0)[0], 0.0); + // Row 1 + assert_eq!(style.get(1)[0], 256.0); + // Row 999 should clamp to row 1 (last available) + assert_eq!(style.get(999)[0], 256.0); + } + + #[test] + fn load_voice_style_rejects_too_small() { + use std::io::Write; + let dir = std::env::temp_dir().join("kokoro_test_small"); + let _ = std::fs::create_dir_all(&dir); + let path = dir.join("tiny.bin"); + let mut f = std::fs::File::create(&path).unwrap(); + // Write only 100 floats (need at least 256) + for i in 0..100u32 { + f.write_all(&(i as f32).to_le_bytes()).unwrap(); + } + drop(f); + let result = load_voice_style(&path); + assert!(result.is_err(), "should reject file with < 256 floats"); + assert!(result.unwrap_err().contains("too small")); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn load_voice_style_rejects_non_multiple_of_256() { + use std::io::Write; + let dir = std::env::temp_dir().join("kokoro_test_nonaligned"); + let _ = std::fs::create_dir_all(&dir); + let path = dir.join("nonaligned.bin"); + let mut f = std::fs::File::create(&path).unwrap(); + // Write 257 floats — not a multiple of 256 (remainder = 1) + for i in 0..257u32 { + f.write_all(&(i as f32).to_le_bytes()).unwrap(); + } + drop(f); + let result = load_voice_style(&path); + assert!( + result.is_err(), + "should reject file with non-multiple-of-256 floats" + ); + let err = result.unwrap_err(); + assert!( + err.contains("257"), + "error should mention float count: {err}" + ); + assert!( + err.contains("remainder"), + "error should mention remainder: {err}" + ); + let _ = std::fs::remove_dir_all(&dir); + } + + // ── Suffix rules ────────────────────────────────────────────────────── + + #[test] + fn apply_s_voiceless() { + // After voiceless consonants: +s + assert!(Lexicon::apply_s("kæt").ends_with('s')); + } + + #[test] + fn apply_s_sibilant() { + // After sibilants: +ᵻz + assert!(Lexicon::apply_s("bʌz").ends_with("ᵻz")); + } + + #[test] + fn apply_s_voiced() { + // After voiced consonants: +z + assert!(Lexicon::apply_s("dɔɡ").ends_with('z')); + } + + #[test] + fn apply_ed_voiceless() { + // After voiceless: +t + assert!(Lexicon::apply_ed("wɔk").ends_with('t')); + } + + #[test] + fn apply_ed_d_ending() { + // After d: +ᵻd + assert!(Lexicon::apply_ed("æd").ends_with("ᵻd")); + } + + #[test] + fn apply_ing_basic() { + assert!(Lexicon::apply_ing("rʌn").ends_with("ɪŋ")); + } + + #[test] + fn hyphenated_word_splits_into_parts() { + let lex = Lexicon { + misaki: HashMap::new(), + cmudict: HashMap::new(), + }; + let result = lex.word_to_ipa("short-and-natural"); + let space_count = result.matches(' ').count(); + assert_eq!( + space_count, 2, + "expected 2 spaces for 3 hyphenated parts, got {space_count}: {result}" + ); + } + + #[test] + fn underscored_word_splits_into_parts() { + let lex = Lexicon { + misaki: HashMap::new(), + cmudict: HashMap::new(), + }; + let result = lex.word_to_ipa("parent_event_id"); + let space_count = result.matches(' ').count(); + assert_eq!( + space_count, 2, + "expected 2 spaces for 3 underscored parts, got {space_count}: {result}" + ); + } + + #[test] + fn compound_word_with_dict_lookup() { + let mut dict = HashMap::new(); + dict.insert("parent".to_string(), "pɛɹənt".to_string()); + dict.insert("event".to_string(), "ɪvɛnt".to_string()); + dict.insert("id".to_string(), "aɪdiː".to_string()); + let lex = Lexicon { + misaki: dict, + cmudict: HashMap::new(), + }; + // Underscore compound + let result = lex.word_to_ipa("parent_event_id"); + assert!(result.contains("pɛɹənt"), "parent not resolved: {result}"); + assert!(result.contains("ɪvɛnt"), "event not resolved: {result}"); + assert!(result.contains("aɪdiː"), "id not resolved: {result}"); + + // Hyphen compound + let result = lex.word_to_ipa("short-and-sweet"); + assert_eq!(result.matches(' ').count(), 2, "hyphen split: {result}"); + } +} diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs new file mode 100644 index 00000000000..8c3032be586 --- /dev/null +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -0,0 +1,983 @@ +//! Huddle (voice/video) state machine and Tauri commands. +//! +//! Mental model: +//! parent channel → start_huddle → ephemeral channel + LiveKit token +//! other clients → join_huddle → LiveKit token +//! any client → leave_huddle → lifecycle event, clear local state +//! creator → end_huddle → archive ephemeral channel, clear state +//! +//! HuddleState is stored in AppState and serialized for get_huddle_state. +//! +//! ## Synchronization Protocol +//! +//! `HuddleState` lives behind a single `Mutex` in `AppState`. Rules: +//! +//! 1. **Never hold the outer lock across `.await`** — acquire, read/write, release. +//! 2. **Pipeline construction happens outside the lock** — the `stt_starting` / +//! `tts_starting` sentinels prevent TOCTOU races during the ~200ms window. +//! 3. **`agent_pubkeys` has its own inner `Arc`** — the transcription task +//! clones the `Arc` and reads at post time without the outer lock. +//! 4. **Atomics for cross-thread signaling** — `tts_active`, `tts_cancel`, +//! `ptt_active`, `session_generation` are shared with pipeline worker threads. +//! 5. **Pipeline teardown extracts handles before dropping** — `teardown_huddle` +//! takes `stt_pipeline`/`tts_pipeline` out of the lock, then calls `shutdown()` +//! and drops them outside the lock (thread joins can block ~200ms). + +pub mod agents; +pub mod kokoro; +pub mod models; +pub mod pipeline; +pub mod preprocessing; +pub mod relay_api; +pub mod state; +pub mod stt; +pub mod tts; + +// ── Shared utilities ────────────────────────────────────────────────────────── + +/// Drain and discard all pending messages until shutdown or disconnect. +/// Shared by both the STT and TTS worker threads for graceful degradation +/// when model files are missing or initialization fails. +pub(super) fn drain_until_shutdown( + rx: std::sync::mpsc::Receiver, + shutdown: &std::sync::atomic::AtomicBool, +) { + loop { + if shutdown.load(std::sync::atomic::Ordering::Acquire) { + break; + } + match rx.recv_timeout(std::time::Duration::from_millis(100)) { + Ok(_) => continue, + Err(_) => break, + } + } +} + +// ── Re-exports ──────────────────────────────────────────────────────────────── + +pub use state::{HuddleJoinInfo, HuddlePhase, HuddleState, VoiceInputMode}; + +// ── Imports ─────────────────────────────────────────────────────────────────── + +use std::sync::{atomic::Ordering, Arc}; +use tauri::State; +use uuid::Uuid; + +use crate::{app_state::AppState, events, relay::submit_event}; + +use pipeline::{maybe_start_stt_pipeline, maybe_start_tts_pipeline, post_connect_setup}; +use relay_api::{ + count_human_members, fetch_channel_members, fetch_livekit_token, parse_channel_uuid, + validate_pubkey_hex, MAX_HUDDLE_AGENTS, +}; +use state::LiveKitTokenResponse; + +// ── Tauri commands ──────────────────────────────────────────────────────────── + +/// Set the voice input mode (push-to-talk or voice-activity detection). +/// +/// When switching mid-huddle, restarts the STT pipeline so it picks up the +/// new mode (PTT gating vs continuous VAD with barge-in). The pipeline +/// captures the mode at construction time, so a restart is required. +#[tauri::command] +pub async fn set_voice_input_mode( + mode: VoiceInputMode, + state: State<'_, AppState>, +) -> Result<(), String> { + let needs_restart = { + let mut hs = state.huddle()?; + let old_mode = hs.voice_input_mode.clone(); + hs.voice_input_mode = mode.clone(); + // Restart STT if mode changed and a huddle is active with a pipeline running. + old_mode != mode + && matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active) + && hs.stt_pipeline.is_some() + }; + + if needs_restart { + let eph_id = { + let hs = state.huddle()?; + hs.ephemeral_channel_id.clone() + }; + if let Some(eph_id) = eph_id { + // Best-effort restart — if models aren't ready, the pipeline + // stays down until the next hotstart cycle picks it up. + if let Err(e) = maybe_start_stt_pipeline(&state, &eph_id).await { + eprintln!("sprout-desktop: STT pipeline restart on mode switch failed: {e}"); + } + } + } + + Ok(()) +} + +/// Return the current voice input mode. +#[tauri::command] +pub fn get_voice_input_mode(state: State<'_, AppState>) -> Result { + let hs = state.huddle()?; + Ok(hs.voice_input_mode.clone()) +} + +/// Start a new huddle in the given parent channel. +/// +/// Steps: +/// 1. Create an ephemeral channel (kind 9007, ttl=3600). +/// 2. Add each invited member to the ephemeral channel (kind 9000). +/// 3. Fetch a LiveKit token from the relay. +/// 4. Emit KIND_HUDDLE_STARTED to the parent channel (kind 48100) — only after +/// token is confirmed, so no phantom announcement on failure. +/// 5. Store state and return join info. +/// +/// If ANY step fails (including channel creation), the orphaned ephemeral +/// channel is archived (best-effort) and state is reset to Idle. +#[tauri::command] +pub async fn start_huddle( + parent_channel_id: String, + member_pubkeys: Vec, + state: State<'_, AppState>, +) -> Result { + // Validate inputs at the Tauri boundary. + if member_pubkeys.len() > MAX_HUDDLE_AGENTS { + return Err(format!( + "too many agents: {} (max {})", + member_pubkeys.len(), + MAX_HUDDLE_AGENTS + )); + } + // Dedup and validate pubkey format. + let member_pubkeys: Vec = { + let mut seen = std::collections::HashSet::new(); + let mut deduped = Vec::new(); + for pk in member_pubkeys { + validate_pubkey_hex(&pk)?; + if seen.insert(pk.clone()) { + deduped.push(pk); + } + } + deduped + }; + + // Transition to Creating. + { + let mut hs = state.huddle()?; + if hs.phase != HuddlePhase::Idle { + return Err(format!( + "cannot start huddle: already in phase {:?}", + hs.phase + )); + } + hs.phase = HuddlePhase::Creating; + hs.parent_channel_id = Some(parent_channel_id.clone()); + } + + let ephemeral_uuid = Uuid::new_v4(); + let ephemeral_channel_id = ephemeral_uuid.to_string(); + let short_id = &ephemeral_channel_id[..8]; + let channel_name = format!("huddle-{short_id}"); + + // All steps wrapped so we can roll back on ANY failure, including step 1. + // channel_was_created tracks whether we need to archive on rollback. + let mut channel_was_created = false; + + let result: Result<(LiveKitTokenResponse, Vec), String> = async { + // 1. Create ephemeral channel. + let create_builder = events::build_create_channel( + ephemeral_uuid, + &channel_name, + "private", + "stream", + None, + Some(3600), + )?; + submit_event(create_builder, &state).await?; + channel_was_created = true; + + // 2. Post voice-mode guidelines as kind:48106 BEFORE adding agents. + // Agents auto-subscribe on membership notification (kind:9000) and may + // complete EOSE before guidelines are stored if we post them after. + // Best-effort: don't fail the huddle if this fails. + let guidelines = agents::voice_mode_guidelines(&parent_channel_id); + if let Ok(guidelines_builder) = + events::build_huddle_guidelines(&ephemeral_channel_id, &guidelines) + { + if let Err(e) = submit_event(guidelines_builder, &state).await { + eprintln!("sprout-desktop: huddle guidelines (kind:48106) failed: {e}"); + } + } + + // 3. Add members to the ephemeral channel; only keep successfully enrolled ones. + let mut successful_agents: Vec = Vec::new(); + for pubkey in &member_pubkeys { + let add_builder = events::build_add_member(ephemeral_uuid, pubkey, Some("bot"))?; + match submit_event(add_builder, &state).await { + Ok(_) => successful_agents.push(pubkey.clone()), + Err(e) => { + eprintln!("sprout-desktop: huddle add_member failed for {pubkey}: {e}"); + // Intentionally not added — policy rejected this agent. + } + } + } + + // 4. Fetch LiveKit token BEFORE emitting HUDDLE_STARTED. + // This prevents a phantom announcement if the token fetch fails. + // Creator is already the channel owner — no auto-add needed (None). + let lk = fetch_livekit_token(&ephemeral_channel_id, None, &state).await?; + + // 5. Emit HUDDLE_STARTED to parent channel — only now that token is confirmed. + let started_builder = + events::build_huddle_started(&parent_channel_id, &ephemeral_channel_id, &lk.room)?; + submit_event(started_builder, &state).await?; + + Ok((lk, successful_agents)) + } + .await; + + match result { + Ok((lk, successful_agents)) => { + // 5. Store active state. + { + let mut hs = state.huddle()?; + hs.phase = HuddlePhase::Connected; + hs.is_creator = true; + hs.ephemeral_channel_id = Some(ephemeral_channel_id.clone()); + hs.livekit_token = Some(lk.token.clone()); + hs.livekit_url = Some(lk.url.clone()); + hs.livekit_room = Some(lk.room.clone()); + // Only store agents that were successfully enrolled (Fix 1). + *hs.agent_pubkeys.lock().unwrap_or_else(|e| e.into_inner()) = + successful_agents.clone(); + // Include the current user + successfully enrolled agents as participants. + // Use successful_agents (not member_pubkeys) so failed enrollments + // are not reflected in the participant list. + let own_pubkey = state + .keys + .lock() + .map(|k| k.public_key().to_hex()) + .unwrap_or_default(); + let mut participants = successful_agents.clone(); + if !own_pubkey.is_empty() && !participants.contains(&own_pubkey) { + participants.insert(0, own_pubkey); + } + hs.participants = participants; + } + + // 6. Notify frontend of state change. + state.emit_huddle_state_changed(); + + // 7. Hydrate members, download models, start pipelines. + if let Err(e) = post_connect_setup(&state, &ephemeral_channel_id).await { + eprintln!("sprout-desktop: post_connect_setup failed (degraded mode): {e}"); + // Non-fatal: huddle works without STT/TTS pipelines. + } + + Ok(HuddleJoinInfo { + ephemeral_channel_id, + livekit_token: lk.token, + livekit_url: lk.url, + livekit_room: lk.room, + }) + } + Err(e) => { + // Rollback: archive the orphaned ephemeral channel if it was created. + if channel_was_created { + if let Ok(archive_builder) = events::build_archive(ephemeral_uuid) { + if let Err(ae) = submit_event(archive_builder, &state).await { + eprintln!( + "sprout-desktop: rollback archive of {ephemeral_channel_id} failed: {ae}" + ); + } + } + } + // Reset state to Idle so the user can retry. + // Preserve session_generation so in-flight transcription tasks + // from a prior session still see a stale generation and exit. + if let Ok(mut hs) = state.huddle_state.lock() { + hs.reset_preserving_generation(); + } + Err(e) + } + } +} + +/// Join an existing huddle in the given parent channel. +/// +/// Steps: +/// 1. Transition to Connecting. +/// 2. Fetch a LiveKit token from the relay (relay auto-adds caller as member +/// of the ephemeral channel via `parent_channel_id` query param). +/// 3. Emit KIND_HUDDLE_PARTICIPANT_JOINED to the parent channel (best-effort). +/// 4. Store state and return join info. +/// 5. Post-connect setup (pipelines, model hydration). +#[tauri::command] +pub async fn join_huddle( + parent_channel_id: String, + ephemeral_channel_id: String, + livekit_room: String, + state: State<'_, AppState>, +) -> Result { + // Transition to Connecting. + { + let mut hs = state.huddle()?; + if hs.phase != HuddlePhase::Idle { + return Err(format!( + "cannot join huddle: already in phase {:?}", + hs.phase + )); + } + hs.phase = HuddlePhase::Connecting; + hs.parent_channel_id = Some(parent_channel_id.clone()); + hs.ephemeral_channel_id = Some(ephemeral_channel_id.clone()); + hs.livekit_room = Some(livekit_room.clone()); + } + + let result: Result<(LiveKitTokenResponse, String), String> = async { + // 1. Fetch LiveKit token — relay auto-adds caller to the ephemeral channel + // when parent_channel_id is provided (caller must be a parent member). + let lk = + fetch_livekit_token(&ephemeral_channel_id, Some(&parent_channel_id), &state).await?; + + // 2. Emit PARTICIPANT_JOINED (best-effort) — include own pubkey as p-tag. + // Fetch own_pubkey AFTER the token call so a key-lock failure doesn't + // block the join; the event is best-effort anyway. + let own_pubkey = state + .keys + .lock() + .map(|k| k.public_key().to_hex()) + .unwrap_or_default(); + if let Ok(joined_builder) = events::build_huddle_participant_joined( + &parent_channel_id, + &ephemeral_channel_id, + if own_pubkey.is_empty() { + None + } else { + Some(own_pubkey.as_str()) + }, + ) { + if let Err(e) = submit_event(joined_builder, &state).await { + eprintln!("sprout-desktop: huddle_participant_joined event failed: {e}"); + } + } + + Ok((lk, own_pubkey)) + } + .await; + + match result { + Ok((lk, own_pubkey)) => { + // 3. Store active state. + { + let mut hs = state.huddle()?; + hs.phase = HuddlePhase::Connected; + hs.livekit_token = Some(lk.token.clone()); + hs.livekit_url = Some(lk.url.clone()); + hs.livekit_room = Some(lk.room.clone()); + // Seed with current user as a fallback until relay responds. + if !own_pubkey.is_empty() { + hs.participants = vec![own_pubkey]; + } + } + + // 4. Notify frontend of state change. + state.emit_huddle_state_changed(); + + // 5. Hydrate members, download models, start pipelines. + if let Err(e) = post_connect_setup(&state, &ephemeral_channel_id).await { + eprintln!("sprout-desktop: post_connect_setup failed (degraded mode): {e}"); + } + + Ok(HuddleJoinInfo { + ephemeral_channel_id, + livekit_token: lk.token, + livekit_url: lk.url, + livekit_room: lk.room, + }) + } + Err(e) => { + // Rollback: just reset state to Idle — the relay auto-added us and + // the ephemeral channel has a TTL, so no manual leave is needed. + if let Ok(mut hs) = state.huddle_state.lock() { + hs.reset_preserving_generation(); + } + Err(e) + } + } +} + +/// Shut down all pipelines and reset huddle state to Idle. +/// +/// Used by both `leave_huddle` and `end_huddle` to avoid duplicating the +/// shutdown-then-reset sequence. +fn teardown_huddle(state: &AppState) -> Result<(), String> { + // Take pipeline handles out of state and drop the lock before shutdown. + // Pipeline Drop impls join worker threads — this avoids blocking while + // the mutex is held (ONNX inference can take ~200ms). + let (old_stt, old_tts) = { + let mut hs = state.huddle()?; + // Increment generation first — this immediately invalidates any + // in-flight transcription task, even before pipelines shut down. + hs.session_generation.fetch_add(1, Ordering::Release); + let stt = hs.stt_pipeline.take(); + let tts = hs.tts_pipeline.take(); + hs.reset_preserving_generation(); + (stt, tts) + }; + // Shut down outside the lock — thread joins happen here. + if let Some(ref p) = old_stt { + p.shutdown(); + } + if let Some(ref p) = old_tts { + p.shutdown(); + } + // Drop the Arcs here (implicit) — triggers thread join via Drop. + drop(old_stt); + drop(old_tts); + // Notify frontend that we're back to Idle. + state.emit_huddle_state_changed(); + Ok(()) +} + +/// Emit HUDDLE_ENDED to the parent channel and archive the ephemeral channel. +/// +/// Both steps are best-effort — failures are logged but do not propagate. +/// Called from `leave_huddle` (auto-end path) and `end_huddle`. +async fn emit_end_and_archive( + parent_channel_id: &str, + ephemeral_channel_id: &str, + state: &AppState, +) { + if !parent_channel_id.is_empty() && !ephemeral_channel_id.is_empty() { + if let Ok(ended_builder) = + events::build_huddle_ended(parent_channel_id, ephemeral_channel_id) + { + if let Err(e) = submit_event(ended_builder, state).await { + eprintln!("sprout-desktop: huddle_ended event failed: {e}"); + } + } + } + if !ephemeral_channel_id.is_empty() { + if let Ok(uuid) = parse_channel_uuid(ephemeral_channel_id) { + if let Ok(archive_builder) = events::build_archive(uuid) { + if let Err(e) = submit_event(archive_builder, state).await { + eprintln!("sprout-desktop: archive ephemeral channel failed: {e}"); + } + } + } + } +} + +/// Leave the current huddle. +/// +/// Steps: +/// 1. Emit KIND_HUDDLE_PARTICIPANT_LEFT to the parent channel. +/// 2. Shut down the STT pipeline (Fix 5). +/// 3. Clear local huddle state. +#[tauri::command] +pub async fn leave_huddle(state: State<'_, AppState>) -> Result<(), String> { + let (parent_channel_id, ephemeral_channel_id) = { + let mut hs = state.huddle()?; + if hs.phase == HuddlePhase::Idle { + return Ok(()); // Nothing to leave. + } + hs.phase = HuddlePhase::Leaving; + ( + hs.parent_channel_id.clone().unwrap_or_default(), + hs.ephemeral_channel_id.clone().unwrap_or_default(), + ) + }; + + // Emit PARTICIPANT_LEFT (best-effort) — include own pubkey as p-tag. + let own_pubkey = state.keys.lock().ok().map(|k| k.public_key().to_hex()); + if !parent_channel_id.is_empty() && !ephemeral_channel_id.is_empty() { + if let Ok(left_builder) = events::build_huddle_participant_left( + &parent_channel_id, + &ephemeral_channel_id, + own_pubkey.as_deref(), + ) { + if let Err(e) = submit_event(left_builder, &state).await { + eprintln!("sprout-desktop: huddle_participant_left event failed: {e}"); + } + } + } + + // Auto-end: check if any human participants remain. If not, end the huddle + // (emit HUDDLE_ENDED + archive). If others remain, just remove self from + // membership so the participant roster stays accurate. + // + // We check BEFORE removing self — the relay counts us as a member until + // we leave. So "1 human remaining" means WE are the last one. + if !parent_channel_id.is_empty() && !ephemeral_channel_id.is_empty() { + let humans_remaining = count_human_members(&ephemeral_channel_id, &state) + .await + // On fetch failure, assume 2 humans remain (safe default). + // unwrap_or(1) would mean "I'm the last human" → triggers auto-archive, + // ending the huddle for everyone on a transient REST failure. Using 2 + // means we skip the auto-end path and just remove ourselves — the huddle + // stays alive and the next real leave will clean up correctly. + .unwrap_or(2); + + if humans_remaining <= 1 { + // We're the last human — end the huddle entirely. + // Archive subsumes leave (the channel is gone, membership is moot). + // This avoids the "cannot remove the last owner" relay error that + // build_leave hits when the creator is the sole remaining member. + eprintln!("sprout-desktop: last human left huddle — auto-ending"); + emit_end_and_archive(&parent_channel_id, &ephemeral_channel_id, &state).await; + } else { + // Other humans still in the huddle — just remove self from membership. + if let Ok(eph_uuid) = parse_channel_uuid(&ephemeral_channel_id) { + if let Ok(leave_builder) = events::build_leave(eph_uuid) { + if let Err(e) = submit_event(leave_builder, &state).await { + eprintln!("sprout-desktop: huddle leave ephemeral channel failed: {e}"); + } + } + } + } + } + + teardown_huddle(&state)?; + + Ok(()) +} + +/// End the current huddle (creator only). +/// +/// Steps: +/// 1. Emit KIND_HUDDLE_ENDED to the parent channel. +/// 2. Archive the ephemeral channel. +/// 3. Shut down the STT pipeline (Fix 5). +/// 4. Clear local huddle state. +#[tauri::command] +pub async fn end_huddle(force: Option, state: State<'_, AppState>) -> Result<(), String> { + let (parent_channel_id, ephemeral_channel_id) = { + let mut hs = state.huddle()?; + if hs.phase == HuddlePhase::Idle { + return Ok(()); // Nothing to end. + } + // Only the creator can end the huddle for everyone. Non-creators + // should use leave_huddle (which auto-ends if they're the last human). + // The `force` flag allows recovery when the creator has disconnected + // ungracefully — the UI should gate this behind a confirmation dialog. + if !hs.is_creator && !force.unwrap_or(false) { + return Err("only the huddle creator can end it — use leave_huddle instead".into()); + } + hs.phase = HuddlePhase::Leaving; + ( + hs.parent_channel_id.clone().unwrap_or_default(), + hs.ephemeral_channel_id.clone().unwrap_or_default(), + ) + }; + + emit_end_and_archive(&parent_channel_id, &ephemeral_channel_id, &state).await; + + teardown_huddle(&state)?; + + Ok(()) +} + +/// Confirm that the frontend has established LiveKit + AudioWorklet. +/// Transitions from Connected → Active. No-op if already Active. +#[tauri::command] +pub async fn confirm_huddle_active(state: State<'_, AppState>) -> Result<(), String> { + let transitioned = { + let mut hs = state.huddle()?; + match hs.phase { + HuddlePhase::Connected => { + hs.phase = HuddlePhase::Active; + true + } + HuddlePhase::Active => false, // Already active — idempotent. + ref other => return Err(format!("cannot confirm active: phase is {:?}", other)), + } + }; + if transitioned { + state.emit_huddle_state_changed(); + } + Ok(()) +} + +/// Return the current HuddleState (serialized for the frontend). +#[tauri::command] +pub fn get_huddle_state(state: State<'_, AppState>) -> Result { + let hs = state.huddle()?; + Ok(hs.clone()) +} + +/// Return the authoritative list of agent (bot-role) pubkeys in the active huddle. +/// +/// Fetches from the relay's channel membership API — works for both creators +/// and joiners. Returns `Ok(Vec::new())` if no huddle is active. Returns +/// `Err` on relay fetch failure so the frontend can keep `agentsLoaded = false` +/// rather than treating a failed lookup as "zero agents". +#[tauri::command] +pub async fn get_huddle_agent_pubkeys(state: State<'_, AppState>) -> Result, String> { + let eph_id = { + let hs = state.huddle()?; + hs.ephemeral_channel_id.clone() + }; + match eph_id { + Some(id) => fetch_channel_members(&id, Some("bot"), &state).await, + None => Ok(Vec::new()), + } +} + +/// Maximum IPC audio batch size: 100 KB. +/// A 100 ms batch at 48 kHz mono f32 is ~19 KB; 100 KB allows headroom +/// without letting a malformed IPC call allocate unbounded memory. +const MAX_AUDIO_BATCH_BYTES: usize = 100 * 1024; + +/// Receive raw PCM audio bytes from the AudioWorklet and feed the STT pipeline. +/// +/// Expects a raw binary body of f32 LE samples at 48 kHz mono. +/// If no STT pipeline is active, the bytes are silently discarded. +#[tauri::command] +pub fn push_audio_pcm( + request: tauri::ipc::Request<'_>, + state: State<'_, AppState>, +) -> Result<(), String> { + match request.body() { + tauri::ipc::InvokeBody::Raw(bytes) => { + if bytes.len() > MAX_AUDIO_BATCH_BYTES { + return Err(format!( + "audio batch too large: {} bytes (max {})", + bytes.len(), + MAX_AUDIO_BATCH_BYTES + )); + } + if let Ok(hs) = state.huddle() { + if let Some(ref pipeline) = hs.stt_pipeline { + pipeline.push_audio(bytes.to_vec())?; + } + } + Ok(()) + } + _ => Err("expected raw binary body".to_string()), + } +} + +/// Hot-start: check if voice models just finished downloading during an active +/// huddle and start the corresponding pipelines. +/// +/// Called by the frontend on a timer or after model status changes. No-op if +/// the huddle is not active or pipelines are already running. +#[tauri::command] +pub async fn check_pipeline_hotstart(state: State<'_, AppState>) -> Result<(), String> { + let (is_active, ephemeral_channel_id) = { + let hs = state.huddle()?; + ( + matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active), + hs.ephemeral_channel_id.clone(), + ) + }; + + if !is_active { + return Ok(()); + } + + // Detect dead pipelines: if the worker thread has exited (init failure or crash), + // clear the pipeline handle so hot-start can retry on the next cycle. + { + let mut hs = state.huddle()?; + if let Some(ref p) = hs.stt_pipeline { + if p.is_finished() { + hs.stt_pipeline = None; + } + } + if let Some(ref p) = hs.tts_pipeline { + if p.is_finished() { + hs.tts_pipeline = None; + } + } + } + // Re-read after potential cleanup. + let (has_stt, has_tts) = { + let hs = state.huddle()?; + (hs.stt_pipeline.is_some(), hs.tts_pipeline.is_some()) + }; + + // Check if models just became ready (one-shot flags). + let moonshine_ready = models::global_model_manager() + .map(|m| m.take_moonshine_ready()) + .unwrap_or(false); + let kokoro_ready = models::global_model_manager() + .map(|m| m.take_kokoro_ready()) + .unwrap_or(false); + + // Start TTS first (so STT can capture tts_cancel). + if !has_tts && (kokoro_ready || models::is_kokoro_ready()) { + if let Err(e) = maybe_start_tts_pipeline(&state).await { + eprintln!("sprout-desktop: TTS hotstart failed: {e}"); + } + } + + if !has_stt && (moonshine_ready || models::is_moonshine_ready()) { + if let Some(eph_id) = &ephemeral_channel_id { + if let Err(e) = maybe_start_stt_pipeline(&state, eph_id).await { + eprintln!("sprout-desktop: STT hotstart failed: {e}"); + } + } + } + + // Periodically refresh agent_pubkeys from relay membership. + // This catches mid-huddle agent additions/removals by other participants, + // keeping STT p-tags authoritative throughout the session. + // Throttled to every 15 s (not on every 5 s hotstart poll). + // + // NOTE: The frontend ALSO polls agent membership independently (every 10 s + // via get_huddle_agent_pubkeys). This is intentional — the two polls have + // different failure semantics: + // - Rust (here): preserves stale list on failure (STT p-tags should not + // disappear on a transient network blip). + // - React (HuddleContext.tsx): clears list on failure (TTS authorization + // must fail-closed — never speak from a stale agent list). + // + // On Ok: always replace (even with empty — agents may have been removed). + // On Err: preserve the existing list (transient failure shouldn't zero it). + if let Some(eph_id) = &ephemeral_channel_id { + let should_refresh = { + let hs = state.huddle()?; + match hs.last_agent_refresh { + None => true, + Some(t) => t.elapsed() >= std::time::Duration::from_secs(15), + } + }; + if should_refresh { + // Fetch agents (for STT p-tags) and all members (for participant list). + // Sequential — tokio::join! requires the `macros` feature. + // Only update the throttle timestamp when at least one fetch succeeds, + // so transient failures retry immediately on the next poll cycle. + // Fetch both lists before acquiring the lock — no lock held across await. + let fresh_agents = fetch_channel_members(eph_id, Some("bot"), &state) + .await + .ok(); + let fresh_members = fetch_channel_members(eph_id, None, &state).await.ok(); + + if fresh_agents.is_some() || fresh_members.is_some() { + let mut hs = state.huddle()?; + if let Some(agents) = fresh_agents { + *hs.agent_pubkeys.lock().unwrap_or_else(|e| e.into_inner()) = agents; + } + if let Some(members) = fresh_members { + hs.participants = members; + } + hs.last_agent_refresh = Some(std::time::Instant::now()); + } + } + } + + Ok(()) +} + +/// Start the STT pipeline for the active huddle. +/// +/// Delegates to `maybe_start_stt_pipeline` — returns `Err` if models are not +/// ready or no huddle is active. Safe to call multiple times: replaces the +/// existing pipeline if already running. +#[tauri::command] +pub async fn start_stt_pipeline(state: State<'_, AppState>) -> Result<(), String> { + let ephemeral_channel_id = { + let hs = state.huddle()?; + hs.ephemeral_channel_id + .clone() + .ok_or("no active huddle — start or join a huddle first")? + }; + + match maybe_start_stt_pipeline(&state, &ephemeral_channel_id).await { + Ok(true) => Ok(()), + Ok(false) => Err("Moonshine model not ready".to_string()), + Err(e) => Err(e), + } +} + +/// Trigger a background download of voice models (Moonshine STT + Kokoro TTS). +/// +/// Returns immediately — downloads run in tokio background tasks. +/// Poll `get_model_status` to track progress. +/// Safe to call multiple times: no-op if already downloading or ready. +#[tauri::command] +pub async fn download_voice_models(state: State<'_, AppState>) -> Result<(), String> { + let manager = models::global_model_manager() + .ok_or("model manager unavailable (home directory could not be resolved)")?; + manager.start_moonshine_download(state.http_client.clone()); + manager.start_kokoro_download(state.http_client.clone()); + Ok(()) +} + +/// Return the current download status for all voice models. +#[tauri::command] +pub fn get_model_status(_state: State<'_, AppState>) -> Result { + let manager = models::global_model_manager() + .ok_or("model manager unavailable (home directory could not be resolved)")?; + Ok(models::VoiceModelStatus { + moonshine: manager.moonshine_status(), + + kokoro: manager.kokoro_status(), + }) +} + +/// Enable or disable TTS output. +/// +/// When disabled, the TTS pipeline is shut down and audio output stops. +/// When re-enabled, the pipeline is restarted if Kokoro models are available. +/// +/// Takes the pipeline handle out of the lock before calling shutdown() — the +/// thread join in Drop can block for ~200 ms (ONNX inference) and we don't +/// want to hold the HuddleState mutex during that time. +#[tauri::command] +pub async fn set_tts_enabled(enabled: bool, state: State<'_, AppState>) -> Result<(), String> { + let old_pipeline = { + let mut hs = state.huddle()?; + hs.tts_enabled = enabled; + if !enabled { + hs.tts_pipeline.take() // Take out of lock. + } else { + None + } + }; + // Shut down outside the lock — thread join happens here. + if let Some(ref pipeline) = old_pipeline { + pipeline.shutdown(); + } + drop(old_pipeline); + + if enabled { + // Re-start TTS pipeline if models are available and huddle is active. + let phase = { + let hs = state.huddle()?; + hs.phase.clone() + }; + if matches!(phase, HuddlePhase::Connected | HuddlePhase::Active) { + if let Err(e) = maybe_start_tts_pipeline(&state).await { + eprintln!("sprout-desktop: TTS pipeline restart failed: {e}"); + } + } + } + + Ok(()) +} + +/// Speak an agent message via TTS. +/// +/// Maximum text length accepted for TTS synthesis. +/// ~2000 chars ≈ 1–2 minutes of speech. Longer messages are truncated. +const MAX_TTS_TEXT_LEN: usize = 2000; + +/// Called by the WebView when it receives an incoming agent kind:9 message. +/// Lazily starts the TTS pipeline if models are ready but the pipeline hasn't +/// been created yet (e.g. models finished downloading after huddle started). +/// +/// No-op if TTS is disabled or models aren't ready. +#[tauri::command] +pub async fn speak_agent_message(text: String, state: State<'_, AppState>) -> Result<(), String> { + // Truncate oversized messages — agents shouldn't monologue in a voice huddle. + // Use char count (not byte length) to avoid panicking on multi-byte UTF-8. + let text = if text.chars().count() > MAX_TTS_TEXT_LEN { + let mut truncated: String = text.chars().take(MAX_TTS_TEXT_LEN).collect(); + truncated.push_str("... message truncated."); + truncated + } else { + text + }; + + let needs_pipeline = { + let hs = state.huddle()?; + hs.tts_enabled + && hs.tts_pipeline.is_none() + && matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active) + }; + + // Lazy-start: models may have finished downloading after the huddle began. + if needs_pipeline { + if let Err(e) = maybe_start_tts_pipeline(&state).await { + eprintln!("sprout-desktop: TTS lazy-start failed: {e}"); + } + } + + let hs = state.huddle()?; + if hs.tts_enabled { + if let Some(ref pipeline) = hs.tts_pipeline { + pipeline.speak(text)?; + } + } + Ok(()) +} + +/// Add an agent to the active huddle. +/// +/// Steps: +/// 1. Validates the huddle is in the Connected or Active phase. +/// 2. Adds the agent to both the ephemeral and parent channels (kind:9000). +/// 3. Only appends the agent pubkey to `agent_pubkeys` if the ephemeral add +/// succeeded — failed adds (policy rejection) are NOT p-tagged. +/// +/// Returns a structured `AgentAddResult` so the frontend can surface +/// parent-channel errors without treating them as hard failures. +/// +/// The running ACP process for this agent auto-subscribes when it receives +/// the kind:9000 membership notification — no separate process spawn needed. +#[tauri::command] +pub async fn add_agent_to_huddle( + agent_pubkey: String, + state: State<'_, AppState>, +) -> Result { + validate_pubkey_hex(&agent_pubkey)?; + + let (eph_id, parent_id) = { + let hs = state.huddle()?; + if !matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active) { + return Err("no active huddle".to_string()); + } + + // Enforce agent cap on incremental adds too. + let current_agent_count = hs + .agent_pubkeys + .lock() + .unwrap_or_else(|e| e.into_inner()) + .len(); + if current_agent_count >= MAX_HUDDLE_AGENTS { + return Err(format!( + "agent limit reached: {} (max {})", + current_agent_count, MAX_HUDDLE_AGENTS + )); + } + + let eph = hs + .ephemeral_channel_id + .clone() + .ok_or("no ephemeral channel")?; + let parent = hs.parent_channel_id.clone().ok_or("no parent channel")?; + (eph, parent) + }; + + let eph_uuid = Uuid::parse_str(&eph_id).map_err(|e| e.to_string())?; + let parent_uuid = Uuid::parse_str(&parent_id).map_err(|e| e.to_string())?; + + // Returns Err only if the ephemeral add fails — parent failure is in the result. + let result = agents::add_agent_to_huddle(eph_uuid, parent_uuid, &agent_pubkey, &state).await?; + + // Ephemeral add succeeded — safe to register for p-tagging. + // Clone the Arc first so we can drop the outer HuddleState lock before + // acquiring the inner pubkeys lock (avoids the E0597 borrow-checker error). + { + let agent_pubkeys_arc = { + let hs = state.huddle()?; + Arc::clone(&hs.agent_pubkeys) + }; + let mut pubkeys = agent_pubkeys_arc.lock().unwrap_or_else(|e| e.into_inner()); + if !pubkeys.contains(&agent_pubkey) { + pubkeys.push(agent_pubkey.clone()); + } + } + + // No guidelines re-post needed — the agent sees the original kind:48106 + // guidelines via EOSE replay when it subscribes to the ephemeral channel. + + // Also add the agent to the visible participants list. + { + let mut hs = state.huddle()?; + if !hs.participants.contains(&agent_pubkey) { + hs.participants.push(agent_pubkey); + } + } + + Ok(result) +} diff --git a/desktop/src-tauri/src/huddle/models.rs b/desktop/src-tauri/src/huddle/models.rs new file mode 100644 index 00000000000..e264493dcb1 --- /dev/null +++ b/desktop/src-tauri/src/huddle/models.rs @@ -0,0 +1,762 @@ +//! Model download manager for STT (Moonshine) and TTS (Kokoro) models. +//! +//! Mental model: +//! app launch → start_moonshine_download (background) → ~/.sprout/models/moonshine-tiny/ +//! app launch → start_kokoro_download (background) → ~/.sprout/models/kokoro/ +//! STT pipeline → is_moonshine_ready() → moonshine_model_dir() → run inference +//! TTS pipeline → is_kokoro_ready() → kokoro_model_dir() → run synthesis +//! +//! Models are downloaded once and cached. A version manifest (`.sprout-model-manifest`) +//! is written alongside model files — if the on-disk version doesn't match the +//! compiled-in version, the model is re-downloaded. + +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +// ── Integrity verification ──────────────────────────────────────────────────── +// +// All model artifacts are verified against pinned SHA-256 hashes before +// installation. This is defense-in-depth: HTTPS protects the transport, +// hashes protect the content. +// +// To recompute hashes: download each file, run `shasum -a 256 `, and +// update the corresponding constant. + +/// SHA-256 hash of the Moonshine archive (sherpa-onnx-moonshine-tiny-en-int8.tar.bz2). +/// Computed from a known-good download. Update when upgrading model versions. +const MOONSHINE_ARCHIVE_SHA256: &str = + "d5fe6ec4334fef36255b2a4010412cad4c007e33103fec62fb5d17cad88086f2"; + +/// SHA-256 hashes for individual Kokoro model files. +/// Computed from known-good downloads. Update when upgrading model versions. +/// +/// model.onnx (model_q8f16.onnx, 86 MB): +/// curl -sL "https://huggingface.co/onnx-community/Kokoro-82M-v1.0-ONNX/resolve/main/onnx/model_q8f16.onnx" | shasum -a 256 +#[rustfmt::skip] +const KOKORO_FILE_HASHES: &[(&str, &str)] = &[ + ("model.onnx", "04c658aec1b6008857c2ad10f8c589d4180d0ec427e7e6118ceb487e215c3cd0"), + ("af_heart.bin", "d583ccff3cdca2f7fae535cb998ac07e9fcb90f09737b9a41fa2734ec44a8f0b"), + ("us_gold.json", "dc414872a49a28ae6c141463d502fd945f3b2fde040484fdc47d00cc4612686f"), + ("us_silver.json", "de8f67be911bb6c659187b4a65fd966b6a30e56350e0f790d763210b053ac475"), + ("cmudict.dict", "81917843c7f44ce2b094ac63873c2c7a4cf802040792c455ba3ca406891c3d22"), +]; + +// ── Model versioning ────────────────────────────────────────────────────────── +// +// A version manifest is written alongside model files after successful download. +// If the on-disk manifest doesn't match the compiled-in version, the model is +// considered stale and re-downloaded. Increment when upgrading model files. + +/// Model manifest version for Moonshine. Increment when upgrading model files. +const MOONSHINE_MODEL_VERSION: &str = "1"; + +/// Model manifest version for Kokoro. Increment when upgrading model files. +const KOKORO_MODEL_VERSION: &str = "1"; + +/// Filename for the version manifest written alongside model files. +const MANIFEST_FILENAME: &str = ".sprout-model-manifest"; + +// ── Constants ───────────────────────────────────────────────────────────────── + +/// Maximum expected Moonshine archive size (200 MB — actual is ~50 MB). +const MAX_MOONSHINE_DOWNLOAD_BYTES: u64 = 200 * 1024 * 1024; + +/// Maximum expected Kokoro file size (200 MB per file — model is 86 MB). +const MAX_KOKORO_FILE_BYTES: u64 = 200 * 1024 * 1024; + +const MOONSHINE_DOWNLOAD_URL: &str = + "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/\ + sherpa-onnx-moonshine-tiny-en-int8.tar.bz2"; + +/// Subdirectory name produced by `tar xjf` on the archive. +const MOONSHINE_ARCHIVE_SUBDIR: &str = "sherpa-onnx-moonshine-tiny-en-int8"; + +/// Final directory name under `~/.sprout/models/`. +const MOONSHINE_MODEL_DIR_NAME: &str = "moonshine-tiny"; + +/// All files that must be present for the model to be considered ready. +const MOONSHINE_EXPECTED_FILES: &[&str] = &[ + "preprocess.onnx", + "encode.int8.onnx", + "cached_decode.int8.onnx", + "uncached_decode.int8.onnx", + "tokens.txt", +]; + +// ── Kokoro TTS model ───────────────────────────────────────────────────────── + +/// HuggingFace base URL for Kokoro ONNX model files. +const KOKORO_HF_BASE: &str = + "https://huggingface.co/onnx-community/Kokoro-82M-v1.0-ONNX/resolve/main"; + +/// Misaki G2P lexicons — pinned to commit fba1236 for reproducibility. +/// Gold = curated pronunciations. Silver = broader coverage (93K words). +/// Both are needed: gold is checked first, silver catches common words gold misses. +const KOKORO_LEXICON_GOLD_URL: &str = + "https://raw.githubusercontent.com/hexgrad/misaki/fba1236/misaki/data/us_gold.json"; +const KOKORO_LEXICON_SILVER_URL: &str = + "https://raw.githubusercontent.com/hexgrad/misaki/fba1236/misaki/data/us_silver.json"; + +/// CMU Pronouncing Dictionary — 135K entries including inflected forms. +/// BSD 2-Clause license (Carnegie Mellon University). Compatible with Apache-2.0. +const KOKORO_CMUDICT_URL: &str = + "https://raw.githubusercontent.com/cmusphinx/cmudict/master/cmudict.dict"; + +/// Final directory name under `~/.sprout/models/`. +const KOKORO_MODEL_DIR_NAME: &str = "kokoro"; + +/// All files that must be present for Kokoro to be considered ready. +const KOKORO_EXPECTED_FILES: &[&str] = &[ + "model.onnx", + "af_heart.bin", + "us_gold.json", + "us_silver.json", + "cmudict.dict", +]; + +// ── Status types ────────────────────────────────────────────────────────────── + +/// Download/readiness status for a single model. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ModelStatus { + NotDownloaded, + Downloading { progress_percent: u8 }, + Ready, + Error(String), +} + +/// Combined status for all voice models (returned to the frontend). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VoiceModelStatus { + pub moonshine: ModelStatus, + pub kokoro: ModelStatus, +} + +// ── Safe archive extraction ─────────────────────────────────────────────────── + +/// Extract a .tar.bz2 archive safely using Rust-native crates. +/// +/// The `tar` crate rejects path traversal (absolute paths, `..` components) +/// by default in `unpack()`. We add an explicit pre-check as defense-in-depth. +fn extract_archive(archive_path: &Path, dest_dir: &Path) -> Result<(), String> { + use bzip2::read::BzDecoder; + use std::fs::File; + use tar::Archive; + + let file = File::open(archive_path).map_err(|e| format!("open archive: {e}"))?; + let decoder = BzDecoder::new(file); + let mut archive = Archive::new(decoder); + + // Pre-validate: check all entries for path safety before extracting anything. + // This is defense-in-depth — the tar crate also rejects traversal in unpack(). + { + let file2 = + File::open(archive_path).map_err(|e| format!("open archive for validation: {e}"))?; + let decoder2 = BzDecoder::new(file2); + let mut check_archive = Archive::new(decoder2); + for entry in check_archive + .entries() + .map_err(|e| format!("read archive entries: {e}"))? + { + let entry = entry.map_err(|e| format!("archive entry: {e}"))?; + let path = entry.path().map_err(|e| format!("entry path: {e}"))?; + let path_str = path.to_string_lossy(); + + // Reject absolute paths. + if path.is_absolute() { + return Err(format!("archive contains absolute path: {path_str}")); + } + // Reject path traversal. + for component in path.components() { + if matches!(component, std::path::Component::ParentDir) { + return Err(format!("archive contains path traversal: {path_str}")); + } + } + // Reject symlinks. + if entry.header().entry_type().is_symlink() + || entry.header().entry_type().is_hard_link() + { + return Err(format!("archive contains symlink/hardlink: {path_str}")); + } + } + } + + // Safe to extract — all entries validated. + archive + .unpack(dest_dir) + .map_err(|e| format!("extract archive: {e}"))?; + + Ok(()) +} + +// ── Hash verification ───────────────────────────────────────────────────────── + +/// Compute SHA-256 hash of a file. Returns lowercase hex string. +async fn sha256_file(path: &Path) -> Result { + let bytes = tokio::fs::read(path) + .await + .map_err(|e| format!("read file for hash: {e}"))?; + let hash = Sha256::digest(&bytes); + Ok(hex::encode(hash)) +} + +// ── Shared HTTP helpers ─────────────────────────────────────────────────────── + +/// Send a GET request and return the response, or a descriptive error. +async fn fetch_url( + client: &reqwest::Client, + url: &str, + label: &str, +) -> Result { + let response = client + .get(url) + .send() + .await + .map_err(|e| format!("download {label} request failed: {e}"))?; + if !response.status().is_success() { + return Err(format!( + "download {label} HTTP {}: {}", + response.status().as_u16(), + response.status().canonical_reason().unwrap_or("unknown"), + )); + } + Ok(response) +} + +/// Create (or recreate) a temp directory, removing any stale one first. +async fn fresh_temp_dir(path: &Path) -> Result<(), String> { + if path.exists() { + tokio::fs::remove_dir_all(path) + .await + .map_err(|e| format!("remove stale temp dir: {e}"))?; + } + tokio::fs::create_dir_all(path) + .await + .map_err(|e| format!("create temp dir: {e}")) +} + +/// Stream an HTTP response to a file with progress reporting and size limits. +/// +/// Calls `progress_fn(bytes_downloaded, content_length)` after each chunk. +/// Returns the total number of bytes written. +async fn download_file( + response: reqwest::Response, + dest: &Path, + max_bytes: u64, + label: &str, + progress_fn: F, +) -> Result +where + F: Fn(u64, Option), +{ + use tokio::io::AsyncWriteExt; + + let content_length = response.content_length(); + if let Some(total) = content_length { + if total > max_bytes { + return Err(format!( + "download {label} too large: {total} bytes (max {max_bytes})" + )); + } + } + + let mut file = tokio::fs::File::create(dest) + .await + .map_err(|e| format!("create {label}: {e}"))?; + let mut downloaded: u64 = 0; + let mut response = response; + + while let Some(chunk) = response + .chunk() + .await + .map_err(|e| format!("download {label} stream error: {e}"))? + { + downloaded += chunk.len() as u64; + if downloaded > max_bytes { + let _ = tokio::fs::remove_file(dest).await; + return Err(format!( + "download {label} exceeded max size: {downloaded} bytes (max {max_bytes})" + )); + } + file.write_all(&chunk) + .await + .map_err(|e| format!("write {label}: {e}"))?; + progress_fn(downloaded, content_length); + } + + file.flush() + .await + .map_err(|e| format!("flush {label}: {e}"))?; + Ok(downloaded) +} + +// ── ModelSlot ───────────────────────────────────────────────────────────────── + +/// Per-model state + config. `ModelManager` owns two of these (moonshine, kokoro). +#[derive(Clone)] +struct ModelSlot { + dir_name: &'static str, // subdir under ~/.sprout/models/ + expected_files: &'static [&'static str], // files required for "ready" + version: &'static str, // manifest version; increment to force re-download + status: Arc>, + just_ready: Arc, // fires once when download completes +} + +impl ModelSlot { + fn new( + dir_name: &'static str, + expected_files: &'static [&'static str], + version: &'static str, + ) -> Self { + Self { + dir_name, + expected_files, + version, + status: Arc::new(Mutex::new(ModelStatus::NotDownloaded)), + just_ready: Arc::new(AtomicBool::new(false)), + } + } + + fn model_dir(&self, models_dir: &Path) -> PathBuf { + models_dir.join(self.dir_name) + } + + fn is_ready(&self, models_dir: &Path) -> bool { + let dir = self.model_dir(models_dir); + std::fs::read_to_string(dir.join(MANIFEST_FILENAME)) + .map(|v| v.trim() == self.version) + .unwrap_or(false) + && self.expected_files.iter().all(|f| dir.join(f).is_file()) + } + + fn dir_if_ready(&self, models_dir: &Path) -> Option { + self.is_ready(models_dir) + .then(|| self.model_dir(models_dir)) + } + + fn status(&self) -> ModelStatus { + self.status + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone() + } + fn set_status(&self, s: ModelStatus) { + *self.status.lock().unwrap_or_else(|e| e.into_inner()) = s; + } + fn take_ready(&self) -> bool { + self.just_ready.swap(false, Ordering::AcqRel) + } + + /// Spawn a background download task if not already ready or downloading. + fn start_download( + &self, + models_dir: &Path, + http_client: reqwest::Client, + name: &'static str, + download_fn: F, + ) where + F: FnOnce(reqwest::Client) -> Fut + Send + 'static, + Fut: std::future::Future> + Send, + { + if self.is_ready(models_dir) { + self.set_status(ModelStatus::Ready); + return; + } + { + let mut st = self.status.lock().unwrap_or_else(|e| e.into_inner()); + match *st { + ModelStatus::Downloading { .. } | ModelStatus::Ready => return, + _ => {} + } + *st = ModelStatus::Downloading { + progress_percent: 0, + }; + } + let slot = self.clone(); + // Use tauri::async_runtime::spawn (not tokio::spawn) because this may + // be called from the Tauri setup callback before the main Tokio runtime + // is accessible on the current thread. Tauri's runtime is always available. + tauri::async_runtime::spawn(async move { + if let Err(e) = download_fn(http_client).await { + eprintln!("sprout-desktop: {name} download failed: {e}"); + slot.set_status(ModelStatus::Error(e)); + } + }); + } + + /// Verify files in `source_dir`, atomic-swap into final location, write manifest, signal ready. + /// `temp_cleanup`: optional extra dir to remove (e.g. outer extraction dir for Moonshine). + async fn verify_and_install( + &self, + models_dir: &Path, + source_dir: &Path, + temp_cleanup: Option<&Path>, + ) -> Result<(), String> { + let missing: Vec<&str> = self + .expected_files + .iter() + .filter(|&&f| !source_dir.join(f).is_file()) + .copied() + .collect(); + if !missing.is_empty() { + return Err(format!( + "model verification failed — missing: {}", + missing.join(", ") + )); + } + + let final_dir = self.model_dir(models_dir); + let backup_dir = final_dir.with_extension("old"); + + if final_dir.exists() { + if backup_dir.exists() { + let _ = tokio::fs::remove_dir_all(&backup_dir).await; + } + tokio::fs::rename(&final_dir, &backup_dir) + .await + .map_err(|e| format!("backup old model: {e}"))?; + } + if let Err(e) = tokio::fs::rename(source_dir, &final_dir).await { + if backup_dir.exists() { + let _ = tokio::fs::rename(&backup_dir, &final_dir).await; + } + return Err(format!("install new model: {e}")); + } + + std::fs::write(final_dir.join(MANIFEST_FILENAME), self.version) + .map_err(|e| format!("write model manifest: {e}"))?; + let _ = tokio::fs::remove_dir_all(&backup_dir).await; + if let Some(extra) = temp_cleanup { + let _ = tokio::fs::remove_dir_all(extra).await; + } + + self.set_status(ModelStatus::Ready); + self.just_ready.store(true, Ordering::Release); + Ok(()) + } +} + +// ── ModelManager ────────────────────────────────────────────────────────────── + +/// Manages download and location of STT/TTS model files. +/// +/// Cheap to clone — all inner state is behind `Arc`. +#[derive(Clone)] +pub struct ModelManager { + /// `~/.sprout/models/` + models_dir: PathBuf, + moonshine: ModelSlot, + kokoro: ModelSlot, +} + +impl ModelManager { + /// Create a new `ModelManager` rooted at `~/.sprout/models/`. + /// + /// Returns `None` if the home directory cannot be resolved. + pub fn new() -> Option { + let models_dir = dirs::home_dir()?.join(".sprout").join("models"); + Some(Self { + models_dir, + moonshine: ModelSlot::new( + MOONSHINE_MODEL_DIR_NAME, + MOONSHINE_EXPECTED_FILES, + MOONSHINE_MODEL_VERSION, + ), + kokoro: ModelSlot::new( + KOKORO_MODEL_DIR_NAME, + KOKORO_EXPECTED_FILES, + KOKORO_MODEL_VERSION, + ), + }) + } + + // ── Moonshine accessors ─────────────────────────────────────────────────── + + /// Path to the Moonshine model directory, or `None` if not ready. + pub fn moonshine_model_dir(&self) -> Option { + self.moonshine.dir_if_ready(&self.models_dir) + } + /// `true` if all Moonshine files are present and the manifest version matches. + pub fn is_moonshine_ready(&self) -> bool { + self.moonshine.is_ready(&self.models_dir) + } + /// Current Moonshine download status. + pub fn moonshine_status(&self) -> ModelStatus { + self.moonshine.status() + } + /// Returns `true` once when Moonshine just became ready. Resets the flag. + pub fn take_moonshine_ready(&self) -> bool { + self.moonshine.take_ready() + } + + // ── Kokoro accessors ────────────────────────────────────────────────────── + + /// Path to the Kokoro model directory, or `None` if not ready. + pub fn kokoro_model_dir(&self) -> Option { + self.kokoro.dir_if_ready(&self.models_dir) + } + /// `true` if all Kokoro files are present and the manifest version matches. + pub fn is_kokoro_ready(&self) -> bool { + self.kokoro.is_ready(&self.models_dir) + } + /// Current Kokoro download status. + pub fn kokoro_status(&self) -> ModelStatus { + self.kokoro.status() + } + /// Returns `true` once when Kokoro just became ready. Resets the flag. + pub fn take_kokoro_ready(&self) -> bool { + self.kokoro.take_ready() + } + + // ── Download triggers ───────────────────────────────────────────────────── + + /// Start a background Moonshine download. No-op if already ready or downloading. + pub fn start_moonshine_download(&self, http_client: reqwest::Client) { + let manager = self.clone(); + self.moonshine.start_download( + &self.models_dir, + http_client, + "moonshine", + move |client| async move { manager.download_moonshine_model(client).await }, + ); + } + + /// Start a background Kokoro download (~87 MB). No-op if already ready or downloading. + pub fn start_kokoro_download(&self, http_client: reqwest::Client) { + let manager = self.clone(); + self.kokoro.start_download( + &self.models_dir, + http_client, + "kokoro", + move |client| async move { manager.download_kokoro_model(client).await }, + ); + } + + // ── Private download implementations ───────────────────────────────────── + + /// Download, extract, and verify the Moonshine model archive. + async fn download_moonshine_model(&self, http_client: reqwest::Client) -> Result<(), String> { + tokio::fs::create_dir_all(&self.models_dir) + .await + .map_err(|e| format!("create models dir: {e}"))?; + + let archive_path = self.models_dir.join("moonshine-tiny.tar.bz2"); + let temp_dir = self.models_dir.join("moonshine-tiny.tmp"); + + eprintln!("sprout-desktop: downloading Moonshine model from {MOONSHINE_DOWNLOAD_URL}"); + let response = fetch_url(&http_client, MOONSHINE_DOWNLOAD_URL, "moonshine archive").await?; + + let slot = self.moonshine.clone(); + let bytes = download_file( + response, + &archive_path, + MAX_MOONSHINE_DOWNLOAD_BYTES, + "moonshine archive", + |downloaded, content_length| { + if let Some(total) = content_length { + if total > 0 { + let pct = ((downloaded * 89) / total).min(89) as u8; + slot.set_status(ModelStatus::Downloading { + progress_percent: pct, + }); + } + } + }, + ) + .await?; + eprintln!("sprout-desktop: downloaded {bytes} bytes, wrote to disk"); + + // Verify archive integrity before extraction. + let hash = sha256_file(&archive_path).await?; + if hash != MOONSHINE_ARCHIVE_SHA256 { + let _ = tokio::fs::remove_file(&archive_path).await; + return Err(format!( + "Moonshine archive integrity check failed: expected {MOONSHINE_ARCHIVE_SHA256}, got {hash}" + )); + } + + self.moonshine.set_status(ModelStatus::Downloading { + progress_percent: 90, + }); + fresh_temp_dir(&temp_dir).await?; + + eprintln!("sprout-desktop: extracting Moonshine archive…"); + let (ap, td) = (archive_path.clone(), temp_dir.clone()); + tokio::task::spawn_blocking(move || extract_archive(&ap, &td)) + .await + .map_err(|e| format!("tar task panicked: {e}"))??; + + let extracted_subdir = temp_dir.join(MOONSHINE_ARCHIVE_SUBDIR); + if !extracted_subdir.is_dir() { + let _ = tokio::fs::remove_dir_all(&temp_dir).await; + return Err(format!( + "expected subdir '{MOONSHINE_ARCHIVE_SUBDIR}' not found after extraction" + )); + } + + // verify_and_install takes the subdir (actual model files); temp_cleanup removes outer dir. + if let Err(e) = self + .moonshine + .verify_and_install(&self.models_dir, &extracted_subdir, Some(&temp_dir)) + .await + { + let _ = tokio::fs::remove_dir_all(&temp_dir).await; + let _ = tokio::fs::remove_file(&archive_path).await; + return Err(e); + } + let _ = tokio::fs::remove_file(&archive_path).await; + + eprintln!( + "sprout-desktop: Moonshine model ready at {}", + self.moonshine.model_dir(&self.models_dir).display() + ); + Ok(()) + } + + /// Download and verify the Kokoro TTS model files from HuggingFace and GitHub. + /// + /// Downloads files into `~/.sprout/models/kokoro/`: + /// - `model.onnx` — Kokoro-82M mixed-precision ONNX (86 MB) + /// - `af_heart.bin` — best-quality American English voice embedding (510 KB) + /// - `us_gold.json` — Misaki G2P lexicon, pinned to commit fba1236 (3 MB) + /// + /// Files are written to a temp directory first, then moved atomically. + async fn download_kokoro_model(&self, http_client: reqwest::Client) -> Result<(), String> { + tokio::fs::create_dir_all(&self.models_dir) + .await + .map_err(|e| format!("create models dir: {e}"))?; + + let temp_dir = self.models_dir.join("kokoro.tmp"); + fresh_temp_dir(&temp_dir).await?; + + // (url, local_filename) + let downloads: &[(&str, &str)] = &[ + ( + &format!("{KOKORO_HF_BASE}/onnx/model_q8f16.onnx"), + "model.onnx", + ), + ( + &format!("{KOKORO_HF_BASE}/voices/af_heart.bin"), + "af_heart.bin", + ), + (KOKORO_LEXICON_GOLD_URL, "us_gold.json"), + (KOKORO_LEXICON_SILVER_URL, "us_silver.json"), + (KOKORO_CMUDICT_URL, "cmudict.dict"), + ]; + let total_files = downloads.len() as u32; + + for (i, (url, filename)) in downloads.iter().enumerate() { + eprintln!("sprout-desktop: downloading Kokoro {filename} from {url}"); + + let response = fetch_url(&http_client, url, filename).await.map_err(|e| { + let _ = std::fs::remove_dir_all(&temp_dir); + e + })?; + + let dest = temp_dir.join(filename); + let slot = self.kokoro.clone(); + let file_index = i as u32; + let bytes = download_file( + response, + &dest, + MAX_KOKORO_FILE_BYTES, + filename, + |downloaded, content_length| { + if let Some(total) = content_length { + if total > 0 { + let file_frac = downloaded as f64 / total as f64; + let base = (file_index as f64 / total_files as f64) * 89.0; + let span = 89.0 / total_files as f64; + let pct = (base + span * file_frac).min(89.0) as u8; + slot.set_status(ModelStatus::Downloading { + progress_percent: pct, + }); + } + } + }, + ) + .await + .map_err(|e| { + let _ = std::fs::remove_dir_all(&temp_dir); + e + })?; + eprintln!("sprout-desktop: downloaded {bytes} bytes ({filename}), wrote to disk"); + + // Verify file integrity against pinned hash. + if let Some(&(_, expected)) = KOKORO_FILE_HASHES.iter().find(|(n, _)| *n == *filename) { + let actual = sha256_file(&dest).await?; + if actual != expected { + let _ = tokio::fs::remove_dir_all(&temp_dir).await; + return Err(format!( + "Kokoro {filename} integrity check failed: expected {expected}, got {actual}" + )); + } + } + + // Ensure progress reflects file completion even without content-length. + let pct = (((i as u32 + 1) * 89) / total_files).min(89) as u8; + self.kokoro.set_status(ModelStatus::Downloading { + progress_percent: pct, + }); + } + + self.kokoro.set_status(ModelStatus::Downloading { + progress_percent: 90, + }); + + if let Err(e) = self + .kokoro + .verify_and_install(&self.models_dir, &temp_dir, None) + .await + { + let _ = tokio::fs::remove_dir_all(&temp_dir).await; + return Err(e); + } + + eprintln!( + "sprout-desktop: Kokoro model ready at {}", + self.kokoro.model_dir(&self.models_dir).display() + ); + Ok(()) + } +} + +// ── Process-global singleton ────────────────────────────────────────────────── + +static GLOBAL_MODEL_MANAGER: OnceLock> = OnceLock::new(); + +/// Return a reference to the process-global `ModelManager`. +pub fn global_model_manager() -> Option<&'static ModelManager> { + GLOBAL_MODEL_MANAGER.get_or_init(ModelManager::new).as_ref() +} + +// ── Standalone helpers ──────────────────────────────────────────────────────── + +/// Path to the Moonshine model directory, or `None` if not ready. +pub fn moonshine_model_dir() -> Option { + global_model_manager()?.moonshine_model_dir() +} + +/// `true` if all expected Moonshine model files are present on disk. +pub fn is_moonshine_ready() -> bool { + global_model_manager() + .map(|m| m.is_moonshine_ready()) + .unwrap_or(false) +} + +/// Path to the Kokoro model directory, or `None` if not ready. +pub fn kokoro_model_dir() -> Option { + global_model_manager()?.kokoro_model_dir() +} + +/// `true` if all expected Kokoro model files are present on disk. +pub fn is_kokoro_ready() -> bool { + global_model_manager() + .map(|m| m.is_kokoro_ready()) + .unwrap_or(false) +} diff --git a/desktop/src-tauri/src/huddle/pipeline.rs b/desktop/src-tauri/src/huddle/pipeline.rs new file mode 100644 index 00000000000..e98370bcb6a --- /dev/null +++ b/desktop/src-tauri/src/huddle/pipeline.rs @@ -0,0 +1,280 @@ +//! STT/TTS pipeline lifecycle management. +//! +//! Handles starting, hot-starting, and spawning transcription tasks for +//! the voice pipelines. Extracted from mod.rs to keep the command layer thin. + +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex, +}; + +use nostr::JsonUtil; +use uuid::Uuid; + +use crate::app_state::AppState; +use crate::events; + +use super::models; +use super::relay_api::{fetch_channel_members, parse_channel_uuid}; +use super::state::{HuddlePhase, VoiceInputMode}; +use super::stt; +use super::tts; + +pub(crate) async fn post_connect_setup( + state: &AppState, + ephemeral_channel_id: &str, +) -> Result<(), String> { + // Hydrate agent pubkeys from relay (authoritative — overrides local guess). + if let Ok(agents) = fetch_channel_members(ephemeral_channel_id, Some("bot"), state).await { + let hs = state.huddle()?; + *hs.agent_pubkeys.lock().unwrap_or_else(|e| e.into_inner()) = agents; + } + + // Hydrate participants from relay (authoritative state). + if let Ok(all_members) = fetch_channel_members(ephemeral_channel_id, None, state).await { + if !all_members.is_empty() { + let mut hs = state.huddle()?; + hs.participants = all_members; + } + } + + // Ensure voice models are downloading (idempotent). + if let Some(mgr) = models::global_model_manager() { + mgr.start_moonshine_download(state.http_client.clone()); + mgr.start_kokoro_download(state.http_client.clone()); + } + + // Start pipelines: TTS first (so STT can capture tts_cancel for barge-in). + if let Err(e) = maybe_start_tts_pipeline(state).await { + eprintln!("sprout-desktop: TTS pipeline failed to start: {e}"); + } + if let Err(e) = maybe_start_stt_pipeline(state, ephemeral_channel_id).await { + eprintln!("sprout-desktop: STT pipeline failed to start: {e}"); + } + + Ok(()) +} + +/// Attempt to start the STT pipeline if models are present. +/// +/// Returns `Ok(true)` if the pipeline was started, `Ok(false)` if models are +/// not ready (voice-only mode), or `Err` on a real failure. +/// +/// Creates the shared `tts_active` flag and passes it to the STT pipeline +/// for barge-in / echo gating. The same flag is later passed to the TTS +/// pipeline so it can signal when audio is playing. +pub(crate) async fn maybe_start_stt_pipeline( + state: &AppState, + ephemeral_channel_id: &str, +) -> Result { + if !models::is_moonshine_ready() { + return Ok(false); // Models not downloaded yet — voice-only mode. + } + let model_dir = + models::moonshine_model_dir().ok_or_else(|| "Moonshine model directory not found")?; + + let channel_uuid = parse_channel_uuid(ephemeral_channel_id)?; + + // Atomically claim the construction slot (mirrors tts_starting pattern). + { + let hs = state.huddle()?; + if hs.stt_starting.swap(true, Ordering::AcqRel) { + return Ok(false); // Another caller is already constructing. + } + } + + // Grab shared flags, agent pubkeys, and session generation from HuddleState. + // If replacing an existing pipeline, bump generation first so the old + // transcription task's next POST sees a stale generation and exits. + // Take the old pipeline OUT of the lock before dropping — Drop joins + // the worker thread (~200ms) and must not block under the mutex. + let (tts_active, tts_cancel, agent_pubkeys_arc, session_gen, ptt_active_for_stt, old_stt) = { + let mut hs = state.huddle()?; + // Invalidate any existing transcription task before replacing the pipeline. + if hs.stt_pipeline.is_some() { + hs.session_generation.fetch_add(1, Ordering::Release); + } + let old = hs.stt_pipeline.take(); + if let Some(ref p) = old { + p.shutdown(); + } + let ptt = if hs.voice_input_mode == VoiceInputMode::PushToTalk { + Some(Arc::clone(&hs.ptt_active)) + } else { + None + }; + ( + Arc::clone(&hs.tts_active), + Some(Arc::clone(&hs.tts_cancel)), + Arc::clone(&hs.agent_pubkeys), + Arc::clone(&hs.session_generation), + ptt, + old, + ) + }; + // Drop the old pipeline OUTSIDE the lock — thread join happens here. + drop(old_stt); + + let (pipeline, text_rx) = + match stt::SttPipeline::new(model_dir, tts_active, tts_cancel, ptt_active_for_stt) { + Ok(p) => p, + Err(e) => { + let hs = state.huddle()?; + hs.stt_starting.store(false, Ordering::Release); + return Err(e); + } + }; + let pipeline = Arc::new(pipeline); + + { + let mut hs = state.huddle()?; + hs.stt_starting.store(false, Ordering::Release); + // Phase check: huddle may have been torn down during construction. + if !matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active) { + return Ok(false); + } + hs.stt_pipeline = Some(Arc::clone(&pipeline)); + } + + spawn_transcription_task(text_rx, channel_uuid, agent_pubkeys_arc, session_gen, state); + Ok(true) +} + +/// Attempt to start the TTS pipeline if Kokoro models are present and TTS is enabled. +/// +/// Returns `Ok(true)` if the pipeline was started, `Ok(false)` if preconditions +/// aren't met (model not ready, pipeline exists, TTS disabled), or `Err` on failure. +/// +/// Uses `tts_starting` sentinel to prevent TOCTOU races: two concurrent callers +/// (e.g. `check_pipeline_hotstart` + `speak_agent_message` lazy-start) could both +/// pass the `is_some()` check, both construct pipelines, and the loser's thread +/// leaks ~200MB of ONNX sessions. The sentinel is set under the lock before +/// releasing it for the expensive construction step. +pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result { + if !models::is_kokoro_ready() { + return Ok(false); // Kokoro not downloaded yet — TTS unavailable. + } + + let model_dir = match models::kokoro_model_dir() { + Some(d) => d, + None => return Ok(false), + }; + + // Atomically check preconditions and claim the construction slot. + // The sentinel prevents a second caller from starting construction + // while we're building outside the lock. + let (tts_active, tts_cancel) = { + let hs = state.huddle()?; + if hs.tts_pipeline.is_some() { + return Ok(false); + } + if !hs.tts_enabled { + return Ok(false); + } + if hs.tts_starting.swap(true, Ordering::AcqRel) { + return Ok(false); // Another caller is already constructing. + } + (Arc::clone(&hs.tts_active), Arc::clone(&hs.tts_cancel)) + }; + + // Construct outside the lock — this spawns the TTS worker thread and + // loads ONNX sessions (~200ms). If this fails, clear the sentinel. + let pipeline = match tts::TtsPipeline::new(model_dir, tts_active, tts_cancel) { + Ok(p) => Arc::new(p), + Err(e) => { + let hs = state.huddle()?; + hs.tts_starting.store(false, Ordering::Release); + return Err(e); + } + }; + + { + let mut hs = state.huddle()?; + hs.tts_starting.store(false, Ordering::Release); + // Phase check: huddle may have been torn down during construction. + if !matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active) { + return Ok(false); + } + // Final check: another path may have created a pipeline while we were constructing. + if hs.tts_pipeline.is_some() { + return Ok(false); + } + hs.tts_pipeline = Some(pipeline); + } + + Ok(true) +} + +/// Spawn a tokio task that reads text_rx and posts kind:9 events. +/// +/// Fix 1: `agent_pubkeys_arc` is an `Arc>>` cloned from +/// `HuddleState` — the task reads it at post time so p-tags are always +/// current, not a stale snapshot. +/// Fix 3: no `.unwrap()` on mutex — poisoned locks are recovered gracefully. +/// Fix 4: `text_rx` is a `tokio::sync::mpsc::Receiver` — fully async `.recv().await` +/// never blocks a Tokio worker thread (unlike std `recv_timeout`). +pub(crate) fn spawn_transcription_task( + mut text_rx: tokio::sync::mpsc::Receiver, + channel_uuid: Uuid, + agent_pubkeys_arc: Arc>>, + session_generation: Arc, + state: &AppState, +) { + // Capture the current generation at spawn time. + let spawned_gen = session_generation.load(Ordering::Acquire); + + let http_client = state.http_client.clone(); + let keys = match state.keys.lock() { + Ok(k) => k.clone(), + Err(_) => return, + }; + let configured_api_token = state.configured_api_token.clone(); + + tauri::async_runtime::spawn(async move { + // recv().await yields (not blocks) until text arrives or sender is dropped. + // When the STT worker exits and drops its Sender, recv() returns None → loop ends. + while let Some(t) = text_rx.recv().await { + if t.is_empty() { + continue; + } + + // Session guard: if the generation has changed, this task is stale. + // Drop the transcript silently — the huddle has ended or been replaced. + if session_generation.load(Ordering::Acquire) != spawned_gen { + break; // Exit the loop entirely — no more posts from this task. + } + + // Fix 1: read current agent pubkeys at post time. + let agent_pubkeys: Vec = agent_pubkeys_arc + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(); + + let p_tags: Vec<&str> = agent_pubkeys.iter().map(|s| s.as_str()).collect(); + let builder = match events::build_message(channel_uuid, &t, None, &p_tags, &[]) { + Ok(b) => b, + Err(e) => { + eprintln!("sprout-desktop: STT build_message: {e}"); + continue; + } + }; + let event = match builder.sign_with_keys(&keys) { + Ok(e) => e, + Err(e) => { + eprintln!("sprout-desktop: STT sign event: {e}"); + continue; + } + }; + let event_json = event.as_json(); + let api_token_ref = configured_api_token.as_deref(); + let pubkey_hex = keys.public_key().to_hex(); + + if let Err(e) = + crate::events::post_event_raw(&http_client, api_token_ref, &pubkey_hex, event_json) + .await + { + eprintln!("sprout-desktop: STT kind:9 post failed: {e}"); + } + } + }); +} diff --git a/desktop/src-tauri/src/huddle/preprocessing.rs b/desktop/src-tauri/src/huddle/preprocessing.rs new file mode 100644 index 00000000000..2f69ac5dd6c --- /dev/null +++ b/desktop/src-tauri/src/huddle/preprocessing.rs @@ -0,0 +1,667 @@ +//! Text preprocessing for TTS output. +//! +//! Mental model: +//! +//! ```text +//! raw agent text +//! → strip fenced code blocks → "code block omitted" +//! → strip inline code → bare text +//! → strip URLs → "link omitted" +//! → strip markdown markers → plain text +//! → strip emoji → (removed) +//! → numbers → words → "forty two" +//! → collapse whitespace → clean string +//! ``` +//! +//! Also provides `split_sentences` — the single sentence-boundary splitter used +//! by both the TTS batching pipeline and the Supertonic text chunker. + +use regex::Regex; +use std::sync::LazyLock; + +// ── Sentence splitting ──────────────────────────────────────────────────────── + +/// Regex: a sentence-ending punctuation mark followed by whitespace. +static RE_SENTENCE_BOUNDARY: LazyLock = LazyLock::new(|| Regex::new(r"([.!?])\s+").unwrap()); + +/// Common abbreviations that end with a period but are NOT sentence boundaries. +const ABBREVIATIONS: &[&str] = &[ + "Dr.", "Mr.", "Mrs.", "Ms.", "Prof.", "Sr.", "Jr.", "St.", "Ave.", "Rd.", "Blvd.", "Dept.", + "Inc.", "Ltd.", "Co.", "Corp.", "etc.", "vs.", "i.e.", "e.g.", "Ph.D.", +]; + +/// Split text into sentence-sized chunks. +/// +/// Combines regex-based boundary detection with: +/// - Abbreviation awareness (`Dr.`, `Mr.`, etc. don't split) +/// - Digit-before-period check (avoids splitting `1.` `2.` numbered lists) +/// - `\n` and `—` treated as sentence breaks +/// +/// Returns non-empty, trimmed strings. +pub fn split_sentences(text: &str) -> Vec { + // First, split on newlines and em-dashes to get coarse segments. + let coarse: Vec<&str> = text.split(|c: char| c == '\n' || c == '—').collect(); + + let mut sentences = Vec::new(); + + for segment in coarse { + let segment = segment.trim(); + if segment.is_empty() { + continue; + } + // Within each segment, split on sentence-ending punctuation. + let matches: Vec<_> = RE_SENTENCE_BOUNDARY.find_iter(segment).collect(); + if matches.is_empty() { + sentences.push(segment.to_string()); + continue; + } + + let mut last_end = 0usize; + for m in &matches { + let before = &segment[last_end..m.start()]; + let punc_char = &segment[m.start()..m.start() + 1]; + + // Skip if this looks like an abbreviation. + let combined = format!("{}{}", before.trim(), punc_char); + let is_abbrev = ABBREVIATIONS.iter().any(|a| combined.ends_with(a)); + + // Skip if the character before the period is a digit (numbered list). + let is_digit_period = punc_char == "." + && !before.is_empty() + && before.ends_with(|c: char| c.is_ascii_digit()); + + if !is_abbrev && !is_digit_period { + let piece = segment[last_end..m.end()].trim(); + if !piece.is_empty() { + sentences.push(piece.to_string()); + } + last_end = m.end(); + } + } + + if last_end < segment.len() { + let tail = segment[last_end..].trim(); + if !tail.is_empty() { + sentences.push(tail.to_string()); + } + } + } + + if sentences.is_empty() { + vec![text.to_string()] + } else { + sentences + } +} + +// ── Public API ──────────────────────────────────────────────────────────────── + +/// Prepare `text` for TTS synthesis. +/// +/// Applies in order: +/// 1. Fenced code blocks → "code block omitted" +/// 2. Inline code → bare content (backticks stripped) +/// 3. URLs → "link omitted" +/// 4. Markdown bold/italic/underline markers stripped +/// 5. Emoji stripped +/// 6. Numbers → words (integers 0–999, times HH:MM) +/// 7. Excess whitespace collapsed +pub fn preprocess_for_tts(text: &str) -> String { + let s = strip_fenced_code_blocks(text); + let s = strip_inline_code(&s); + let s = strip_urls(&s); + let s = strip_markdown_markers(&s); + let s = strip_emoji(&s); + let s = expand_numbers(&s); + let s = collapse_whitespace(&s); + // Filter trivially short results — ".", ",", etc. would be spoken as + // "period", "comma" by TTS. Agents that have nothing relevant to say + // should not respond at all, but defense-in-depth catches edge cases. + if s.len() <= 1 { + return String::new(); + } + s +} + +// ── Step implementations ────────────────────────────────────────────────────── + +/// Replace fenced code blocks with "code block omitted". +/// +/// Handles both ` ``` ` and `~~~` fences. Multi-line aware. +fn strip_fenced_code_blocks(text: &str) -> String { + let s = replace_fenced(text, "```"); + replace_fenced(&s, "~~~") +} + +fn replace_fenced(text: &str, fence: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + loop { + match rest.find(fence) { + None => { + out.push_str(rest); + break; + } + Some(start) => { + // Everything before the opening fence. + out.push_str(&rest[..start]); + rest = &rest[start + fence.len()..]; + // Skip optional language tag on the same line. + if let Some(nl) = rest.find('\n') { + rest = &rest[nl + 1..]; + } + // Find the closing fence. + match rest.find(fence) { + None => { + // Unclosed fence — treat rest as omitted. + out.push_str(" code block omitted "); + break; + } + Some(end) => { + out.push_str(" code block omitted "); + rest = &rest[end + fence.len()..]; + // Skip trailing newline after closing fence. + if rest.starts_with('\n') { + rest = &rest[1..]; + } + } + } + } + } + } + out +} + +/// Strip backtick-delimited inline code, leaving the inner text. +/// +/// Single-backtick only — triple backtick already handled above. +fn strip_inline_code(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + loop { + match rest.find('`') { + None => { + out.push_str(rest); + break; + } + Some(start) => { + out.push_str(&rest[..start]); + rest = &rest[start + 1..]; + match rest.find('`') { + None => { + // Unclosed — emit as-is. + out.push_str(rest); + break; + } + Some(end) => { + out.push_str(&rest[..end]); + rest = &rest[end + 1..]; + } + } + } + } + } + out +} + +/// Replace http/https URLs with "link omitted". +/// +/// Trailing sentence-ending punctuation (`.`, `!`, `?`) that immediately follows +/// a URL and is at end-of-string or followed by whitespace is preserved so that +/// sentence splitting and TTS prosody are not degraded. +/// +/// Example: `"See https://x.y/z."` → `"See link omitted."` +fn strip_urls(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + loop { + // Find the earliest URL prefix. + let http = rest.find("http://"); + let https = rest.find("https://"); + let url_start = match (http, https) { + (None, None) => { + out.push_str(rest); + break; + } + (Some(a), None) => a, + (None, Some(b)) => b, + (Some(a), Some(b)) => a.min(b), + }; + out.push_str(&rest[..url_start]); + rest = &rest[url_start..]; + // Consume until whitespace or structural delimiter. + let url_end = rest + .find(|c: char| c.is_whitespace() || c == ')' || c == ']' || c == '"' || c == '\'') + .unwrap_or(rest.len()); + let url_token = &rest[..url_end]; + rest = &rest[url_end..]; + + // Check if the URL token ends with sentence-ending punctuation that + // belongs to the surrounding sentence rather than the URL itself. + // A trailing `.`, `!`, or `?` is preserved when it is at end-of-string + // or followed by whitespace (i.e. it is a sentence boundary). + let trailing_punct = if url_token.ends_with(|c: char| matches!(c, '.' | '!' | '?')) { + let after = rest; // rest is already past url_end + if after.is_empty() || after.starts_with(|c: char| c.is_whitespace()) { + // Preserve the trailing punctuation. + &url_token[url_token.len() - 1..] + } else { + "" + } + } else { + "" + }; + + out.push_str("link omitted"); + out.push_str(trailing_punct); + } + out +} + +/// Strip `**`, `*`, `__`, `_emphasis_`, `~~` markdown markers. +/// +/// Underscores are only stripped when they wrap a word (`_text_`). +/// Standalone underscores (e.g. `snake_case` identifiers) are preserved. +fn strip_markdown_markers(text: &str) -> String { + // Order matters: strip multi-char markers before single-char. + let s = text.replace("**", ""); + let s = s.replace("__", ""); + let s = s.replace("~~", ""); + let s = s.replace('*', ""); + strip_underscore_emphasis(&s) +} + +/// Strip `_text_` emphasis markers while preserving underscores in identifiers. +/// +/// A `_` is treated as an emphasis delimiter only when it is preceded by +/// whitespace or the start of the string AND followed by a non-whitespace char, +/// or vice-versa for the closing delimiter. +fn strip_underscore_emphasis(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let chars: Vec = text.chars().collect(); + let len = chars.len(); + let mut i = 0; + while i < len { + if chars[i] == '_' { + // Opening delimiter: preceded by whitespace/start, followed by non-whitespace. + let prev_is_boundary = i == 0 || chars[i - 1].is_whitespace(); + let next_is_nonspace = i + 1 < len && !chars[i + 1].is_whitespace(); + if prev_is_boundary && next_is_nonspace { + // Look for a matching closing `_`. + if let Some(close) = (i + 1..len).find(|&j| { + chars[j] == '_' + && !chars[j - 1].is_whitespace() + && (j + 1 >= len + || chars[j + 1].is_whitespace() + || chars[j + 1].is_ascii_punctuation()) + }) { + // Emit the inner text without the delimiters. + for &ch in &chars[i + 1..close] { + out.push(ch); + } + i = close + 1; + continue; + } + } + // Not an emphasis delimiter — emit as-is. + out.push('_'); + } else { + out.push(chars[i]); + } + i += 1; + } + out +} + +/// Strip Unicode emoji (characters in common emoji ranges). +/// +/// Covers the main Emoji block (U+1F300–U+1FAFF) and supplemental ranges. +/// ASCII emoticons like `:)` are left as-is. +fn strip_emoji(text: &str) -> String { + text.chars().filter(|&c| !is_emoji(c)).collect() +} + +#[inline] +fn is_emoji(c: char) -> bool { + matches!(c, + '\u{1F300}'..='\u{1FAFF}' // Misc symbols, emoticons, transport, etc. + | '\u{2600}'..='\u{27BF}' // Misc symbols, dingbats + | '\u{FE00}'..='\u{FE0F}' // Variation selectors + | '\u{1F000}'..='\u{1F02F}'// Mahjong/domino tiles + | '\u{1F0A0}'..='\u{1F0FF}'// Playing cards + | '\u{200D}' // Zero-width joiner (used in emoji sequences) + | '\u{20E3}' // Combining enclosing keycap + ) +} + +/// Expand numbers to spoken words. +/// +/// Handles: +/// - Times: `HH:MM` → "eleven thirty" +/// - Integers 0–999,999 +/// - Leaves other numeric strings (e.g. "3.14", "1000000+") as-is. +fn expand_numbers(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut chars = text.char_indices().peekable(); + + while let Some((i, c)) = chars.next() { + if c.is_ascii_digit() { + // Collect the full token (digits, colon, dots). + let start = i; + let mut end = i + c.len_utf8(); + while let Some(&(j, nc)) = chars.peek() { + if nc.is_ascii_digit() || nc == ':' || nc == '.' { + end = j + nc.len_utf8(); + chars.next(); + } else { + break; + } + } + let token = &text[start..end]; + out.push_str(&expand_numeric_token(token)); + } else { + out.push(c); + } + } + out +} + +fn expand_numeric_token(token: &str) -> String { + // Strip trailing punctuation that the token collector may have included + // (e.g. "11:30." from "at 11:30.") before attempting to parse. + let token = token.trim_end_matches(|c: char| !c.is_ascii_digit()); + + // Time: HH:MM + if let Some(colon) = token.find(':') { + let h = &token[..colon]; + let m = &token[colon + 1..]; + if let (Ok(hh), Ok(mm)) = (h.parse::(), m.parse::()) { + if hh < 24 && mm < 60 { + let hour_word = int_to_words(hh); + let min_word = if mm == 0 { + String::new() + } else if mm < 10 { + // "9:05" → "nine oh five" (not "nine five") + format!(" oh {}", int_to_words(mm)) + } else { + format!(" {}", int_to_words(mm)) + }; + return format!("{}{}", hour_word, min_word); + } + } + // Not a valid time — return as-is. + return token.to_string(); + } + + // Plain integer 0–999,999. + if token.chars().all(|c| c.is_ascii_digit()) { + if let Ok(n) = token.parse::() { + if n <= 999_999 { + return int_to_words(n); + } + } + } + + // Anything else (decimals, millions+) — leave as-is. + token.to_string() +} + +/// Convert an integer 0–999,999 to English words. +fn int_to_words(n: u32) -> String { + const ONES: &[&str] = &[ + "zero", + "one", + "two", + "three", + "four", + "five", + "six", + "seven", + "eight", + "nine", + "ten", + "eleven", + "twelve", + "thirteen", + "fourteen", + "fifteen", + "sixteen", + "seventeen", + "eighteen", + "nineteen", + ]; + const TENS: &[&str] = &[ + "", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety", + ]; + + if n < 20 { + return ONES[n as usize].to_string(); + } + if n < 100 { + let ten = TENS[(n / 10) as usize]; + let one = n % 10; + return if one == 0 { + ten.to_string() + } else { + format!("{} {}", ten, ONES[one as usize]) + }; + } + if n < 1000 { + let hundreds = n / 100; + let remainder = n % 100; + let hundred_word = format!("{} hundred", ONES[hundreds as usize]); + return if remainder == 0 { + hundred_word + } else { + format!("{} {}", hundred_word, int_to_words(remainder)) + }; + } + // 1,000–999,999 + let thousands = n / 1000; + let remainder = n % 1000; + let thousand_word = format!("{} thousand", int_to_words(thousands)); + if remainder == 0 { + thousand_word + } else { + format!("{} {}", thousand_word, int_to_words(remainder)) + } +} + +/// Collapse runs of whitespace (spaces, tabs, newlines) to a single space. +/// Trims leading/trailing whitespace. +fn collapse_whitespace(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut prev_space = true; // Start true to trim leading whitespace. + for c in text.chars() { + if c.is_whitespace() { + if !prev_space { + out.push(' '); + prev_space = true; + } + } else { + out.push(c); + prev_space = false; + } + } + // Trim trailing space. + if out.ends_with(' ') { + out.pop(); + } + out +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strips_fenced_code_block() { + let input = "Here is some code:\n```rust\nfn main() {}\n```\nDone."; + let out = preprocess_for_tts(input); + assert!(out.contains("code block omitted"), "got: {out}"); + assert!(!out.contains("fn main"), "got: {out}"); + } + + #[test] + fn strips_inline_code() { + let out = preprocess_for_tts("Call `foo()` now."); + assert_eq!(out, "Call foo() now."); + } + + #[test] + fn strips_urls() { + let out = preprocess_for_tts("See https://example.com for details."); + assert!(out.contains("link omitted"), "got: {out}"); + assert!(!out.contains("example.com"), "got: {out}"); + } + + #[test] + fn strips_url_preserves_trailing_period() { + // Trailing `.` at end of sentence must be preserved for sentence splitting. + let out = strip_urls("See https://x.y/z."); + assert_eq!(out, "See link omitted.", "got: {out}"); + } + + #[test] + fn strips_url_preserves_trailing_exclamation() { + let out = strip_urls("Visit https://example.com!"); + assert_eq!(out, "Visit link omitted!", "got: {out}"); + } + + #[test] + fn strips_url_preserves_trailing_question() { + let out = strip_urls("Did you see https://example.com?"); + assert_eq!(out, "Did you see link omitted?", "got: {out}"); + } + + #[test] + fn strips_url_mid_sentence_no_punct_preserved() { + // URL in the middle of a sentence — no trailing punct to preserve. + let out = strip_urls("Check https://example.com for more info."); + assert_eq!(out, "Check link omitted for more info.", "got: {out}"); + } + + #[test] + fn strips_bold_italic() { + let out = preprocess_for_tts("**bold** and *italic* and _under_"); + assert_eq!(out, "bold and italic and under"); + } + + #[test] + fn preserves_standalone_underscores() { + // snake_case identifiers should not be mangled. + let out = preprocess_for_tts("call foo_bar() or baz_qux"); + assert!(out.contains("foo_bar"), "got: {out}"); + assert!(out.contains("baz_qux"), "got: {out}"); + } + + #[test] + fn strips_tilde_fenced_block() { + let input = "Here:\n~~~python\nprint('hi')\n~~~\nDone."; + let out = preprocess_for_tts(input); + assert!(out.contains("code block omitted"), "got: {out}"); + assert!(!out.contains("print"), "got: {out}"); + } + + #[test] + fn expands_integers() { + assert_eq!(preprocess_for_tts("42"), "forty two"); + assert_eq!(preprocess_for_tts("0"), "zero"); + assert_eq!(preprocess_for_tts("11"), "eleven"); + assert_eq!(preprocess_for_tts("100"), "one hundred"); + } + + #[test] + fn expands_thousands() { + assert_eq!(preprocess_for_tts("1000"), "one thousand"); + assert_eq!( + preprocess_for_tts("1234"), + "one thousand two hundred thirty four" + ); + assert_eq!(preprocess_for_tts("10000"), "ten thousand"); + assert_eq!(preprocess_for_tts("100000"), "one hundred thousand"); + assert_eq!( + preprocess_for_tts("999999"), + "nine hundred ninety nine thousand nine hundred ninety nine" + ); + } + + #[test] + fn expands_times() { + assert_eq!(preprocess_for_tts("11:30"), "eleven thirty"); + assert_eq!(preprocess_for_tts("9:00"), "nine"); + assert_eq!(preprocess_for_tts("9:05"), "nine oh five"); + assert_eq!(preprocess_for_tts("10:09"), "ten oh nine"); + } + + #[test] + fn collapses_whitespace() { + let out = preprocess_for_tts(" hello world "); + assert_eq!(out, "hello world"); + } + + #[test] + fn split_sentences_basic() { + let result = split_sentences("Hello world. How are you? I'm fine!"); + assert_eq!(result, vec!["Hello world.", "How are you?", "I'm fine!"]); + } + + #[test] + fn split_sentences_newline_break() { + let result = split_sentences("First line.\nSecond line."); + assert_eq!(result, vec!["First line.", "Second line."]); + } + + #[test] + fn split_sentences_em_dash_break() { + let result = split_sentences("Start here—then continue."); + assert_eq!(result, vec!["Start here", "then continue."]); + } + + #[test] + fn split_sentences_abbreviations() { + let result = split_sentences("Dr. Smith went home. He was tired."); + assert_eq!(result, vec!["Dr. Smith went home.", "He was tired."]); + } + + #[test] + fn split_sentences_numbered_list() { + let result = split_sentences("1. First item. 2. Second item."); + // "1." and "2." should NOT cause a split (digit before period). + assert_eq!(result, vec!["1. First item.", "2. Second item."]); + } + + #[test] + fn split_sentences_single() { + let result = split_sentences("Just one sentence"); + assert_eq!(result, vec!["Just one sentence"]); + } + + #[test] + fn split_sentences_empty() { + let result = split_sentences(""); + assert_eq!(result, vec![""]); + } + + #[test] + fn filters_trivial_responses() { + assert_eq!(preprocess_for_tts("."), ""); + assert_eq!(preprocess_for_tts(","), ""); + assert_eq!(preprocess_for_tts("!"), ""); + assert_eq!(preprocess_for_tts(" "), ""); + assert_eq!(preprocess_for_tts("ok"), "ok"); + } + + #[test] + fn full_pipeline() { + let input = + "**Agent says:** check https://relay.example.com at 11:30.\n```\nsome code\n```"; + let out = preprocess_for_tts(input); + assert!(!out.contains("**"), "got: {out}"); + assert!(!out.contains("https://"), "got: {out}"); + assert!(out.contains("eleven thirty"), "got: {out}"); + assert!(out.contains("code block omitted"), "got: {out}"); + } +} diff --git a/desktop/src-tauri/src/huddle/relay_api.rs b/desktop/src-tauri/src/huddle/relay_api.rs new file mode 100644 index 00000000000..c3ccc54c34f --- /dev/null +++ b/desktop/src-tauri/src/huddle/relay_api.rs @@ -0,0 +1,107 @@ +//! Relay HTTP helpers for huddle operations. +//! +//! Thin wrappers around the relay REST API for LiveKit token requests, +//! channel membership queries, and human participant counting. + +use reqwest::Method; +use serde::Deserialize; +use uuid::Uuid; + +use crate::app_state::AppState; +use crate::relay::{api_path, build_authed_request, send_json_request}; + +use super::state::LiveKitTokenResponse; + +/// Maximum number of agents that can be invited to a single huddle. +pub(crate) const MAX_HUDDLE_AGENTS: usize = 20; + +/// Validate that a string looks like a Nostr pubkey hex (64 hex chars). +pub(crate) fn validate_pubkey_hex(pubkey: &str) -> Result<(), String> { + if pubkey.len() != 64 || !pubkey.chars().all(|c| c.is_ascii_hexdigit()) { + let preview: String = pubkey.chars().take(16).collect(); + return Err(format!("invalid pubkey hex: {preview}")); + } + Ok(()) +} + +pub(crate) fn parse_channel_uuid(channel_id: &str) -> Result { + Uuid::parse_str(channel_id).map_err(|_| format!("invalid channel UUID: {channel_id}")) +} + +/// Fetch a LiveKit token from the relay for the given channel. +/// +/// When `parent_channel_id` is `Some`, appends `?parent_channel_id={id}` to +/// the URL so the relay can auto-add the caller as a member of the ephemeral +/// channel (used by joiners — creators are already owners and pass `None`). +pub(crate) async fn fetch_livekit_token( + channel_id: &str, + parent_channel_id: Option<&str>, + state: &AppState, +) -> Result { + let base = api_path(&["huddles", channel_id, "token"]); + let path = match parent_channel_id { + Some(pid) => format!("{base}?parent_channel_id={pid}"), + None => base, + }; + let request = build_authed_request(&state.http_client, Method::POST, &path, state)?; + send_json_request(request).await +} + +/// Fetch channel members with their roles from the relay. +/// Returns (pubkey, role) tuples — the authoritative source for both +/// `fetch_channel_members` (filtered by role) and `count_human_members`. +pub(crate) async fn fetch_channel_members_with_roles( + channel_id: &str, + state: &AppState, +) -> Result)>, String> { + #[derive(Deserialize)] + struct Member { + pubkey: String, + role: Option, + } + #[derive(Deserialize)] + struct MembersResponse { + members: Vec, + } + + let path = api_path(&["channels", channel_id, "members"]); + let request = build_authed_request(&state.http_client, Method::GET, &path, state)?; + let resp: MembersResponse = send_json_request(request).await.map_err(|e| { + eprintln!("sprout-desktop: fetch channel members failed: {e}"); + e + })?; + + Ok(resp + .members + .into_iter() + .map(|m| (m.pubkey, m.role)) + .collect()) +} + +/// Fetch channel members from the relay. If `role_filter` is Some, only return +/// members with that role (e.g., "bot" for agents). Returns all members if None. +pub(crate) async fn fetch_channel_members( + channel_id: &str, + role_filter: Option<&str>, + state: &AppState, +) -> Result, String> { + let all = fetch_channel_members_with_roles(channel_id, state).await?; + Ok(all + .into_iter() + .filter(|(_, role)| role_filter.map_or(true, |r| role.as_deref() == Some(r))) + .map(|(pubkey, _)| pubkey) + .collect()) +} + +/// Count human (non-bot) members remaining in a channel. +/// Built on `fetch_channel_members_with_roles` — fetches all members then counts non-bots. +pub(crate) async fn count_human_members( + channel_id: &str, + state: &AppState, +) -> Result { + let all = fetch_channel_members_with_roles(channel_id, state).await?; + Ok(all + .iter() + .filter(|(_, role)| role.as_deref() != Some("bot")) + .count()) +} diff --git a/desktop/src-tauri/src/huddle/state.rs b/desktop/src-tauri/src/huddle/state.rs new file mode 100644 index 00000000000..b73e41a2d6a --- /dev/null +++ b/desktop/src-tauri/src/huddle/state.rs @@ -0,0 +1,244 @@ +//! Huddle state types and serialization. +//! +//! Contains `HuddleState` (the god-object behind `AppState.huddle_state`), +//! phase enum, voice input mode, and response types. + +use serde::{Deserialize, Serialize}; +use std::sync::{ + atomic::{AtomicBool, AtomicU64}, + Arc, Mutex, +}; + +use super::{stt, tts}; + +/// Voice input mode: push-to-talk (PTT) or voice-activity detection (VAD). +/// +/// PTT (default): mic is gated by a global shortcut (Ctrl+Space). Pressing the +/// key sets `ptt_active` and immediately cancels any playing TTS. Releasing +/// the key (after a 200 ms delay) stops mic capture and flushes the utterance. +/// +/// VAD: the earshot VAD runs continuously and speech is accumulated whenever +/// the probability exceeds the threshold. Barge-in is enabled in this mode. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum VoiceInputMode { + #[default] + PushToTalk, + VoiceActivity, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum HuddlePhase { + Idle, + Creating, + Connecting, + Connected, // Backend ready, waiting for frontend media confirmation. + Active, + Leaving, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct HuddleState { + pub phase: HuddlePhase, + pub parent_channel_id: Option, + pub ephemeral_channel_id: Option, + /// Skipped from serialization — the frontend gets the token from + /// start_huddle/join_huddle return values. Exposing the LiveKit JWT + /// on every 2-second get_huddle_state poll is unnecessary attack surface. + #[serde(skip)] + pub livekit_token: Option, + /// Skipped from serialization — same rationale as livekit_token. + #[serde(skip)] + pub livekit_url: Option, + pub livekit_room: Option, + /// Participant pubkey hex strings (all members, including humans). + pub participants: Vec, + /// Agent pubkeys only — used as p-tags on transcribed messages. + /// + /// Stored as `Arc>>` so the transcription task can clone + /// the `Arc` and read the current list at post time without holding the + /// outer `HuddleState` lock across an await point. + /// + /// Populated from `member_pubkeys` in `start_huddle` (the UI sends agent + /// pubkeys specifically). Joiners don't add agents — they were already + /// added by the creator. Serialized as a plain `Vec` for the + /// frontend via the custom `Serialize`/`Deserialize` impls below. + #[serde( + serialize_with = "serialize_agent_pubkeys", + deserialize_with = "deserialize_agent_pubkeys" + )] + pub agent_pubkeys: Arc>>, + /// Active STT pipeline — not serialized, not cloned. + #[serde(skip)] + pub stt_pipeline: Option>, + /// Active TTS pipeline — not serialized, not cloned. + #[serde(skip)] + pub tts_pipeline: Option>, + /// Whether this client created the huddle (vs. joined it). + /// Used to enforce that only the creator can end/archive the huddle. + pub is_creator: bool, + /// Whether TTS output is enabled (user-toggled). + pub tts_enabled: bool, + /// Shared flag: true while TTS is playing audio. + /// Shared with the STT pipeline for barge-in / echo gating. + #[serde(skip)] + pub tts_active: Arc, + /// Shared barge-in cancel flag. Set by STT when it detects speech during TTS. + /// Read by TTS to stop playback. Lives in HuddleState so it survives pipeline + /// restarts — both STT and TTS reference the same flag for the entire huddle. + #[serde(skip)] + pub tts_cancel: Arc, + /// Sentinel: true while a TTS pipeline is being constructed (outside the lock). + /// Prevents TOCTOU races where two concurrent callers both pass the `is_some()` + /// check and both spawn TTS worker threads — the loser's thread would leak. + #[serde(skip)] + pub tts_starting: Arc, + /// Sentinel: true while an STT pipeline is being constructed. + /// Mirrors `tts_starting` — prevents TOCTOU races where two concurrent + /// callers both pass the `is_some()` check and both spawn STT workers. + #[serde(skip)] + pub stt_starting: Arc, + /// Timestamp of the last agent pubkey refresh from the relay. + /// Used to throttle the refresh in check_pipeline_hotstart to every 15 s. + #[serde(skip)] + pub last_agent_refresh: Option, + /// Session generation — incremented on every teardown. The transcription + /// task captures this at spawn time and checks before each POST. If the + /// generation has changed, the task silently drops the transcript. + #[serde(skip)] + pub session_generation: Arc, + /// Voice input mode: push-to-talk or voice-activity detection. + pub voice_input_mode: VoiceInputMode, + /// True while the PTT key is held (+ 200 ms release delay). + /// Shared with the STT pipeline for mic gating. + #[serde(skip)] + pub ptt_active: Arc, +} + +fn serialize_agent_pubkeys(v: &Arc>>, s: S) -> Result +where + S: serde::Serializer, +{ + use serde::ser::SerializeSeq; + let guard = v.lock().unwrap_or_else(|e| e.into_inner()); + let mut seq = s.serialize_seq(Some(guard.len()))?; + for item in guard.iter() { + seq.serialize_element(item)?; + } + seq.end() +} + +fn deserialize_agent_pubkeys<'de, D>(d: D) -> Result>>, D::Error> +where + D: serde::Deserializer<'de>, +{ + let v: Vec = serde::Deserialize::deserialize(d)?; + Ok(Arc::new(Mutex::new(v))) +} + +impl Clone for HuddleState { + fn clone(&self) -> Self { + let agent_pubkeys_snapshot = self + .agent_pubkeys + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(); + Self { + phase: self.phase.clone(), + parent_channel_id: self.parent_channel_id.clone(), + ephemeral_channel_id: self.ephemeral_channel_id.clone(), + livekit_token: self.livekit_token.clone(), + livekit_url: self.livekit_url.clone(), + livekit_room: self.livekit_room.clone(), + participants: self.participants.clone(), + agent_pubkeys: Arc::new(Mutex::new(agent_pubkeys_snapshot)), + stt_pipeline: None, // Never clone the pipeline handle. + tts_pipeline: None, // Never clone the pipeline handle. + is_creator: self.is_creator, + tts_enabled: self.tts_enabled, + tts_active: Arc::clone(&self.tts_active), + tts_cancel: Arc::clone(&self.tts_cancel), + tts_starting: Arc::clone(&self.tts_starting), + stt_starting: Arc::clone(&self.stt_starting), + last_agent_refresh: self.last_agent_refresh, + session_generation: Arc::clone(&self.session_generation), + voice_input_mode: self.voice_input_mode.clone(), + ptt_active: Arc::clone(&self.ptt_active), + } + } +} + +impl Default for HuddleState { + fn default() -> Self { + Self { + phase: HuddlePhase::Idle, + parent_channel_id: None, + ephemeral_channel_id: None, + livekit_token: None, + livekit_url: None, + livekit_room: None, + participants: Vec::new(), + agent_pubkeys: Arc::new(Mutex::new(Vec::new())), + stt_pipeline: None, + tts_pipeline: None, + is_creator: false, + tts_enabled: true, + tts_active: Arc::new(AtomicBool::new(false)), + tts_cancel: Arc::new(AtomicBool::new(false)), + tts_starting: Arc::new(AtomicBool::new(false)), + stt_starting: Arc::new(AtomicBool::new(false)), + last_agent_refresh: None, + session_generation: Arc::new(AtomicU64::new(0)), + voice_input_mode: VoiceInputMode::default(), + ptt_active: Arc::new(AtomicBool::new(false)), + } + } +} + +impl HuddleState { + /// Reset to default state while preserving the session generation counter. + /// Used by start_huddle rollback, join_huddle rollback, and teardown_huddle + /// to invalidate in-flight transcription tasks without losing the generation. + pub(crate) fn reset_preserving_generation(&mut self) { + let gen = Arc::clone(&self.session_generation); + *self = Self::default(); + self.session_generation = gen; + } +} + +// ── Event emission ──────────────────────────────────────────────────────────── + +/// Emit the current huddle state to the frontend via a Tauri event. +/// +/// The frontend listens for `"huddle-state-changed"` and updates its UI +/// immediately, replacing the previous 2-second polling loop. +/// +/// **Call sites** — called from `AppState::emit_huddle_state_changed()` in +/// `app_state.rs` after every state transition the frontend needs to observe +/// (phase changes, participant updates, tts_enabled toggle). +/// +/// Best-effort — silently ignores errors (e.g., no listeners attached yet). +pub fn emit_huddle_state(app: &tauri::AppHandle, state: &HuddleState) { + use tauri::Emitter; + let _ = app.emit("huddle-state-changed", state); +} + +// ── Response types ──────────────────────────────────────────────────────────── + +/// Returned by start_huddle and join_huddle. +#[derive(Debug, Serialize, Deserialize)] +pub struct HuddleJoinInfo { + pub ephemeral_channel_id: String, + pub livekit_token: String, + pub livekit_url: String, + pub livekit_room: String, +} + +/// Raw response from `POST /api/huddles/{channel_id}/token`. +#[derive(Debug, Deserialize)] +pub(crate) struct LiveKitTokenResponse { + pub token: String, + pub url: String, + pub room: String, +} diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs new file mode 100644 index 00000000000..6566ab553df --- /dev/null +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -0,0 +1,562 @@ +//! Speech-to-Text pipeline for huddle voice transcription. +//! +//! Mental model: +//! +//! ```text +//! AudioWorklet (48 kHz f32 PCM) +//! → push_audio_pcm (Tauri cmd) +//! → SttPipeline::push_audio [bounded sync_channel] +//! → stt_worker thread +//! rubato: 48 kHz → 16 kHz mono +//! earshot VAD: accumulate speech frames +//! sherpa-onnx Moonshine: transcribe on silence +//! → text_rx [mpsc channel] +//! → tokio task (start_stt_pipeline) +//! builds kind:9 event → relay +//! ``` +//! +//! The worker runs on a dedicated `std::thread` (not async) because +//! sherpa-onnx is CPU-bound and not Send-safe across await points. + +use std::{ + path::PathBuf, + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc::{self, Receiver, SyncSender}, + Arc, + }, + thread, + time::Duration, +}; + +use tokio::sync::mpsc as tokio_mpsc; + +// ── Public pipeline handle ──────────────────────────────────────────────────── + +/// Bounded audio queue capacity. +/// 100 ms batches at 48 kHz ≈ 19 KB each → 50 slots ≈ 5 s / ~1 MB max backlog. +const AUDIO_QUEUE_DEPTH: usize = 50; + +/// Maximum speech buffer size: 30 seconds at 16 kHz. +/// Prevents OOM if VAD stays in speech mode (noisy environment). +const MAX_SPEECH_SAMPLES: usize = 16_000 * 30; + +/// Handle to the running STT pipeline. +/// +/// Not Clone — wrap in `Arc` to share across threads. +/// +/// The text receiver (`tokio::sync::mpsc::Receiver`) is returned +/// separately from `new()` so the caller can move it directly into an async +/// task without holding a Mutex across await points. +#[derive(Debug)] +pub struct SttPipeline { + /// Send raw PCM bytes (f32 LE, 48 kHz mono) into the pipeline. + audio_tx: SyncSender>, + /// Signals the worker thread to stop. + shutdown: Arc, + /// Worker thread handle — taken on drop to join cleanly. + thread: Option>, + /// TTS cancel flag shared with the TTS pipeline. + /// When VAD detects speech onset during TTS playback, the STT worker sets + /// this flag to trigger barge-in (stops TTS immediately). + /// Stored here so it lives as long as the pipeline; the worker holds a clone. + #[allow(dead_code)] + pub tts_cancel: Option>, +} + +impl SttPipeline { + /// Spawn the pipeline thread. + /// + /// `tts_active` is a shared flag set by the TTS pipeline while audio is + /// playing. The STT worker uses it to: + /// - discard accumulated speech (echo prevention / barge-in gating) + /// - apply a 200 ms cooldown after TTS stops before re-enabling STT + /// - detect barge-in: speech onset during TTS → set `tts_cancel` + /// + /// `tts_cancel` (optional) is the TTS pipeline's cancel flag. When the STT + /// worker detects speech onset while TTS is active, it sets this flag to + /// stop playback immediately (barge-in). Pass `None` if TTS is unavailable. + /// + /// `ptt_active` (optional) is the push-to-talk flag. When `Some`, the STT + /// pipeline only accumulates speech while the flag is true (key held). + /// When `None`, the pipeline runs in continuous VAD mode. + /// + /// Returns `Err` only if the thread cannot be spawned (OS error). + /// If model files are missing, the worker logs and exits cleanly — + /// the pipeline handle is still returned but will never produce text. + /// + /// The `tokio::sync::mpsc::Receiver` is returned separately so the + /// caller can move it directly into an async task. This avoids holding a + /// `Mutex` across await points (which would block a Tokio worker + /// thread on every `recv_timeout` call). + pub fn new( + model_dir: PathBuf, + tts_active: Arc, + tts_cancel: Option>, + ptt_active: Option>, + ) -> Result<(Self, tokio_mpsc::Receiver), String> { + let (audio_tx, audio_rx) = mpsc::sync_channel::>(AUDIO_QUEUE_DEPTH); + let (text_tx, text_rx) = tokio_mpsc::channel::(64); + let shutdown = Arc::new(AtomicBool::new(false)); + + let shutdown_worker = Arc::clone(&shutdown); + let tts_cancel_worker = tts_cancel.as_ref().map(Arc::clone); + let ptt_active_worker = ptt_active.as_ref().map(Arc::clone); + let handle = thread::Builder::new() + .name("stt-worker".into()) + .spawn(move || { + stt_worker( + model_dir, + audio_rx, + text_tx, + shutdown_worker, + tts_active, + tts_cancel_worker, + ptt_active_worker, + ) + }) + .map_err(|e| format!("failed to spawn stt-worker thread: {e}"))?; + + let pipeline = Self { + audio_tx, + shutdown, + thread: Some(handle), + tts_cancel, + }; + Ok((pipeline, text_rx)) + } + + /// Signal the worker thread to stop. + pub fn shutdown(&self) { + self.shutdown.store(true, Ordering::Release); + } + + /// Returns `true` if the worker thread has exited (init failure, crash, or normal exit). + /// Used by hot-start to detect dead pipelines and clear them for retry. + pub fn is_finished(&self) -> bool { + self.thread.as_ref().map_or(true, |h| h.is_finished()) + } + + /// Feed raw PCM bytes into the pipeline. + /// + /// Non-blocking. Drops audio silently if the pipeline can't keep up — + /// better to lose frames than to stall the UI thread. + pub fn push_audio(&self, pcm_bytes: Vec) -> Result<(), String> { + // Reject non-4-byte-aligned input — would silently truncate in bytes_to_f32. + if pcm_bytes.len() % 4 != 0 { + return Err(format!( + "audio input not 4-byte aligned ({} bytes) — expected f32 LE samples", + pcm_bytes.len() + )); + } + // Drop audio if the pipeline can't keep up — better than blocking the UI. + let _ = self.audio_tx.try_send(pcm_bytes); + Ok(()) + } +} + +impl Drop for SttPipeline { + fn drop(&mut self) { + // Signal the worker to stop. + self.shutdown.store(true, Ordering::Release); + // Dropping `audio_tx` (implicitly when self is dropped after this fn) + // unblocks the worker's recv_timeout loop. Join to ensure clean exit. + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +// ── Worker thread ───────────────────────────────────────────────────────────── + +/// How many 16 kHz samples of silence before we flush to STT. +/// 300 ms × 16 000 Hz / 256 samples-per-frame ≈ 19 frames. +/// Previous value (28 frames / 450 ms) felt sluggish in conversation. +const SILENCE_FLUSH_FRAMES: usize = 19; + +/// Consecutive VAD speech frames required before triggering barge-in during TTS. +/// 20 frames × 256 samples / 16 kHz ≈ 320 ms — must be long enough to filter +/// speaker-to-mic feedback (TTS audio bleeding through the mic) while still +/// catching real human interruptions. 80 ms (previous: 5 frames) was too +/// aggressive — laptop speakers without headphones triggered false barge-in +/// within the first word of TTS playback. +const BARGE_IN_DEBOUNCE_FRAMES: usize = 20; + +/// earshot requires exactly 256 samples per frame at 16 kHz. +const VAD_FRAME_SAMPLES: usize = 256; + +/// VAD probability threshold — above this is considered speech. +const VAD_THRESHOLD: f32 = 0.5; + +/// How long the worker waits on the audio channel before checking the shutdown flag. +const RECV_TIMEOUT: Duration = Duration::from_millis(50); + +/// 50 ms cooldown after TTS stops before STT re-enables. +/// Prevents the tail of TTS audio from being transcribed as speech. +/// Previous value (200 ms) was eating the first word when the user spoke +/// immediately after the agent finished. +const TTS_COOLDOWN: Duration = Duration::from_millis(50); + +fn stt_worker( + model_dir: PathBuf, + audio_rx: Receiver>, + text_tx: tokio_mpsc::Sender, + shutdown: Arc, + tts_active: Arc, + tts_cancel: Option>, + ptt_active: Option>, +) { + // ── 1. Initialise rubato resampler (48 kHz → 16 kHz, mono) ─────────────── + use rubato::{Fft, FixedSync, Resampler}; + + let mut resampler = match Fft::::new(48_000, 16_000, 1024, 2, 1, FixedSync::Input) { + Ok(r) => r, + Err(e) => { + eprintln!("sprout-desktop: STT resampler init failed: {e}"); + return; + } + }; + let chunk_in = resampler.input_frames_next(); + + // ── 2. Initialise earshot VAD ───────────────────────────────────────────── + use earshot::{DefaultPredictor, Detector}; + let mut vad = Detector::new(DefaultPredictor::new()); + + // ── 3. Initialise sherpa-onnx recognizer ───────────────────────────────── + use sherpa_onnx::{OfflineMoonshineModelConfig, OfflineRecognizer, OfflineRecognizerConfig}; + + let tokens_path = model_dir.join("tokens.txt"); + if !tokens_path.exists() { + eprintln!( + "sprout-desktop: STT models not found at {} — STT disabled", + model_dir.display() + ); + drain_until_shutdown(audio_rx, &shutdown); + return; + } + + let model_dir_str = model_dir.to_string_lossy().into_owned(); + + let mut cfg = OfflineRecognizerConfig::default(); + cfg.model_config.moonshine = OfflineMoonshineModelConfig { + preprocessor: Some(format!("{model_dir_str}/preprocess.onnx")), + encoder: Some(format!("{model_dir_str}/encode.int8.onnx")), + uncached_decoder: Some(format!("{model_dir_str}/uncached_decode.int8.onnx")), + cached_decoder: Some(format!("{model_dir_str}/cached_decode.int8.onnx")), + merged_decoder: None, + }; + cfg.model_config.tokens = Some(tokens_path.to_string_lossy().into_owned()); + cfg.model_config.num_threads = 1; + cfg.model_config.model_type = Some("moonshine".into()); + + let recognizer = match OfflineRecognizer::create(&cfg) { + Some(r) => r, + None => { + eprintln!("sprout-desktop: OfflineRecognizer::create returned None — STT disabled"); + drain_until_shutdown(audio_rx, &shutdown); + return; + } + }; + + // ── 4. Processing state ─────────────────────────────────────────────────── + // Leftover 48 kHz samples that didn't fill a full resampler chunk. + let mut input_buf_48k: Vec = Vec::with_capacity(chunk_in * 2); + // Leftover 16 kHz samples that didn't fill a full VAD frame. + let mut leftover_16k: Vec = Vec::new(); + // Accumulated speech frames (16 kHz). + let mut speech_buf: Vec = Vec::new(); + // Consecutive silence frame count. + let mut silence_frames: usize = 0; + // Whether we're currently in a speech segment. + let mut in_speech = false; + // Consecutive speech frames seen during TTS — used for barge-in debounce. + let mut barge_in_frames: usize = 0; + // Timestamp when TTS last stopped — used for the 200 ms cooldown. + let mut tts_stopped_at: Option = None; + + // ── 5. Main loop ────────────────────────────────────────────────────────── + let mut tts_was_active = false; + let mut ptt_was_active = ptt_active + .as_ref() + .map_or(false, |p| p.load(Ordering::Acquire)); + loop { + // Check shutdown flag before blocking. + if shutdown.load(Ordering::Acquire) { + break; + } + + // Track TTS transitions to set the cooldown timer. + let tts_now = tts_active.load(Ordering::Acquire); + if tts_was_active && !tts_now { + // TTS just stopped — record the timestamp for the cooldown window. + tts_stopped_at = Some(std::time::Instant::now()); + } + tts_was_active = tts_now; + + // Track PTT transitions — flush accumulated speech when key is released. + // The worklet stops sending frames when PTT is inactive, so the normal + // silence-accumulation flush path never runs. We must flush here on the + // active→inactive edge to avoid buffering speech across PTT presses. + if let Some(ref ptt) = ptt_active { + let ptt_now = ptt.load(Ordering::Acquire); + if ptt_was_active && !ptt_now && in_speech && !speech_buf.is_empty() { + flush_to_stt(&speech_buf, &recognizer, &text_tx); + speech_buf.clear(); + silence_frames = 0; + in_speech = false; + } + ptt_was_active = ptt_now; + } + + // Use recv_timeout so we can periodically check the shutdown flag. + let bytes = match audio_rx.recv_timeout(RECV_TIMEOUT) { + Ok(b) => b, + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(mpsc::RecvTimeoutError::Disconnected) => break, // Sender dropped. + }; + + // Drain any additional pending messages to batch-process. + let mut batch = vec![bytes]; + while let Ok(b) = audio_rx.try_recv() { + batch.push(b); + } + + for bytes in batch { + // Convert raw bytes to f32 samples (little-endian). + let samples_48k = bytes_to_f32(&bytes); + input_buf_48k.extend_from_slice(&samples_48k); + + // Resample in chunk_in-sized blocks. + while input_buf_48k.len() >= chunk_in { + let chunk: Vec = input_buf_48k.drain(..chunk_in).collect(); + let resampled = resample_chunk(&mut resampler, &chunk); + process_16k_samples( + &resampled, + &mut leftover_16k, + &mut vad, + &mut speech_buf, + &mut silence_frames, + &mut in_speech, + &mut barge_in_frames, + &recognizer, + &text_tx, + &tts_active, + tts_cancel.as_deref(), + &mut tts_stopped_at, + ptt_active.as_ref(), + ); + } + } + } + + // No final flush — leave_huddle/end_huddle emit lifecycle events before + // the STT worker exits, so a final flush would post a kind:9 message AFTER + // the user has "left." Losing the last partial utterance is acceptable. +} + +/// Resample a mono 48 kHz chunk to 16 kHz using rubato. +/// Returns the resampled samples (may be empty on error). +fn resample_chunk(resampler: &mut rubato::Fft, chunk_48k: &[f32]) -> Vec { + use audioadapter_buffers::direct::InterleavedSlice; + use rubato::Resampler; + + // rubato expects interleaved layout even for mono. + let input = match InterleavedSlice::new(chunk_48k, 1, chunk_48k.len()) { + Ok(a) => a, + Err(e) => { + eprintln!("sprout-desktop: STT resample input error: {e}"); + return Vec::new(); + } + }; + + match resampler.process(&input, 0, None) { + Ok(out) => out.take_data(), + Err(e) => { + eprintln!("sprout-desktop: STT resample error: {e}"); + Vec::new() + } + } +} + +/// Feed 16 kHz samples through the VAD and accumulate speech. +/// Flushes to STT when silence exceeds threshold. +/// +/// When `tts_active` is set: +/// - In PTT mode: skip accumulation (PTT press handles TTS cancellation). +/// - In VAD mode: speech onset triggers barge-in via `tts_cancel`. +/// - After TTS stops, a cooldown prevents tail audio from being transcribed. +/// +/// When `ptt_active` is `Some`: +/// - VAD `is_speech` is ANDed with the PTT flag — when the key is released, +/// `is_speech` becomes false, silence_frames accumulates, and the existing +/// flush logic kicks in naturally. The 200 ms release delay + ~300 ms +/// silence flush gives a natural utterance tail. +#[allow(clippy::too_many_arguments)] +fn process_16k_samples( + samples: &[f32], + leftover: &mut Vec, + vad: &mut earshot::Detector, + speech_buf: &mut Vec, + silence_frames: &mut usize, + in_speech: &mut bool, + barge_in_frames: &mut usize, + recognizer: &sherpa_onnx::OfflineRecognizer, + text_tx: &tokio_mpsc::Sender, + tts_active: &Arc, + tts_cancel: Option<&AtomicBool>, + tts_stopped_at: &mut Option, + ptt_active: Option<&Arc>, +) { + leftover.extend_from_slice(samples); + + while leftover.len() >= VAD_FRAME_SAMPLES { + let frame: Vec = leftover.drain(..VAD_FRAME_SAMPLES).collect(); + let clamped: Vec = frame.iter().map(|&s| s.clamp(-1.0, 1.0)).collect(); + let prob = vad.predict_f32(&clamped); + let is_speech = prob > VAD_THRESHOLD; + + // PTT gating: when PTT key is not held, treat as silence. + // This causes natural flush when the key is released — silence_frames + // accumulates and the existing flush logic kicks in after + // SILENCE_FLUSH_FRAMES. The 200 ms release delay + ~300 ms silence + // flush gives a natural utterance tail. + let is_speech = if let Some(ptt) = ptt_active { + is_speech && ptt.load(Ordering::Acquire) + } else { + is_speech + }; + + let tts_playing = tts_active.load(Ordering::Acquire); + + // While TTS is playing: skip accumulation (echo prevention). + if tts_playing { + if ptt_active.is_some() { + // PTT mode — PTT press handles TTS cancellation directly + // (via the global shortcut handler). Just skip accumulation. + *in_speech = false; + *barge_in_frames = 0; + speech_buf.clear(); + *silence_frames = 0; + continue; + } + + // VAD mode — barge-in detection. + // Without acoustic echo cancellation, this requires a longer + // debounce (BARGE_IN_DEBOUNCE_FRAMES ≈ 320 ms) to filter + // speaker-to-mic feedback. + if is_speech { + *barge_in_frames += 1; + if *barge_in_frames >= BARGE_IN_DEBOUNCE_FRAMES { + // Real speech detected during TTS — trigger barge-in. + if let Some(cancel) = tts_cancel { + cancel.store(true, Ordering::Release); + } + *barge_in_frames = 0; + } + } else { + *barge_in_frames = 0; + } + // Don't accumulate speech during TTS (echo prevention). + *in_speech = false; + speech_buf.clear(); + *silence_frames = 0; + continue; + } + + // TTS not playing — check cooldown window. + if let Some(stopped) = *tts_stopped_at { + if stopped.elapsed() < TTS_COOLDOWN { + // Still in cooldown — discard but keep tracking speech state. + if !is_speech { + *in_speech = false; + } + speech_buf.clear(); + *silence_frames = 0; + *barge_in_frames = 0; + continue; + } else { + // Cooldown expired — clear the timer and reset all segment state. + *tts_stopped_at = None; + *in_speech = false; + *silence_frames = 0; + *barge_in_frames = 0; + } + } + + if is_speech { + *silence_frames = 0; + *in_speech = true; + speech_buf.extend_from_slice(&frame); + + // OOM guard: flush and reset if the buffer exceeds 30 s of audio. + if speech_buf.len() >= MAX_SPEECH_SAMPLES { + flush_to_stt(speech_buf, recognizer, text_tx); + speech_buf.clear(); + *silence_frames = 0; + *in_speech = false; + } + } else if *in_speech { + // Still accumulate during brief silence gaps. + speech_buf.extend_from_slice(&frame); + *silence_frames += 1; + + // In PTT mode, don't flush on silence — accumulate the entire + // key-hold as one utterance. The PTT release edge in the main + // loop handles the flush. In VAD mode, flush after the silence + // threshold so each natural pause becomes a separate message. + if ptt_active.is_none() && *silence_frames >= SILENCE_FLUSH_FRAMES { + // End of utterance — transcribe. + flush_to_stt(speech_buf, recognizer, text_tx); + speech_buf.clear(); + *silence_frames = 0; + *in_speech = false; + } + } + // If not in speech and not accumulating, just discard the frame. + } +} + +/// Run sherpa-onnx on the accumulated speech buffer and send the text. +/// +/// Uses `blocking_send` because this runs on a `std::thread` (not async). +/// The tokio channel's `blocking_send` is safe to call from sync contexts. +fn flush_to_stt( + speech_buf: &[f32], + recognizer: &sherpa_onnx::OfflineRecognizer, + text_tx: &tokio_mpsc::Sender, +) { + if speech_buf.is_empty() { + return; + } + + let stream = recognizer.create_stream(); + stream.accept_waveform(16_000, speech_buf); + recognizer.decode(&stream); + + let text = stream + .get_result() + .map(|r| r.text.trim().to_string()) + .unwrap_or_default(); + + if !text.is_empty() { + if let Err(e) = text_tx.blocking_send(text) { + eprintln!("sprout-desktop: STT text channel closed: {e}"); + } + } +} + +/// Convert raw bytes (f32 LE) to f32 samples. +/// Caller should ensure `bytes.len() % 4 == 0`; extra bytes are silently truncated. +/// +/// Assumes little-endian — matches all current Tauri targets (macOS ARM64, +/// Windows/Linux x86). The JS AudioWorklet's Float32Array uses platform-native +/// byte order, which is LE on all supported platforms. +fn bytes_to_f32(bytes: &[u8]) -> Vec { + bytes + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect() +} + +// drain_until_shutdown lives in super (huddle/mod.rs) — shared with tts.rs. +use super::drain_until_shutdown; diff --git a/desktop/src-tauri/src/huddle/tts.rs b/desktop/src-tauri/src/huddle/tts.rs new file mode 100644 index 00000000000..054121cb463 --- /dev/null +++ b/desktop/src-tauri/src/huddle/tts.rs @@ -0,0 +1,440 @@ +//! Text-to-Speech pipeline for huddle agent voice output. +//! +//! Mental model: +//! +//! ```text +//! caller: pipeline.speak("Hello world. How are you?") +//! → bounded sync_channel (TEXT_QUEUE_DEPTH = 8) +//! → tts_worker thread (owns 1 Kokoro engine) +//! 1. Preprocess text +//! 2. Split into sentences +//! 3. Synthesize each sentence individually → f32 PCM +//! 4. Apply volume boost + fade in/out to each sentence +//! 5. Append each buffer to a single rodio Player (gapless playback) +//! 6. Wait for player.empty() before accepting next text item +//! → tts_active = true while playing, false when idle +//! → cancel flag: player.clear() + drain queue +//! ``` +//! +//! Lookahead pipelining: synthesis of sentence N+1 begins immediately after +//! appending sentence N to the Player. Rodio queues buffers and plays them +//! sequentially — synthesis overlaps with playback for near-zero gaps. +//! +//! `tts_active` is an `Arc` shared with the STT pipeline so STT +//! can gate microphone input while the agent is speaking. + +use std::{ + num::NonZero, + path::PathBuf, + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc::{self, SyncSender}, + Arc, + }, + thread, + time::Duration, +}; + +use super::kokoro::{load_text_to_speech, load_voice_style, SAMPLE_RATE}; +use super::preprocessing::{preprocess_for_tts, split_sentences}; + +// ── Constants ───────────────────────────────────────────────────────────────── + +/// Maximum number of queued text items. +/// Prevents unbounded accumulation when the agent produces text faster than +/// TTS can play it. Excess items are dropped with a warning. +const TEXT_QUEUE_DEPTH: usize = 8; + +/// How long the worker waits on the text channel before checking the shutdown flag. +const RECV_TIMEOUT: Duration = Duration::from_millis(100); + +/// Kokoro ignores denoising steps (not a diffusion model). Kept for API compat. +const SYNTH_STEPS: usize = 1; + +/// Synthesis speed multiplier. Slightly faster than natural speech. +const SYNTH_SPEED: f32 = 1.05; + +/// Volume boost applied after synthesis — Kokoro output is normalized. +/// Start at 1.5 and tune empirically. +const VOLUME_BOOST: f32 = 1.5; + +/// Fade in/out length in samples (8ms at 24kHz ≈ 192 samples). +/// Eliminates clicks/pops at sentence boundaries. +const FADE_SAMPLES: usize = (SAMPLE_RATE as f64 * 0.008) as usize; + +/// Sentence-by-sentence synthesis for lower TTFA (≈200ms vs ≈600ms for 3-sentence batches). +const BATCH_SIZE: usize = 1; + +/// Silence inserted between sentences by the TTS pipeline (seconds). +/// Injected as a silent buffer between each synthesized sentence chunk. +const INTER_SENTENCE_SILENCE: f32 = 0.1; + +// ── Public pipeline handle ──────────────────────────────────────────────────── + +/// Handle to the running TTS pipeline. +/// +/// Not Clone — wrap in `Arc` to share across threads. +#[derive(Debug)] +pub struct TtsPipeline { + /// Send preprocessed text into the pipeline. + text_tx: SyncSender, + /// `true` while the agent is speaking. Shared with the STT pipeline for gating. + #[allow(dead_code)] + pub tts_active: Arc, + /// Signals the worker thread to stop. + shutdown: Arc, + /// Cancel flag: worker drains the queue and stops current playback. + /// Kept alive here so the Arc isn't dropped — the worker holds a clone. + #[allow(dead_code)] + cancel: Arc, + /// Voice name (e.g. "af_heart"). Stored for future voice-switching support. + #[allow(dead_code)] + voice: String, + /// Worker thread handle — taken on drop to join cleanly. + thread: Option>, +} + +impl TtsPipeline { + /// Spawn the TTS pipeline thread using the default voice. + /// + /// `model_dir` must contain the Kokoro model files: + /// `model_quantized.onnx`, `tokenizer.json`, `voices/.bin` + /// + /// `tts_active` is set to `true` while audio is playing and `false` when idle. + /// Pass the same `Arc` to the STT pipeline to gate microphone input. + /// + /// `cancel` is the shared barge-in flag from `HuddleState.tts_cancel`. Pass the + /// same `Arc` to the STT pipeline so both sides reference the same flag for the + /// entire huddle session — no stale references after pipeline restarts. + pub fn new( + model_dir: PathBuf, + tts_active: Arc, + cancel: Arc, + ) -> Result { + use super::kokoro::DEFAULT_VOICE; + Self::new_with_voice(model_dir, tts_active, cancel, DEFAULT_VOICE) + } + + /// Spawn the TTS pipeline thread with a specific voice name (e.g. `"af_heart"`, `"am_michael"`). + pub fn new_with_voice( + model_dir: PathBuf, + tts_active: Arc, + cancel: Arc, + voice: &str, + ) -> Result { + let (text_tx, text_rx) = mpsc::sync_channel::(TEXT_QUEUE_DEPTH); + let shutdown = Arc::new(AtomicBool::new(false)); + // cancel is passed in from HuddleState.tts_cancel — shared with STT for barge-in. + + let shutdown_worker = Arc::clone(&shutdown); + let cancel_worker = Arc::clone(&cancel); + let tts_active_worker = Arc::clone(&tts_active); + let voice_name = voice.to_string(); + let model_dir_worker = model_dir.clone(); + + let handle = thread::Builder::new() + .name("tts-worker".into()) + .spawn(move || { + tts_worker( + model_dir_worker, + voice_name, + text_rx, + tts_active_worker, + shutdown_worker, + cancel_worker, + ) + }) + .map_err(|e| format!("failed to spawn tts-worker thread: {e}"))?; + + Ok(Self { + text_tx, + tts_active, + shutdown, + cancel, + voice: voice.to_string(), + thread: Some(handle), + }) + } + + /// Queue `text` for TTS synthesis and playback. + /// + /// Non-blocking. Returns `Err` if the queue is full (bounded at + /// `TEXT_QUEUE_DEPTH`) — caller may log and discard. + pub fn speak(&self, text: String) -> Result<(), String> { + self.text_tx.try_send(text).map_err(|e| { + eprintln!("sprout-desktop: TTS queue saturated, dropping message: {e}"); + format!("TTS queue full, dropping: {e}") + }) + } + + /// Signal the worker thread to stop. + pub fn shutdown(&self) { + self.shutdown.store(true, Ordering::Release); + } + + /// Returns `true` if the worker thread has exited (init failure, crash, or normal exit). + /// Used by hot-start to detect dead pipelines and clear them for retry. + pub fn is_finished(&self) -> bool { + self.thread.as_ref().map_or(true, |h| h.is_finished()) + } +} + +impl Drop for TtsPipeline { + fn drop(&mut self) { + self.shutdown.store(true, Ordering::Release); + // Dropping `text_tx` unblocks the worker's recv_timeout loop. + // Join to ensure the audio thread exits cleanly. + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +// ── Worker thread ───────────────────────────────────────────────────────────── + +fn tts_worker( + model_dir: PathBuf, + voice_name: String, + text_rx: mpsc::Receiver, + tts_active: Arc, + shutdown: Arc, + cancel: Arc, +) { + // ── 1. Initialise Kokoro engine ─────────────────────────────────────────── + let model_dir_str = model_dir.to_string_lossy().to_string(); + + let mut engine = match load_text_to_speech(&model_dir_str) { + Ok(e) => e, + Err(e) => { + eprintln!( + "sprout-desktop: TTS Kokoro init failed (model_dir={}): {e}. TTS disabled.", + model_dir.display() + ); + drain_until_shutdown(text_rx, &shutdown); + return; + } + }; + + // ── 2. Load voice style ─────────────────────────────────────────────────── + let voice_path = model_dir.join(format!("{voice_name}.bin")); + let style = match load_voice_style(&voice_path) { + Ok(s) => s, + Err(e) => { + eprintln!( + "sprout-desktop: TTS voice style load failed ({voice_name}): {e}. TTS disabled." + ); + drain_until_shutdown(text_rx, &shutdown); + return; + } + }; + + // ── 3. Initialise rodio output device ───────────────────────────────────── + use rodio::{DeviceSinkBuilder, Player}; + + let sink_handle = match DeviceSinkBuilder::open_default_sink() { + Ok(h) => h, + Err(e) => { + eprintln!("sprout-desktop: TTS audio output failed: {e}. TTS disabled."); + drain_until_shutdown(text_rx, &shutdown); + return; + } + }; + + // Prime the audio output stream with a short silent buffer. + // On macOS, CoreAudio initializes the output device lazily on first use. + // Without this, the first real Player races against device startup and + // player.empty() returns true before audio has started draining — causing + // the first TTS message to be truncated after a few words. + { + use rodio::buffer::SamplesBuffer; + let channels = NonZero::new(1u16).unwrap(); + let rate = NonZero::new(SAMPLE_RATE).unwrap(); + let silence = vec![0.0f32; SAMPLE_RATE as usize / 10]; // 100ms of silence + let player = Player::connect_new(&sink_handle.mixer()); + player.append(SamplesBuffer::new(channels, rate, silence)); + // Wait for the silent buffer to drain — this ensures the output stream + // is fully initialized before the main loop creates its first Player. + while !player.empty() { + thread::sleep(Duration::from_millis(10)); + } + } + + // ── 4. Main loop ────────────────────────────────────────────────────────── + loop { + // Check shutdown/cancel before blocking (no player yet). + if handle_cancel_or_shutdown(&cancel, &shutdown, &tts_active, &text_rx, None) { + if shutdown.load(Ordering::Acquire) { + break; + } + continue; + } + + let raw_text = match text_rx.recv_timeout(RECV_TIMEOUT) { + Ok(t) => t, + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(mpsc::RecvTimeoutError::Disconnected) => break, + }; + + // Check cancel again after unblocking — a cancel may have arrived + // while we were waiting. + if handle_cancel_or_shutdown(&cancel, &shutdown, &tts_active, &text_rx, None) { + if shutdown.load(Ordering::Acquire) { + break; + } + continue; + } + + // Preprocess text. + let text = preprocess_for_tts(&raw_text); + if text.is_empty() { + continue; + } + + // Split into sentences. Each sentence is synthesized individually and + // appended to the Player immediately — synthesis of sentence N+1 overlaps + // with playback of sentence N (lookahead pipelining). + let sentences: Vec = split_sentences(&text) + .into_iter() + .filter(|s| !s.trim().is_empty()) + .collect(); + + if sentences.is_empty() { + continue; + } + + use rodio::buffer::SamplesBuffer; + let channels = match NonZero::new(1u16) { + Some(c) => c, + None => { + eprintln!("sprout-desktop: TTS channel count invariant violated"); + break; + } + }; + let rate = match NonZero::new(SAMPLE_RATE) { + Some(r) => r, + None => { + eprintln!("sprout-desktop: TTS sample rate invariant violated"); + break; + } + }; + + // Single persistent Player — all sentences append here, rodio plays + // them gaplessly without per-sentence device setup overhead. + let player = Player::connect_new(&sink_handle.mixer()); + // NOTE: tts_active is set AFTER the first player.append(), not before. + // Setting it before synthesis would cause STT to discard user speech + // during the synthesis window as "echo" even though no audio is + // actually playing yet. See crossfire review C3. + let mut first_append = true; + + // Lookahead pipeline: synthesize each sentence and append immediately. + // Rodio queues buffers sequentially — synthesis of the next sentence + // overlaps with playback of the current one. + let silence_samples = (INTER_SENTENCE_SILENCE * SAMPLE_RATE as f32) as usize; + let silence_buf = vec![0.0f32; silence_samples]; + for sentence in &sentences { + if handle_cancel_or_shutdown(&cancel, &shutdown, &tts_active, &text_rx, Some(&player)) { + break; + } + + let text = sentence.trim(); + if text.is_empty() { + continue; + } + + match engine.synth_chunk(text, "en", &style, SYNTH_STEPS, SYNTH_SPEED) { + Ok(samples) if !samples.is_empty() => { + let mut boosted: Vec = samples + .iter() + .map(|&s| (s * VOLUME_BOOST).clamp(-1.0, 1.0)) + .collect(); + apply_fades(&mut boosted); + player.append(SamplesBuffer::new(channels, rate, boosted)); + // Insert inter-sentence silence after each synthesized chunk. + player.append(SamplesBuffer::new(channels, rate, silence_buf.clone())); + if first_append { + tts_active.store(true, Ordering::Release); + first_append = false; + } + } + Ok(_) => {} + Err(e) => { + eprintln!("sprout-desktop: TTS synth failed: {e}"); + } + } + } + + // Wait for all queued audio to finish playing. + loop { + if handle_cancel_or_shutdown(&cancel, &shutdown, &tts_active, &text_rx, Some(&player)) { + break; + } + if player.empty() { + break; + } + thread::sleep(Duration::from_millis(50)); + } + + tts_active.store(false, Ordering::Release); + + if shutdown.load(Ordering::Acquire) { + break; + } + } + + tts_active.store(false, Ordering::Release); +} + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +/// Check for cancel or shutdown. Returns `true` if the caller should break/continue. +/// On cancel: drains the text queue and clears the cancel flag. +fn handle_cancel_or_shutdown( + cancel: &AtomicBool, + shutdown: &AtomicBool, + tts_active: &AtomicBool, + text_rx: &mpsc::Receiver, + player: Option<&rodio::Player>, +) -> bool { + if shutdown.load(Ordering::Acquire) { + if let Some(p) = player { + p.clear(); + } + tts_active.store(false, Ordering::Release); + return true; + } + if cancel.load(Ordering::Acquire) { + if let Some(p) = player { + p.clear(); + } + while text_rx.try_recv().is_ok() {} + cancel.store(false, Ordering::Release); + tts_active.store(false, Ordering::Release); + return true; + } + false +} + +/// Apply a short linear fade-in at the start and fade-out at the end of `samples`. +/// +/// Uses `FADE_SAMPLES` (8ms) or half the buffer length, whichever is smaller. +/// Eliminates clicks/pops at sentence boundaries. +fn apply_fades(samples: &mut Vec) { + let len = samples.len(); + let fade = FADE_SAMPLES.min(len / 2); + // Fade in: ramp from 0 → 1 over `fade` samples. + for i in 0..fade { + samples[i] *= i as f32 / fade as f32; + } + // Fade out: ramp from 1 → 0 over the last `fade` samples. + for i in 0..fade { + samples[len - 1 - i] *= i as f32 / fade as f32; + } +} + +// drain_until_shutdown lives in super (huddle/mod.rs) — shared with stt.rs. +use super::drain_until_shutdown; + +// BATCH_SIZE is used implicitly (one sentence per iteration). Suppress dead_code +// lint since it documents the design intent. +const _: () = assert!(BATCH_SIZE == 1); diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index f26ab2d81e4..b04648366a7 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -1,6 +1,7 @@ mod app_state; mod commands; mod events; +mod huddle; mod managed_agents; mod migration; mod models; @@ -9,6 +10,12 @@ mod util; use app_state::{build_app_state, resolve_persisted_identity, AppState}; use commands::*; +use huddle::{ + add_agent_to_huddle, check_pipeline_hotstart, confirm_huddle_active, download_voice_models, + end_huddle, get_huddle_agent_pubkeys, get_huddle_state, get_model_status, get_voice_input_mode, + join_huddle, leave_huddle, push_audio_pcm, set_tts_enabled, set_voice_input_mode, + speak_agent_message, start_huddle, start_stt_pipeline, +}; use managed_agents::{ ensure_nest, find_managed_agent_mut, kill_stale_tracked_processes, load_managed_agents, save_managed_agents, start_managed_agent_process, sync_managed_agent_processes, BackendKind, @@ -18,7 +25,7 @@ use std::sync::{ atomic::{AtomicBool, Ordering}, Arc, }; -use tauri::{http, Manager, RunEvent}; +use tauri::{http, Emitter, Manager, RunEvent}; use tauri_plugin_window_state::StateFlags; fn restore_managed_agents_on_launch( @@ -345,7 +352,94 @@ pub fn run() { ) .plugin(tauri_plugin_websocket::init()) .plugin(tauri_plugin_dialog::init()) - .plugin(tauri_plugin_process::init()); + .plugin(tauri_plugin_process::init()) + .plugin({ + use tauri_plugin_global_shortcut::ShortcutState; + + // Generation counter for the release delay task. Incremented on + // every press — a delayed release only fires if the generation + // hasn't changed (i.e. no new press happened during the delay). + // This prevents press→release→press within 200 ms from having + // the first release clobber the second press. + let ptt_press_gen = Arc::new(std::sync::atomic::AtomicU64::new(0)); + + tauri_plugin_global_shortcut::Builder::new() + .with_handler(move |app, _shortcut, event| { + let state = match app.try_state::() { + Some(s) => s, + None => return, + }; + + // Only act if a huddle is active and mode is PTT. + let (is_ptt_mode, is_active) = match state.huddle_state.lock() { + Ok(hs) => ( + hs.voice_input_mode == huddle::VoiceInputMode::PushToTalk, + matches!( + hs.phase, + huddle::HuddlePhase::Connected | huddle::HuddlePhase::Active + ), + ), + Err(_) => return, + }; + + if !is_ptt_mode || !is_active { + return; + } + + match event.state { + ShortcutState::Pressed => { + // Bump generation — invalidates any pending release delay. + ptt_press_gen.fetch_add(1, std::sync::atomic::Ordering::Release); + + if let Ok(hs) = state.huddle_state.lock() { + hs.ptt_active + .store(true, std::sync::atomic::Ordering::Release); + // Only cancel TTS if it's actually playing — avoids + // a stale cancel flag that drops the next queued message. + if hs.tts_active.load(std::sync::atomic::Ordering::Acquire) { + hs.tts_cancel + .store(true, std::sync::atomic::Ordering::Release); + } + } + // Emit ptt-state=true to the frontend. + // The React side plays the press audio cue on this event + // (Web Audio API via HuddleContext). Rust-side rodio audio + // was considered but rejected: the rodio OutputStream must + // outlive the handler and sharing it across the shortcut + // closure adds lifecycle complexity for marginal gain. + // The React implementation is sufficient and simpler. + let _ = app.emit("ptt-state", true); + } + ShortcutState::Released => { + // Capture generation at release time. + let gen_at_release = + ptt_press_gen.load(std::sync::atomic::Ordering::Acquire); + let gen_arc = Arc::clone(&ptt_press_gen); + let app_handle = app.clone(); + // 200 ms release delay — captures the tail of the utterance. + // Only applies if no new press happened during the delay. + tauri::async_runtime::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + // Check generation — if it changed, a new press arrived. + if gen_arc.load(std::sync::atomic::Ordering::Acquire) + != gen_at_release + { + return; // Superseded by a new press. + } + if let Some(state) = app_handle.try_state::() { + if let Ok(hs) = state.huddle_state.lock() { + hs.ptt_active + .store(false, std::sync::atomic::Ordering::Release); + } + } + // Emit ptt-state=false — React plays the release audio cue. + let _ = app_handle.emit("ptt-state", false); + }); + } + } + }) + .build() + }); // Only register the updater in release builds that were compiled with a // real updater configuration. Local unsigned builds omit that config and @@ -385,6 +479,14 @@ pub fn run() { // that will be lost on restart, as that silently breaks channel // memberships, DMs, and relay identity. let state = app_handle.state::(); + + // Store the AppHandle so huddle commands can emit `huddle-state-changed` + // events via `huddle::emit_huddle_state` without threading the handle + // through every call site. + if let Ok(mut guard) = state.app_handle.lock() { + *guard = Some(app_handle.clone()); + } + resolve_persisted_identity(&app_handle, &state) .map_err(|e| -> Box { e.into() })?; @@ -395,6 +497,25 @@ pub fn run() { eprintln!("sprout-desktop: failed to create nest: {error}"); } + // Pre-download voice models in the background so they're ready + // when the user starts their first huddle. Idempotent — no-op if + // already downloaded. ~87 MB total (50 MB Moonshine + 87 MB Kokoro). + if let Some(mgr) = huddle::models::global_model_manager() { + mgr.start_moonshine_download(state.http_client.clone()); + mgr.start_kokoro_download(state.http_client.clone()); + } + + // Register PTT global shortcut (Ctrl+Space). + // Non-fatal: huddle works without the shortcut (user can switch to VAD mode). + #[cfg(desktop)] + { + use tauri_plugin_global_shortcut::{Code, GlobalShortcutExt, Modifiers, Shortcut}; + let shortcut = Shortcut::new(Some(Modifiers::CONTROL), Code::Space); + if let Err(e) = app.handle().global_shortcut().register(shortcut) { + eprintln!("sprout-desktop: failed to register PTT shortcut: {e}"); + } + } + // Keep launch-time agent restoration off the synchronous setup path // so the frontend can mount and reveal the window promptly. tauri::async_runtime::spawn_blocking(move || { @@ -501,6 +622,23 @@ pub fn run() { get_contact_list, set_contact_list, get_notes_timeline, + start_huddle, + join_huddle, + leave_huddle, + end_huddle, + get_huddle_state, + push_audio_pcm, + start_stt_pipeline, + download_voice_models, + get_model_status, + set_tts_enabled, + speak_agent_message, + add_agent_to_huddle, + check_pipeline_hotstart, + confirm_huddle_active, + get_huddle_agent_pubkeys, + set_voice_input_mode, + get_voice_input_mode, ]) .build(tauri::generate_context!()) .expect("error while building tauri application"); diff --git a/desktop/src-tauri/src/managed_agents/runtime.rs b/desktop/src-tauri/src/managed_agents/runtime.rs index a13755b0210..01485a5e3c0 100644 --- a/desktop/src-tauri/src/managed_agents/runtime.rs +++ b/desktop/src-tauri/src/managed_agents/runtime.rs @@ -513,6 +513,8 @@ pub fn start_managed_agent_process( .unwrap_or(super::types::DEFAULT_AGENT_MAX_TURN_DURATION_SECONDS); command.env("SPROUT_ACP_MAX_TURN_DURATION", max_dur.to_string()); command.env("SPROUT_ACP_AGENTS", record.parallelism.to_string()); + command.env("SPROUT_ACP_MULTIPLE_EVENT_HANDLING", "owner-interrupt"); + command.env("SPROUT_ACP_DEDUP", "queue"); command.env( "GOOSE_MODE", std::env::var("GOOSE_MODE").unwrap_or_else(|_| "auto".to_string()), diff --git a/desktop/src-tauri/tauri.conf.json b/desktop/src-tauri/tauri.conf.json index 5bdfc39f1c7..1d88b3f5c49 100644 --- a/desktop/src-tauri/tauri.conf.json +++ b/desktop/src-tauri/tauri.conf.json @@ -23,7 +23,8 @@ "titleBarStyle": "Overlay", "hiddenTitle": true, "dragDropEnabled": false, - "trafficLightPosition": { "x": 12, "y": 22 } + "trafficLightPosition": { "x": 12, "y": 22 }, + "backgroundThrottling": "disabled" } ], "security": { @@ -41,7 +42,8 @@ "icons/icon.ico" ], "macOS": { - "infoPlist": "Info.plist" + "infoPlist": "Info.plist", + "entitlements": "Entitlements.plist" } } } diff --git a/desktop/src/app/AppShell.tsx b/desktop/src/app/AppShell.tsx index 51007f4653e..1683e5840e6 100644 --- a/desktop/src/app/AppShell.tsx +++ b/desktop/src/app/AppShell.tsx @@ -29,6 +29,7 @@ import { import { usePresenceSession } from "@/features/presence/hooks"; import { useProfileQuery } from "@/features/profile/hooks"; import type { SettingsSection } from "@/features/settings/ui/SettingsPanels"; +import { HuddleBar, HuddleProvider } from "@/features/huddle"; import { AppSidebar } from "@/features/sidebar/ui/AppSidebar"; import { relayClient } from "@/shared/api/relayClient"; import { useIdentityQuery } from "@/shared/api/hooks"; @@ -385,166 +386,174 @@ export function AppShell() { }, }} > - -
- - - -
- { - const createdChannel = await createChannelMutation.mutateAsync({ - name, + + +
+ + + +
+ { - const createdForum = await createForumMutation.mutateAsync({ - name, + }) => { + const createdChannel = await createChannelMutation.mutateAsync({ + name, + description, + channelType: "stream", + visibility, + ttlSeconds, + }); + + await goChannel(createdChannel.id); + }} + onCreateForum={async ({ description, - channelType: "forum", + name, visibility, ttlSeconds, - }); - - await goChannel(createdForum.id); - }} - onHideDm={handleHideDm} - onOpenBrowseChannels={handleOpenBrowseChannels} - onOpenBrowseForums={handleOpenBrowseForums} - onOpenDm={async ({ pubkeys }) => { - const directMessage = await openDmMutation.mutateAsync({ - pubkeys, - }); - await goChannel(directMessage.id); - }} - onOpenSearch={handleOpenSearch} - onSelectAgents={() => { - void goAgents(); - }} - onSelectChannel={(channelId) => { - void goChannel(channelId); - }} - onSelectHome={() => { - void goHome(); - }} - onSelectPulse={() => { - void goPulse(); - }} - onSelectSettings={handleOpenSettings} - onSelectWorkflows={() => { - void goWorkflows(); - }} - onSetPresenceStatus={(status) => presenceSession.setStatus(status)} - profile={profileQuery.data} - selectedChannelId={selectedChannelId} - selectedView={selectedView} - unreadChannelIds={unreadChannelIds} - /> - - - - - - { - setIsChannelManagementOpen(false); - void goHome({ replace: true }); - }} - onOpenSearchResult={handleOpenSearchResult} - onSearchOpenChange={setIsSearchOpen} - onSelectChannel={(channelId) => { - void goChannel(channelId); - }} - /> - - {settingsOpen ? ( - - - - ) : null} - + }) => { + const createdForum = await createForumMutation.mutateAsync({ + name, + description, + channelType: "forum", + visibility, + ttlSeconds, + }); + + await goChannel(createdForum.id); + }} + onHideDm={handleHideDm} + onOpenBrowseChannels={handleOpenBrowseChannels} + onOpenBrowseForums={handleOpenBrowseForums} + onOpenDm={async ({ pubkeys }) => { + const directMessage = await openDmMutation.mutateAsync({ + pubkeys, + }); + await goChannel(directMessage.id); + }} + onOpenSearch={handleOpenSearch} + onSelectAgents={() => { + void goAgents(); + }} + onSelectChannel={(channelId) => { + void goChannel(channelId); + }} + onSelectHome={() => { + void goHome(); + }} + onSelectPulse={() => { + void goPulse(); + }} + onSelectSettings={handleOpenSettings} + onSelectWorkflows={() => { + void goWorkflows(); + }} + onSetPresenceStatus={(status) => + presenceSession.setStatus(status) + } + profile={profileQuery.data} + selectedChannelId={selectedChannelId} + selectedView={selectedView} + unreadChannelIds={unreadChannelIds} + /> + + + + + + { + setIsChannelManagementOpen(false); + void goHome({ replace: true }); + }} + onOpenSearchResult={handleOpenSearchResult} + onSearchOpenChange={setIsSearchOpen} + onSelectChannel={(channelId) => { + void goChannel(channelId); + }} + /> + + + + {settingsOpen ? ( + + + + ) : null} + + ); diff --git a/desktop/src/features/channels/ui/ChannelMembersBar.tsx b/desktop/src/features/channels/ui/ChannelMembersBar.tsx index 5c9c779244b..69e6b288fed 100644 --- a/desktop/src/features/channels/ui/ChannelMembersBar.tsx +++ b/desktop/src/features/channels/ui/ChannelMembersBar.tsx @@ -1,6 +1,8 @@ import { Plus, Settings2, Users, Zap } from "lucide-react"; import * as React from "react"; - +import { useQueryClient } from "@tanstack/react-query"; +import { useHuddle } from "@/features/huddle"; +import { HuddleIndicator } from "@/features/huddle/components/HuddleIndicator"; import { useAcpProvidersQuery, useBackendProvidersQuery, @@ -38,6 +40,8 @@ export function ChannelMembersBar({ }: ChannelMembersBarProps) { const [isAddBotOpen, setIsAddBotOpen] = React.useState(false); const [isCreateWorkflowOpen, setIsCreateWorkflowOpen] = React.useState(false); + const { startHuddle, isStarting: isStartingHuddle } = useHuddle(); + const queryClient = useQueryClient(); const membersQuery = useChannelMembersQuery(channel.id); const providersQuery = useAcpProvidersQuery(); const backendProvidersQuery = useBackendProvidersQuery(); @@ -186,6 +190,21 @@ export function ChannelMembersBar({ + { + try { + await startHuddle(channel.id, []); + // Refetch channels so the new ephemeral channel appears in the sidebar immediately + // (default poll interval is 60s — too slow for huddle UX). + void queryClient.invalidateQueries({ queryKey: ["channels"] }); + } catch (e) { + console.error("Failed to start huddle:", e); + } + }} + startDisabled={!canAddAgents || isStartingHuddle} + /> + + + )} + + {loading ? ( +

+ Loading agents… +

+ ) : runningAgents.length === 0 ? ( +

+ {agents.filter((a) => a.status === "running").length > 0 + ? "All running agents are already in this huddle." + : "No running agents found."} +

+ ) : ( +
    + {runningAgents.map((agent) => ( +
  • + +
    • + ))} +
+ )} + + + ); +} + +export type { AgentAddResult }; diff --git a/desktop/src/features/huddle/components/HuddleBar.tsx b/desktop/src/features/huddle/components/HuddleBar.tsx new file mode 100644 index 00000000000..aca1acf1aa5 --- /dev/null +++ b/desktop/src/features/huddle/components/HuddleBar.tsx @@ -0,0 +1,476 @@ +import { invoke } from "@tauri-apps/api/core"; +import { listen } from "@tauri-apps/api/event"; +import { + Mic, + MicOff, + PhoneOff, + Plus, + Users, + Volume2, + VolumeX, +} from "lucide-react"; +import * as React from "react"; + +import { cn } from "@/shared/lib/cn"; +import { Button } from "@/shared/ui/button"; +import { useHuddle } from "../HuddleContext"; +import { AddAgentDialog, type AgentAddResult } from "./AddAgentDialog"; +import { ParticipantList } from "./ParticipantList"; + +// Shape returned by the `get_huddle_state` Tauri command. +// NOTE: This mirrors the HuddleState struct in the Rust backend (src-tauri/src/huddle/mod.rs). +// If you add/remove fields here, update the Rust struct (and vice versa). +type HuddleState = { + phase: + | "idle" + | "creating" + | "connecting" + | "connected" + | "active" + | "leaving"; + parent_channel_id: string | null; + ephemeral_channel_id: string | null; + livekit_room: string | null; + participants: string[]; // pubkey hex strings + agent_pubkeys: string[]; + tts_enabled: boolean; + is_creator: boolean; + voice_input_mode: "push_to_talk" | "voice_activity"; +}; + +type HuddleBarProps = { + className?: string; +}; + +export function HuddleBar({ className }: HuddleBarProps) { + const { + localAudioTrack, + leaveHuddle, + endHuddle, + micConnected, + micLevel, + pttActive, + voiceInputMode, + setVoiceInputMode, + activeSpeakers, + isReconnecting, + huddleError, + clearHuddleError, + } = useHuddle(); + + const isPttMode = voiceInputMode === "push_to_talk"; + const [state, setState] = React.useState(null); + const [isMuted, setIsMuted] = React.useState(false); + // Derive TTS enabled from backend state (single source of truth). + // Fall back to true if state hasn't loaded yet. + const ttsEnabled = state?.tts_enabled ?? true; + const [isLeaving, setIsLeaving] = React.useState(false); + const [showAddAgent, setShowAddAgent] = React.useState(false); + const [agentAddError, setAgentAddError] = React.useState(null); + const [modelStatus, setModelStatus] = React.useState<{ + moonshine: string; + kokoro: string; + } | null>(null); + + // Huddle state: event-driven primary path + 10s fallback poll. + React.useEffect(() => { + let cancelled = false; + let unlisten: (() => void) | null = null; + + async function fetchState() { + try { + const s = await invoke("get_huddle_state"); + if (!cancelled) setState(s); + } catch { + // Only clear state if we never had an active huddle. + if (!cancelled) { + setState((prev) => + prev?.phase === "active" || prev?.phase === "connected" + ? prev + : null, + ); + } + } + } + + // Initial fetch + void fetchState(); + + // Primary: listen for Rust-emitted state change events + listen("huddle-state-changed", (event) => { + if (!cancelled) setState(event.payload); + }).then((fn) => { + if (cancelled) fn(); + else unlisten = fn; + }); + + // Fallback: 10s poll in case events are missed + const id = window.setInterval(() => void fetchState(), 10_000); + + return () => { + cancelled = true; + unlisten?.(); + window.clearInterval(id); + }; + }, []); + + // Poll model download status while huddle is active + const huddlePhase = state?.phase; + React.useEffect(() => { + if (huddlePhase !== "active" && huddlePhase !== "connected") return; + + let cancelled = false; + + // ModelStatus serializes as: "ready" | "not_downloaded" (strings) + // or { downloading: { progress_percent: N } } | { error: "msg" } (objects). + const fmt = (s: unknown): string => { + if (typeof s === "string") return s === "ready" ? "ready" : "pending"; + if (typeof s === "object" && s !== null) { + if ("downloading" in s) { + const d = (s as { downloading: { progress_percent: number } }) + .downloading; + return `${d.progress_percent}%`; + } + if ("error" in s) return "error"; + } + return "pending"; + }; + + async function pollModels() { + try { + const status = await invoke<{ + moonshine: unknown; + kokoro: unknown; + }>("get_model_status"); + if (cancelled) return; + + setModelStatus({ + moonshine: fmt(status.moonshine), + kokoro: fmt(status.kokoro), + }); + } catch { + // best-effort + } + } + + void pollModels(); + const id = window.setInterval(() => void pollModels(), 3_000); + + return () => { + cancelled = true; + window.clearInterval(id); + setModelStatus(null); // Clear stale status on huddle end/phase change. + }; + }, [huddlePhase]); + + // Sync mute state to the audio track + React.useEffect(() => { + if (localAudioTrack) { + localAudioTrack.enabled = !isMuted; + } + }, [isMuted, localAudioTrack]); + + if (!state || (state.phase !== "active" && state.phase !== "connected")) + return null; + + async function handleLeave() { + if (isLeaving) return; + setIsLeaving(true); + try { + const backendClean = await leaveHuddle(); + if (backendClean) { + setState(null); + } + // If backend cleanup failed, keep the bar visible so the user can retry. + // leaveHuddle retains rustActiveRef=true for the next attempt. + } catch (e) { + console.error("Failed to leave huddle:", e); + } finally { + setIsLeaving(false); + } + } + + async function handleEnd() { + if (isLeaving) return; + const confirmed = window.confirm( + "End the huddle for everyone? This will disconnect all participants.", + ); + if (!confirmed) return; + setIsLeaving(true); + try { + const backendClean = await endHuddle(); + if (backendClean) { + setState(null); + } + // If backend cleanup failed, keep the bar visible so the user can retry. + } catch (e) { + console.error("Failed to end huddle:", e); + } finally { + setIsLeaving(false); + } + } + + return ( +
+ {/* Error banner — dismissible, shown when start/join fails */} + {huddleError && ( +
+ {huddleError} + +
+ )} + + {/* Room label */} + Huddle + + {/* Huddle status */} +
+ + In huddle +
+ + {/* Reconnecting indicator */} + {isReconnecting && ( +
+ Reconnecting… +
+ )} + + {/* Model download progress */} + {modelStatus && + (modelStatus.moonshine !== "ready" || + modelStatus.kokoro !== "ready") && ( + + + {modelStatus.moonshine !== "ready" && + modelStatus.kokoro !== "ready" + ? `Voice models: STT ${modelStatus.moonshine}, TTS ${modelStatus.kokoro}` + : modelStatus.moonshine !== "ready" + ? `STT model: ${modelStatus.moonshine}` + : `TTS model: ${modelStatus.kokoro}`} + + + )} + + {/* Participant avatars */} + {state.participants.length > 0 && ( + + )} + + {/* Voice input mode indicator */} +
+ {micConnected ? ( + isPttMode ? ( + <> +
+ PTT + Ctrl+Space + + ) : ( + <> +
0.05 + ? `rgba(34, 197, 94, ${0.4 + micLevel * 0.6})` + : "rgba(100, 116, 139, 0.4)", + }} + title={`Mic level: ${Math.round(micLevel * 100)}%`} + /> + VAD + + ) + ) : ( + no mic + )} +
+ + {/* Voice input mode toggle */} + + + {/* Add agent button */} + + + {agentAddError && ( + + {agentAddError} + + )} + + {showAddAgent && ( + setShowAddAgent(false)} + onAdd={async (pubkey: string): Promise => { + setAgentAddError(null); + try { + return await invoke("add_agent_to_huddle", { + agentPubkey: pubkey, + }); + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e); + setAgentAddError(`Failed to add agent: ${msg}`); + throw e; // Re-throw so AddAgentDialog shows its inline error. + } + }} + /> + )} + + {/* Mute toggle — in PTT mode acts as hard mute override (even PTT won't transmit) */} + + + {/* TTS toggle */} + + + {/* Leave / End buttons — available to all participants */} + + + {state?.is_creator && ( + + )} + + {/* Screen reader announcements for huddle state changes */} + + {isReconnecting + ? "Huddle reconnecting" + : micConnected + ? "In huddle, microphone connected" + : "In huddle, no microphone"} + {`, voice input: ${isPttMode ? "push to talk, press Ctrl+Space to transmit" : "voice activity detection"}`} + {modelStatus && + modelStatus.moonshine !== "ready" && + `, STT model ${modelStatus.moonshine}`} + {modelStatus && + modelStatus.kokoro !== "ready" && + `, TTS model ${modelStatus.kokoro}`} + +
+ ); +} diff --git a/desktop/src/features/huddle/components/HuddleIndicator.tsx b/desktop/src/features/huddle/components/HuddleIndicator.tsx new file mode 100644 index 00000000000..424163f6bcd --- /dev/null +++ b/desktop/src/features/huddle/components/HuddleIndicator.tsx @@ -0,0 +1,237 @@ +import { Headphones } from "lucide-react"; +import * as React from "react"; +import { useQueryClient } from "@tanstack/react-query"; + +import { relayClient } from "@/shared/api/relayClient"; +import type { RelayEvent } from "@/shared/api/types"; +import { cn } from "@/shared/lib/cn"; +import { Button } from "@/shared/ui/button"; +import { useHuddle } from "../HuddleContext"; + +/** Huddle lifecycle event kinds */ +const KIND_HUDDLE_STARTED = 48100; +const KIND_HUDDLE_PARTICIPANT_JOINED = 48101; +const KIND_HUDDLE_PARTICIPANT_LEFT = 48102; +const KIND_HUDDLE_ENDED = 48103; + +type ActiveHuddle = { + ephemeralChannelId: string; + livekitRoom: string; + participants: Set; +}; + +type HuddleIndicatorProps = { + channelId: string; + className?: string; + /** Called when the user clicks the button and no huddle is active (start). */ + onStart?: () => void; + /** Whether the start action is disabled (e.g., permissions, already starting). */ + startDisabled?: boolean; +}; + +/** + * Detects active huddles in a channel via kind:48100-48103 events. + * Shows a glowing headphone icon when a huddle is active, with participant count. + * Click to join the huddle. + */ +export function HuddleIndicator({ + channelId, + className, + onStart, + startDisabled, +}: HuddleIndicatorProps) { + const { joinHuddle, isStarting } = useHuddle(); + const queryClient = useQueryClient(); + const [activeHuddle, setActiveHuddle] = React.useState( + null, + ); + const [isJoining, setIsJoining] = React.useState(false); + + React.useEffect(() => { + if (!channelId) return; + + let disposed = false; + let cleanup: (() => void) | null = null; + + // Track all seen events for reconstruction. Keyed by event.id for dedup. + const seenEvents = new Map(); + + /** Reconstruct huddle state from the full set of seen events. + * Sort by created_at, then kind (causal: start < join < left < end), + * then event id for final tiebreak. This handles out-of-order delivery, + * reconnect replay, late mounts, and same-second event batches. + * + * Resilient to missing start event: if we see join/left events for an + * ephemeral channel without a prior start, we infer the huddle exists. + * This covers the edge case where >100 lifecycle events push the start + * event out of the subscription window. */ + function reconstruct() { + const sorted = [...seenEvents.values()].sort( + (a, b) => + a.created_at - b.created_at || + a.kind - b.kind || + a.id.localeCompare(b.id), + ); + + let huddle: ActiveHuddle | null = null; + + for (const ev of sorted) { + let ephId: string | null = null; + let room = ""; + try { + const content = JSON.parse(ev.content); + ephId = content.ephemeral_channel_id ?? null; + room = content.livekit_room ?? ""; + } catch { + continue; // Malformed — skip + } + + switch (ev.kind) { + case KIND_HUDDLE_STARTED: { + if (!ephId) break; + huddle = { + ephemeralChannelId: ephId, + livekitRoom: room, + participants: new Set([ev.pubkey]), + }; + break; + } + case KIND_HUDDLE_PARTICIPANT_JOINED: { + if (!ephId) break; + // Infer huddle exists if we missed the start event (late mount + // or >100 lifecycle events pushed it out of the window). + if (!huddle || ephId !== huddle.ephemeralChannelId) { + huddle = { + ephemeralChannelId: ephId, + livekitRoom: room, + participants: new Set(), + }; + } + huddle.participants.add(ev.pubkey); + break; + } + case KIND_HUDDLE_PARTICIPANT_LEFT: { + if (!ephId) break; + // Infer huddle exists from LEFT too — if the window starts with + // only LEFT events (all joiners departed, creator still active), + // we still need to know the huddle is alive. + if (!huddle || ephId !== huddle.ephemeralChannelId) { + huddle = { + ephemeralChannelId: ephId, + livekitRoom: room, + participants: new Set(), + }; + } + huddle.participants.delete(ev.pubkey); + break; + } + case KIND_HUDDLE_ENDED: { + if (!huddle || !ephId || ephId !== huddle.ephemeralChannelId) break; + huddle = null; + break; + } + } + } + + if (!disposed) { + setActiveHuddle(huddle); + } + } + + // Subscribe to huddle lifecycle events only (kinds 48100–48103). + // limit: 100 covers long-lived huddles with many join/leave cycles. + relayClient + .subscribeToHuddleEvents(channelId, (event: RelayEvent) => { + if (disposed) return; + + // Dedup by event ID — ignore replayed events from reconnect. + if (seenEvents.has(event.id)) return; + seenEvents.set(event.id, event); + + // Reconstruct from full history on every new event. + // This is cheap — huddle lifecycle events are rare (typically <20). + reconstruct(); + }) + .then((dispose) => { + if (disposed) { + void dispose(); + return; + } + cleanup = () => void dispose(); + }) + .catch((err) => { + console.error("[HuddleIndicator] subscription failed:", err); + }); + + return () => { + disposed = true; + cleanup?.(); + setActiveHuddle(null); + }; + }, [channelId]); + + // No active huddle — render the start button (if onStart provided). + if (!activeHuddle) { + if (!onStart) return null; + return ( + + ); + } + + // At least 1 participant must exist for the huddle to be active. + // When START fell out of the event window, the creator isn't in the + // reconstructed set — floor at 1 to avoid showing "0 participants". + const participantCount = Math.max(1, activeHuddle.participants.size); + + async function handleJoin() { + if (!activeHuddle || isJoining) return; + setIsJoining(true); + try { + await joinHuddle( + channelId, + activeHuddle.ephemeralChannelId, + activeHuddle.livekitRoom, + ); + // Refetch channels so the ephemeral channel appears in the sidebar. + void queryClient.invalidateQueries({ queryKey: ["channels"] }); + } catch (e) { + console.error("Failed to join huddle:", e); + } finally { + setIsJoining(false); + } + } + + return ( + + ); +} diff --git a/desktop/src/features/huddle/components/ParticipantList.tsx b/desktop/src/features/huddle/components/ParticipantList.tsx new file mode 100644 index 00000000000..f53666c8583 --- /dev/null +++ b/desktop/src/features/huddle/components/ParticipantList.tsx @@ -0,0 +1,112 @@ +import * as React from "react"; + +import { cn } from "@/shared/lib/cn"; +import { useUsersBatchQuery } from "@/features/profile/hooks"; +import { ProfileAvatar } from "@/features/profile/ui/ProfileAvatar"; + +type ParticipantListProps = { + /** Pubkey hex strings from the Rust huddle state */ + participants: string[]; + activeSpeakers?: string[]; + /** Pubkeys of agent participants — rendered with a bot badge */ + agentPubkeys?: string[]; + className?: string; +}; + +export function ParticipantList({ + participants, + activeSpeakers, + agentPubkeys, + className, +}: ParticipantListProps) { + const { data } = useUsersBatchQuery(participants); + const profiles = data?.profiles ?? {}; + const agentSet = React.useMemo( + () => new Set(agentPubkeys ?? []), + [agentPubkeys], + ); + + if (participants.length === 0) return null; + + return ( +
    + {participants.map((pubkey) => { + const profile = profiles[pubkey.toLowerCase()]; + const hasProfile = profile?.displayName || profile?.avatarUrl; + const isActive = activeSpeakers?.includes(pubkey); + const isAgent = agentSet.has(pubkey); + const ariaLabel = `${profile?.displayName || `Participant ${pubkey.slice(0, 8)}`}${isAgent ? " (agent)" : ""}`; + + return ( +
  • + {hasProfile ? ( +
    + +
    + ) : ( + + )} + {isAgent && ( + + )} +
    • + ); + })} +
+ ); +} + +/** Compact hex-prefix avatar for participants without a loaded profile. */ +function HexAvatar({ + pubkey, + activeSpeakers, + ariaLabel, +}: { + pubkey: string; + activeSpeakers?: string[]; + ariaLabel?: string; +}) { + const shortId = pubkey.slice(0, 6).toUpperCase(); + const parsed = parseInt(pubkey.slice(0, 4), 16); + const hue = Number.isNaN(parsed) ? 0 : parsed % 360; + const sat = Number.isNaN(parsed) ? 0 : 60; + const isActive = activeSpeakers?.includes(pubkey); + + return ( +
+ {shortId} +
+ ); +} diff --git a/desktop/src/features/huddle/index.ts b/desktop/src/features/huddle/index.ts new file mode 100644 index 00000000000..8cf5ce8ea8b --- /dev/null +++ b/desktop/src/features/huddle/index.ts @@ -0,0 +1,6 @@ +export { HuddleProvider, useHuddle } from "./HuddleContext"; +export { connectToHuddle } from "./lib/livekit"; +export type { HuddleConnection, HuddleRoomCallbacks } from "./lib/livekit"; +export { setupAudioWorklet } from "./lib/audioWorklet"; +export { HuddleBar } from "./components/HuddleBar"; +export { ParticipantList } from "./components/ParticipantList"; diff --git a/desktop/src/features/huddle/lib/audioWorklet.ts b/desktop/src/features/huddle/lib/audioWorklet.ts new file mode 100644 index 00000000000..4022e8b859e --- /dev/null +++ b/desktop/src/features/huddle/lib/audioWorklet.ts @@ -0,0 +1,141 @@ +import { listen, type UnlistenFn } from "@tauri-apps/api/event"; + +/** + * Raw binary invoke — uses Tauri's internal IPC for zero-copy ArrayBuffer transfer. + * + * The typed @tauri-apps/api doesn't support raw binary payloads (InvokeBody::Raw). + * This wrapper isolates the internal API dependency to a single call site. + * Tested against Tauri v2. If this breaks on upgrade, only this function needs updating. + */ +function invokeRawBinary(cmd: string, payload: Uint8Array): Promise { + // biome-ignore lint/suspicious/noExplicitAny: Tauri internals have no public type definition + const internals = (window as any).__TAURI_INTERNALS__; + if (!internals?.invoke) { + return Promise.reject(new Error("Tauri internals not available")); + } + return internals.invoke(cmd, payload); +} + +/** Return type for setupAudioWorklet — stop + mode control. */ +export type AudioWorkletHandle = { + stop: () => void; + /** Send PTT state to the worklet processor. */ + setTransmitting: (active: boolean) => void; + /** Switch voice input mode. In VAD mode, always transmitting (PTT events ignored). + * In PTT mode, gated by Ctrl+Space. */ + setMode: (mode: "push_to_talk" | "voice_activity") => void; +}; + +/** + * AudioWorklet → Rust STT pipeline: + * + * MediaStreamTrack (mic, 48kHz) + * → AudioContext.createMediaStreamSource() + * → AudioWorkletNode("stt-tap-processor") + * worklet.js accumulates 100ms batches (4800 samples) + * posts Float32Array to main thread via port.postMessage + * → onmessage: convert to Uint8Array view (zero-copy) + * → invokeRawBinary("push_audio_pcm", bytes) + * Rust: SttPipeline::push_audio → bounded sync_channel + * + * PTT gating: + * Main thread listens for Tauri "ptt-state" events (from Rust global shortcut) + * and forwards them to the worklet via port.postMessage({ type: 'ptt', active }). + * The worklet discards audio frames when transmitting=false. + * + * @param audioTrack - Mic track from LiveKit + * @param initialTransmitting - Initial PTT state. true=open mic (VAD), false=muted until PTT press. + */ +export async function setupAudioWorklet( + audioTrack: MediaStreamTrack, + initialTransmitting = true, +): Promise { + const audioContext = new AudioContext({ sampleRate: 48000 }); + + // Resume after user gesture (required by autoplay policy) + if (audioContext.state === "suspended") { + await audioContext.resume(); + } + + // Load the worklet processor (must live in public/ for Vite to serve it) + await audioContext.audioWorklet.addModule("/worklet.js"); + + // Create source from the mic track + const source = audioContext.createMediaStreamSource( + new MediaStream([audioTrack]), + ); + + // Create worklet node + const workletNode = new AudioWorkletNode(audioContext, "stt-tap-processor"); + + // Connect: mic → worklet (tap only — no playback) + source.connect(workletNode); + + // Set initial PTT state (worklet defaults to transmitting=true). + // In PTT mode, immediately gate audio until the user presses the key. + if (!initialTransmitting) { + workletNode.port.postMessage({ type: "ptt", active: false }); + } + + // Forward PCM batches to Rust via raw binary invoke. + // Direction: worklet→main (receives PCM data from worklet processor). + workletNode.port.onmessage = (event: MessageEvent) => { + const float32 = event.data; + // Fire-and-forget — Rust side uses try_send which drops on backpressure. + // No await: prevents main-thread backpressure from slow Rust processing. + // Create a zero-copy Uint8Array view over the same underlying buffer. + // Rust reinterprets the bytes as f32 on the other side. + invokeRawBinary( + "push_audio_pcm", + new Uint8Array(float32.buffer, float32.byteOffset, float32.byteLength), + ).catch(() => { + /* silently drop — Rust handles backpressure */ + }); + }; + + // Track the current mode so PTT events are only forwarded in PTT mode. + // In VAD mode, the worklet stays in transmitting=true regardless of + // Ctrl+Space presses — prevents accidental muting. (Crossfire fix I1.) + let currentMode: "push_to_talk" | "voice_activity" = initialTransmitting + ? "voice_activity" + : "push_to_talk"; + + // Listen for PTT state from Rust global shortcut (Ctrl+Space press/release). + // Direction: Rust→main→worklet. The Tauri event carries a boolean payload. + let pttUnlisten: UnlistenFn | null = null; + try { + pttUnlisten = await listen("ptt-state", (event) => { + // Only forward PTT events to the worklet when in PTT mode. + // In VAD mode, Ctrl+Space is ignored — the worklet stays open. + if (currentMode === "push_to_talk") { + workletNode.port.postMessage({ type: "ptt", active: event.payload }); + } + }); + } catch { + // PTT events not available — worklet stays in current transmit mode. + // This is fine for VAD mode (always transmitting) and degrades gracefully + // for PTT mode (user won't be able to transmit, but audio won't leak). + } + + return { + stop: () => { + workletNode.port.onmessage = null; + pttUnlisten?.(); + source.disconnect(); + workletNode.disconnect(); + void audioContext.close(); + }, + setTransmitting: (active: boolean) => { + workletNode.port.postMessage({ type: "ptt", active }); + }, + setMode: (mode: "push_to_talk" | "voice_activity") => { + currentMode = mode; + // When switching to VAD, immediately open the mic. + // When switching to PTT, immediately gate until key press. + workletNode.port.postMessage({ + type: "ptt", + active: mode === "voice_activity", + }); + }, + }; +} diff --git a/desktop/src/features/huddle/lib/livekit.ts b/desktop/src/features/huddle/lib/livekit.ts new file mode 100644 index 00000000000..37f10d1466e --- /dev/null +++ b/desktop/src/features/huddle/lib/livekit.ts @@ -0,0 +1,103 @@ +import { + LocalAudioTrack, + Room, + RoomEvent, + type Participant, +} from "livekit-client"; + +export interface HuddleConnection { + room: Room; + localAudioTrack: MediaStreamTrack; + disconnect: () => Promise; +} + +export type HuddleRoomCallbacks = { + onActiveSpeakersChanged?: (speakers: Participant[]) => void; + onDisconnected?: () => void; + onReconnecting?: () => void; + onReconnected?: () => void; +}; + +/** + * LiveKit connection lifecycle: + * + * connectToHuddle(url, token, callbacks?) + * → getUserMedia({ audio: true }) [mic permission] + * → room.connect(url, token) [WebRTC signaling] + * → register room event listeners [active speakers, disconnect, reconnect] + * → room.localParticipant.publishTrack(audioTrack) + * → returns { room, localAudioTrack, disconnect } + * + * disconnect() + * → room.removeAllListeners() + * → room.disconnect() + * → stream.getTracks().forEach(t => t.stop()) + * + * Error handling: mic stream is always cleaned up, even on partial failure. + */ +export async function connectToHuddle( + url: string, + token: string, + callbacks?: HuddleRoomCallbacks, +): Promise { + const room = new Room(); + let stream: MediaStream | null = null; + + try { + stream = await navigator.mediaDevices.getUserMedia({ + audio: { echoCancellation: true, noiseSuppression: true }, + }); + const audioTrack = stream.getAudioTracks()[0]; + + await room.connect(url, token); + + // Register room event listeners before publishing track + if (callbacks?.onActiveSpeakersChanged) { + room.on( + RoomEvent.ActiveSpeakersChanged, + callbacks.onActiveSpeakersChanged, + ); + } + if (callbacks?.onDisconnected) { + room.on(RoomEvent.Disconnected, callbacks.onDisconnected); + } + if (callbacks?.onReconnecting) { + room.on(RoomEvent.Reconnecting, callbacks.onReconnecting); + } + if (callbacks?.onReconnected) { + room.on(RoomEvent.Reconnected, callbacks.onReconnected); + } + + try { + // false = don't let LiveKit manage the track lifecycle + const localTrack = new LocalAudioTrack(audioTrack, undefined, false); + await room.localParticipant.publishTrack(localTrack); + } catch (publishErr) { + // Publish failed after connect — disconnect room before propagating + room.removeAllListeners(); + room.disconnect(); + throw publishErr; + } + + return { + room, + localAudioTrack: audioTrack, + disconnect: async () => { + try { + room.removeAllListeners(); + room.disconnect(); + } finally { + stream?.getTracks().forEach((t) => { + t.stop(); + }); + } + }, + }; + } catch (err) { + // Clean up mic stream on any failure (getUserMedia, connect, or publish) + stream?.getTracks().forEach((t) => { + t.stop(); + }); + throw err; + } +} diff --git a/desktop/src/shared/api/relayClientSession.ts b/desktop/src/shared/api/relayClientSession.ts index 6069ceea13e..721cce43283 100644 --- a/desktop/src/shared/api/relayClientSession.ts +++ b/desktop/src/shared/api/relayClientSession.ts @@ -166,6 +166,47 @@ export class RelayClient { return this.subscribe(this.buildChannelFilter(channelId, 50), onEvent); } + /** + * Subscribe to a channel starting from NOW — no history backfill. + * Used by huddle TTS where only live kind:9 messages should be spoken. + * The `since` filter ensures the relay never sends historical backlog. + * The high `limit` ensures reconnect replay can recover all missed events. + */ + async subscribeToChannelLive( + channelId: string, + onEvent: (event: RelayEvent) => void, + ) { + return this.subscribe( + { + kinds: [KIND_STREAM_MESSAGE], + "#h": [channelId], + limit: 1000, + since: Math.floor(Date.now() / 1_000), + }, + onEvent, + ); + } + + /** + * Subscribe to huddle lifecycle events (kinds 48100–48103) for a channel. + * Used by HuddleIndicator to detect active huddles without being drowned + * out by regular channel messages in the generic subscription window. + * Includes both historical (last 10) and live events. + */ + async subscribeToHuddleEvents( + channelId: string, + onEvent: (event: RelayEvent) => void, + ) { + return this.subscribe( + { + kinds: [48100, 48101, 48102, 48103], + "#h": [channelId], + limit: 100, + }, + onEvent, + ); + } + async subscribeToTypingIndicators( channelId: string, onEvent: (event: RelayEvent) => void,