From 76b5403ff7859ee639fcc06f51772bf8b5f72206 Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Sat, 11 Apr 2026 12:14:58 -0400 Subject: [PATCH 01/41] =?UTF-8?q?feat(huddles):=20Phase=201=20=E2=80=94=20?= =?UTF-8?q?Voice=20Call=20Foundation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Humans can voice chat via LiveKit WebRTC. Ephemeral text channel is created for the huddle transcript. No STT/TTS yet (Phase 2+). Relay: - Wire HuddleService into AppState (optional, env-var gated) - POST /api/huddles/{channel_id}/token endpoint with auth + scope + channel access checks - Verify huddle kinds 48100-48105 stored/fanned-out (no changes needed) SDK: - 4 huddle lifecycle event builders (48100-48103) with tests Desktop Rust: - HuddleManager state machine (Idle → Creating → Active → Leaving) - 5 Tauri commands: start/join/leave/end_huddle, get_huddle_state - push_audio_pcm stub (Phase 1 placeholder for Phase 2 STT) - 4 huddle event builders in events.rs - Bulletproof rollback: all error paths reset to Idle, orphaned channels archived, HUDDLE_STARTED emitted only after token success - NSMicrophoneUsageDescription in Info.plist Desktop WebView: - LiveKit JS SDK integration (livekit-client 2.18.1) - AudioWorklet mic tap with 100ms PCM batching → Rust via InvokeBody::Raw - HuddleBar floating UI (mute, leave, participant list) - Resource cleanup on all failure paths Crossfired: 3 rounds, Opus + Codex, both APPROVE 10/10 on final round. Phase 0 spikes validated: AudioWorklet IPC, sherpa-onnx compilation, LiveKit JS in WKWebView — all green. --- Cargo.lock | 1 + crates/sprout-relay/Cargo.toml | 1 + crates/sprout-relay/src/api/huddles.rs | 72 ++++ crates/sprout-relay/src/api/mod.rs | 3 + crates/sprout-relay/src/main.rs | 22 + crates/sprout-relay/src/router.rs | 5 + crates/sprout-relay/src/state.rs | 8 + crates/sprout-sdk/src/builders.rs | 179 ++++++++ desktop/package.json | 1 + desktop/pnpm-lock.yaml | 240 +++++++---- desktop/public/worklet.js | 39 ++ desktop/src-tauri/Info.plist | 2 + desktop/src-tauri/src/app_state.rs | 3 + desktop/src-tauri/src/events.rs | 56 +++ desktop/src-tauri/src/huddle/mod.rs | 399 ++++++++++++++++++ desktop/src-tauri/src/lib.rs | 8 + .../features/huddle/components/HuddleBar.tsx | 126 ++++++ .../huddle/components/ParticipantList.tsx | 51 +++ desktop/src/features/huddle/index.ts | 6 + .../src/features/huddle/lib/audioWorklet.ts | 59 +++ desktop/src/features/huddle/lib/livekit.ts | 45 ++ 21 files changed, 1253 insertions(+), 73 deletions(-) create mode 100644 crates/sprout-relay/src/api/huddles.rs create mode 100644 desktop/public/worklet.js create mode 100644 desktop/src-tauri/src/huddle/mod.rs create mode 100644 desktop/src/features/huddle/components/HuddleBar.tsx create mode 100644 desktop/src/features/huddle/components/ParticipantList.tsx create mode 100644 desktop/src/features/huddle/index.ts create mode 100644 desktop/src/features/huddle/lib/audioWorklet.ts create mode 100644 desktop/src/features/huddle/lib/livekit.ts diff --git a/Cargo.lock b/Cargo.lock index 25bb742f16..d50f283fa9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3891,6 +3891,7 @@ dependencies = [ "sprout-auth", "sprout-core", "sprout-db", + "sprout-huddle", "sprout-media", "sprout-pubsub", "sprout-search", diff --git a/crates/sprout-relay/Cargo.toml b/crates/sprout-relay/Cargo.toml index ab56a601c3..ba5433e2d1 100644 --- a/crates/sprout-relay/Cargo.toml +++ b/crates/sprout-relay/Cargo.toml @@ -38,6 +38,7 @@ deadpool-redis = { workspace = true } redis = { workspace = true } sqlx = { workspace = true } base64 = "0.22" +sprout-huddle = { workspace = true } sprout-workflow = { workspace = true, features = ["reqwest"] } sprout-media = { workspace = true } bytes = "1" diff --git a/crates/sprout-relay/src/api/huddles.rs b/crates/sprout-relay/src/api/huddles.rs new file mode 100644 index 0000000000..c0bce5331d --- /dev/null +++ b/crates/sprout-relay/src/api/huddles.rs @@ -0,0 +1,72 @@ +//! LiveKit huddle token endpoint. +//! +//! ## Routes +//! - `POST /api/huddles/{channel_id}/token` — generate a LiveKit access token +//! for the authenticated user to join the channel's huddle room. + +use std::sync::Arc; + +use axum::{ + extract::{Path, State}, + http::{HeaderMap, StatusCode}, + response::Json, +}; +use uuid::Uuid; + +use super::{ + check_channel_membership, check_token_channel_access, extract_auth_context, internal_error, + scope_error, +}; +use crate::state::AppState; + +/// `POST /api/huddles/{channel_id}/token` — generate a LiveKit access token. +/// +/// Returns `{ "token": "", "url": "", "room": "sprout-" }`. +/// Returns 501 if LiveKit is not configured on this relay. +/// Returns 403 if the caller is not a member of the channel. +pub async fn huddle_token( + State(state): State>, + headers: HeaderMap, + Path(channel_id): Path, +) -> Result, (StatusCode, Json)> { + let ctx = extract_auth_context(&headers, &state).await?; + sprout_auth::require_scope(&ctx.scopes, sprout_auth::Scope::MessagesRead) + .map_err(scope_error)?; + check_token_channel_access(&ctx, &channel_id)?; + + // Require LiveKit to be configured. + let huddle_service = state.huddle_service.as_ref().ok_or_else(|| { + ( + StatusCode::NOT_IMPLEMENTED, + Json(serde_json::json!({ + "error": "huddles_not_configured", + "message": "LiveKit is not configured on this relay" + })), + ) + })?; + + // Verify the caller is a member of the channel (or it's an open channel). + check_channel_membership(&state, channel_id, &ctx.pubkey_bytes).await?; + + // Generate the LiveKit token. + let room = sprout_huddle::HuddleService::create_room_name(channel_id); + let identity = ctx.pubkey.to_hex(); + let lk_token = huddle_service + .generate_token(&room, &identity, &identity) + .map_err(|e| internal_error(&format!("token generation failed: {e}")))?; + + let livekit_url = state + .livekit_url + .as_deref() + .unwrap_or_default(); + + if livekit_url.is_empty() { + return Err(internal_error("livekit_url is not configured")); + } + + Ok(Json(serde_json::json!({ + "token": lk_token.token, + "url": livekit_url, + "room": room, + }))) +} diff --git a/crates/sprout-relay/src/api/mod.rs b/crates/sprout-relay/src/api/mod.rs index 8cddece093..e0ca9fd4e4 100644 --- a/crates/sprout-relay/src/api/mod.rs +++ b/crates/sprout-relay/src/api/mod.rs @@ -12,6 +12,8 @@ /// Agent directory and status endpoints. pub mod agents; +/// LiveKit huddle token endpoint. +pub mod huddles; /// Workflow approval grant/deny endpoints. pub mod approvals; /// Canvas (shared document) endpoints. @@ -51,6 +53,7 @@ pub mod workflows; // Re-export all public handlers so router.rs can use `api::*_handler` unchanged. pub use agents::agents_handler; +pub use huddles::huddle_token; pub use approvals::{deny_approval, deny_approval_by_hash, grant_approval, grant_approval_by_hash}; pub use canvas::get_canvas; pub use channels::channels_handler; diff --git a/crates/sprout-relay/src/main.rs b/crates/sprout-relay/src/main.rs index 3ced6651bc..28f939e678 100644 --- a/crates/sprout-relay/src/main.rs +++ b/crates/sprout-relay/src/main.rs @@ -10,6 +10,7 @@ use sprout_db::{Db, DbConfig}; use sprout_pubsub::PubSubManager; use sprout_search::{SearchConfig, SearchService}; +use sprout_huddle::{HuddleConfig, HuddleService}; use sprout_relay::config::Config; use sprout_relay::metrics as relay_metrics; use sprout_relay::router::{build_health_router, build_router}; @@ -127,6 +128,26 @@ async fn main() -> anyhow::Result<()> { .map_err(|e| anyhow::anyhow!("failed to initialize media storage: {e}"))?; info!("Media storage connected"); + let huddle_service = match ( + std::env::var("LIVEKIT_URL"), + std::env::var("LIVEKIT_API_KEY"), + std::env::var("LIVEKIT_API_SECRET"), + ) { + (Ok(url), Ok(key), Ok(secret)) => { + info!("LiveKit configured — huddles enabled"); + let svc = HuddleService::new(HuddleConfig { + livekit_url: url.clone(), + livekit_api_key: key, + livekit_api_secret: secret, + }); + Some((svc, url)) + } + _ => { + info!("LiveKit not configured — huddles disabled"); + None + } + }; + let state = Arc::new(AppState::new( config.clone(), db, @@ -138,6 +159,7 @@ async fn main() -> anyhow::Result<()> { Arc::clone(&workflow_engine), relay_keypair, media_storage, + huddle_service, )); // Wire the action sink — must happen after AppState (which creates diff --git a/crates/sprout-relay/src/router.rs b/crates/sprout-relay/src/router.rs index e449b66d5b..2af83e217b 100644 --- a/crates/sprout-relay/src/router.rs +++ b/crates/sprout-relay/src/router.rs @@ -104,6 +104,11 @@ pub fn build_router(state: Arc) -> Router { "/api/approvals/by-hash/{hash}/deny", post(api::deny_approval_by_hash), ) + // Huddle routes + .route( + "/api/huddles/{channel_id}/token", + post(api::huddle_token), + ) // Membership routes .route("/api/channels/{channel_id}/members", get(api::list_members)) // Channel detail + metadata routes diff --git a/crates/sprout-relay/src/state.rs b/crates/sprout-relay/src/state.rs index 6db0731329..1c19168e59 100644 --- a/crates/sprout-relay/src/state.rs +++ b/crates/sprout-relay/src/state.rs @@ -15,6 +15,7 @@ use sprout_audit::AuditService; use sprout_auth::AuthService; use sprout_core::event::StoredEvent; use sprout_db::Db; +use sprout_huddle::HuddleService; use sprout_media::MediaStorage; use sprout_pubsub::PubSubManager; use sprout_search::SearchService; @@ -202,6 +203,10 @@ pub struct AppState { pub search_index_tx: mpsc::Sender, /// Media storage client (S3/MinIO). pub media_storage: Arc, + /// LiveKit huddle service — `None` when LiveKit is not configured. + pub huddle_service: Option>, + /// LiveKit server URL — populated when `huddle_service` is `Some`. + pub livekit_url: Option, /// Set to `true` on SIGTERM — readiness probe returns 503. pub shutting_down: Arc, /// Process start time — used by `/_status` endpoint. @@ -222,6 +227,7 @@ impl AppState { workflow_engine: Arc, relay_keypair: nostr::Keys, media_storage: MediaStorage, + huddle_service: Option<(HuddleService, String)>, ) -> Self { let max_connections = config.max_connections; let max_concurrent_handlers = config.max_concurrent_handlers; @@ -280,6 +286,8 @@ impl AppState { search_index_tx, media_storage: Arc::new(media_storage), + livekit_url: huddle_service.as_ref().map(|(_, url)| url.clone()), + huddle_service: huddle_service.map(|(svc, _)| Arc::new(svc)), shutting_down: Arc::new(AtomicBool::new(false)), started_at: Instant::now(), } diff --git a/crates/sprout-sdk/src/builders.rs b/crates/sprout-sdk/src/builders.rs index a000f6a9c0..707dc48c42 100644 --- a/crates/sprout-sdk/src/builders.rs +++ b/crates/sprout-sdk/src/builders.rs @@ -563,6 +563,96 @@ pub fn build_contact_list( Ok(EventBuilder::new(Kind::Custom(3), "", tags)) } +// ── Builder 26: build_huddle_started ───────────────────────────────────────── + +/// Build a huddle-started event (kind 48100). +/// +/// Posted to the parent channel as an advisory UI hint that a huddle has begun. +/// - `parent_channel_id`: the channel the huddle belongs to (h-tag) +/// - `ephemeral_channel_id`: the short-lived channel UUID for the huddle session +/// - `livekit_room`: LiveKit room name participants should join +pub fn build_huddle_started( + parent_channel_id: Uuid, + ephemeral_channel_id: Uuid, + livekit_room: &str, +) -> Result { + let tags = vec![tag(&["h", &parent_channel_id.to_string()])?]; + let mut map = serde_json::Map::new(); + map.insert( + "ephemeral_channel_id".into(), + serde_json::Value::String(ephemeral_channel_id.to_string()), + ); + map.insert( + "livekit_room".into(), + serde_json::Value::String(livekit_room.into()), + ); + let content = serde_json::Value::Object(map).to_string(); + Ok(EventBuilder::new(Kind::Custom(48100), content, tags)) +} + +// ── Builder 27: build_huddle_participant_joined ─────────────────────────────── + +/// Build a huddle-participant-joined event (kind 48101). +/// +/// Posted to the parent channel when a participant enters the huddle. +/// - `parent_channel_id`: the channel the huddle belongs to (h-tag) +/// - `ephemeral_channel_id`: the short-lived channel UUID for the huddle session +pub fn build_huddle_participant_joined( + parent_channel_id: Uuid, + ephemeral_channel_id: Uuid, +) -> Result { + let tags = vec![tag(&["h", &parent_channel_id.to_string()])?]; + let mut map = serde_json::Map::new(); + map.insert( + "ephemeral_channel_id".into(), + serde_json::Value::String(ephemeral_channel_id.to_string()), + ); + let content = serde_json::Value::Object(map).to_string(); + Ok(EventBuilder::new(Kind::Custom(48101), content, tags)) +} + +// ── Builder 28: build_huddle_participant_left ───────────────────────────────── + +/// Build a huddle-participant-left event (kind 48102). +/// +/// Posted to the parent channel when a participant exits the huddle. +/// - `parent_channel_id`: the channel the huddle belongs to (h-tag) +/// - `ephemeral_channel_id`: the short-lived channel UUID for the huddle session +pub fn build_huddle_participant_left( + parent_channel_id: Uuid, + ephemeral_channel_id: Uuid, +) -> Result { + let tags = vec![tag(&["h", &parent_channel_id.to_string()])?]; + let mut map = serde_json::Map::new(); + map.insert( + "ephemeral_channel_id".into(), + serde_json::Value::String(ephemeral_channel_id.to_string()), + ); + let content = serde_json::Value::Object(map).to_string(); + Ok(EventBuilder::new(Kind::Custom(48102), content, tags)) +} + +// ── Builder 29: build_huddle_ended ─────────────────────────────────────────── + +/// Build a huddle-ended event (kind 48103). +/// +/// Posted to the parent channel when the huddle session concludes. +/// - `parent_channel_id`: the channel the huddle belongs to (h-tag) +/// - `ephemeral_channel_id`: the short-lived channel UUID for the huddle session +pub fn build_huddle_ended( + parent_channel_id: Uuid, + ephemeral_channel_id: Uuid, +) -> Result { + let tags = vec![tag(&["h", &parent_channel_id.to_string()])?]; + let mut map = serde_json::Map::new(); + map.insert( + "ephemeral_channel_id".into(), + serde_json::Value::String(ephemeral_channel_id.to_string()), + ); + let content = serde_json::Value::Object(map).to_string(); + Ok(EventBuilder::new(Kind::Custom(48103), content, tags)) +} + // ── Helper: extract_channel_id ─────────────────────────────────────────────── /// Extract the channel UUID from an event's `h` tag. @@ -1393,4 +1483,93 @@ mod tests { let err = build_contact_list(&contacts).unwrap_err(); assert!(matches!(err, SdkError::InvalidInput(_))); } + + // ── build_huddle_started ────────────────────────────────────────────────── + + #[test] + fn huddle_started_happy_path() { + let parent = uuid(); + let ephemeral = uuid(); + let ev = sign(build_huddle_started(parent, ephemeral, "my-room").unwrap()); + assert_eq!(ev.kind.as_u16(), 48100); + assert!(has_tag(&ev, "h", &parent.to_string())); + let v: serde_json::Value = serde_json::from_str(&ev.content).unwrap(); + assert_eq!(v["ephemeral_channel_id"], ephemeral.to_string()); + assert_eq!(v["livekit_room"], "my-room"); + } + + #[test] + fn huddle_started_h_tag_is_parent_not_ephemeral() { + let parent = uuid(); + let ephemeral = uuid(); + let ev = sign(build_huddle_started(parent, ephemeral, "room").unwrap()); + assert!(has_tag(&ev, "h", &parent.to_string())); + assert!(!has_tag(&ev, "h", &ephemeral.to_string())); + } + + // ── build_huddle_participant_joined ─────────────────────────────────────── + + #[test] + fn huddle_participant_joined_happy_path() { + let parent = uuid(); + let ephemeral = uuid(); + let ev = sign(build_huddle_participant_joined(parent, ephemeral).unwrap()); + assert_eq!(ev.kind.as_u16(), 48101); + assert!(has_tag(&ev, "h", &parent.to_string())); + let v: serde_json::Value = serde_json::from_str(&ev.content).unwrap(); + assert_eq!(v["ephemeral_channel_id"], ephemeral.to_string()); + } + + #[test] + fn huddle_participant_joined_h_tag_is_parent_not_ephemeral() { + let parent = uuid(); + let ephemeral = uuid(); + let ev = sign(build_huddle_participant_joined(parent, ephemeral).unwrap()); + assert!(has_tag(&ev, "h", &parent.to_string())); + assert!(!has_tag(&ev, "h", &ephemeral.to_string())); + } + + // ── build_huddle_participant_left ───────────────────────────────────────── + + #[test] + fn huddle_participant_left_happy_path() { + let parent = uuid(); + let ephemeral = uuid(); + let ev = sign(build_huddle_participant_left(parent, ephemeral).unwrap()); + assert_eq!(ev.kind.as_u16(), 48102); + assert!(has_tag(&ev, "h", &parent.to_string())); + let v: serde_json::Value = serde_json::from_str(&ev.content).unwrap(); + assert_eq!(v["ephemeral_channel_id"], ephemeral.to_string()); + } + + #[test] + fn huddle_participant_left_h_tag_is_parent_not_ephemeral() { + let parent = uuid(); + let ephemeral = uuid(); + let ev = sign(build_huddle_participant_left(parent, ephemeral).unwrap()); + assert!(has_tag(&ev, "h", &parent.to_string())); + assert!(!has_tag(&ev, "h", &ephemeral.to_string())); + } + + // ── build_huddle_ended ──────────────────────────────────────────────────── + + #[test] + fn huddle_ended_happy_path() { + let parent = uuid(); + let ephemeral = uuid(); + let ev = sign(build_huddle_ended(parent, ephemeral).unwrap()); + assert_eq!(ev.kind.as_u16(), 48103); + assert!(has_tag(&ev, "h", &parent.to_string())); + let v: serde_json::Value = serde_json::from_str(&ev.content).unwrap(); + assert_eq!(v["ephemeral_channel_id"], ephemeral.to_string()); + } + + #[test] + fn huddle_ended_h_tag_is_parent_not_ephemeral() { + let parent = uuid(); + let ephemeral = uuid(); + let ev = sign(build_huddle_ended(parent, ephemeral).unwrap()); + assert!(has_tag(&ev, "h", &parent.to_string())); + assert!(!has_tag(&ev, "h", &ephemeral.to_string())); + } } diff --git a/desktop/package.json b/desktop/package.json index 25848ae1b2..4822b86601 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -46,6 +46,7 @@ "clsx": "^2.1.1", "emoji-mart": "^5.6.0", "jdenticon": "^3.3.0", + "livekit-client": "^2.18.1", "lucide-react": "^0.577.0", "react": "^19.1.0", "react-diff-view": "^3.3.2", diff --git a/desktop/pnpm-lock.yaml b/desktop/pnpm-lock.yaml index cdbbd4f230..95ded99cb5 100644 --- a/desktop/pnpm-lock.yaml +++ b/desktop/pnpm-lock.yaml @@ -74,6 +74,9 @@ importers: jdenticon: specifier: ^3.3.0 version: 3.3.0 + livekit-client: + specifier: ^2.18.1 + version: 2.18.1(@types/dom-mediacapture-record@1.0.22) lucide-react: specifier: ^0.577.0 version: 0.577.0(react@19.2.4) @@ -261,57 +264,60 @@ packages: hasBin: true '@biomejs/cli-darwin-arm64@2.4.6': - resolution: {integrity: sha512-NW18GSyxr+8sJIqgoGwVp5Zqm4SALH4b4gftIA0n62PTuBs6G2tHlwNAOj0Vq0KKSs7Sf88VjjmHh0O36EnzrQ==} + resolution: {integrity: sha512-NW18GSyxr+8sJIqgoGwVp5Zqm4SALH4b4gftIA0n62PTuBs6G2tHlwNAOj0Vq0KKSs7Sf88VjjmHh0O36EnzrQ==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-2.4.6.tgz} engines: {node: '>=14.21.3'} cpu: [arm64] os: [darwin] '@biomejs/cli-darwin-x64@2.4.6': - resolution: {integrity: sha512-4uiE/9tuI7cnjtY9b07RgS7gGyYOAfIAGeVJWEfeCnAarOAS7qVmuRyX6d7JTKw28/mt+rUzMasYeZ+0R/U1Mw==} + resolution: {integrity: sha512-4uiE/9tuI7cnjtY9b07RgS7gGyYOAfIAGeVJWEfeCnAarOAS7qVmuRyX6d7JTKw28/mt+rUzMasYeZ+0R/U1Mw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@biomejs/cli-darwin-x64/-/cli-darwin-x64-2.4.6.tgz} engines: {node: '>=14.21.3'} cpu: [x64] os: [darwin] '@biomejs/cli-linux-arm64-musl@2.4.6': - resolution: {integrity: sha512-F/JdB7eN22txiTqHM5KhIVt0jVkzZwVYrdTR1O3Y4auBOQcXxHK4dxULf4z43QyZI5tsnQJrRBHZy7wwtL+B3A==} + resolution: {integrity: sha512-F/JdB7eN22txiTqHM5KhIVt0jVkzZwVYrdTR1O3Y4auBOQcXxHK4dxULf4z43QyZI5tsnQJrRBHZy7wwtL+B3A==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.4.6.tgz} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] libc: [musl] '@biomejs/cli-linux-arm64@2.4.6': - resolution: {integrity: sha512-kMLaI7OF5GN1Q8Doymjro1P8rVEoy7BKQALNz6fiR8IC1WKduoNyteBtJlHT7ASIL0Cx2jR6VUOBIbcB1B8pew==} + resolution: {integrity: sha512-kMLaI7OF5GN1Q8Doymjro1P8rVEoy7BKQALNz6fiR8IC1WKduoNyteBtJlHT7ASIL0Cx2jR6VUOBIbcB1B8pew==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@biomejs/cli-linux-arm64/-/cli-linux-arm64-2.4.6.tgz} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] libc: [glibc] '@biomejs/cli-linux-x64-musl@2.4.6': - resolution: {integrity: sha512-C9s98IPDu7DYarjlZNuzJKTjVHN03RUnmHV5htvqsx6vEUXCDSJ59DNwjKVD5XYoSS4N+BYhq3RTBAL8X6svEg==} + resolution: {integrity: sha512-C9s98IPDu7DYarjlZNuzJKTjVHN03RUnmHV5htvqsx6vEUXCDSJ59DNwjKVD5XYoSS4N+BYhq3RTBAL8X6svEg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-2.4.6.tgz} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] libc: [musl] '@biomejs/cli-linux-x64@2.4.6': - resolution: {integrity: sha512-oHXmUFEoH8Lql1xfc3QkFLiC1hGR7qedv5eKNlC185or+o4/4HiaU7vYODAH3peRCfsuLr1g6v2fK9dFFOYdyw==} + resolution: {integrity: sha512-oHXmUFEoH8Lql1xfc3QkFLiC1hGR7qedv5eKNlC185or+o4/4HiaU7vYODAH3peRCfsuLr1g6v2fK9dFFOYdyw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@biomejs/cli-linux-x64/-/cli-linux-x64-2.4.6.tgz} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] libc: [glibc] '@biomejs/cli-win32-arm64@2.4.6': - resolution: {integrity: sha512-xzThn87Pf3YrOGTEODFGONmqXpTwUNxovQb72iaUOdcw8sBSY3+3WD8Hm9IhMYLnPi0n32s3L3NWU6+eSjfqFg==} + resolution: {integrity: sha512-xzThn87Pf3YrOGTEODFGONmqXpTwUNxovQb72iaUOdcw8sBSY3+3WD8Hm9IhMYLnPi0n32s3L3NWU6+eSjfqFg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@biomejs/cli-win32-arm64/-/cli-win32-arm64-2.4.6.tgz} engines: {node: '>=14.21.3'} cpu: [arm64] os: [win32] '@biomejs/cli-win32-x64@2.4.6': - resolution: {integrity: sha512-7++XhnsPlr1HDbor5amovPjOH6vsrFOCdp93iKXhFn6bcMUI6soodj3WWKfgEO6JosKU1W5n3uky3WW9RlRjTg==} + resolution: {integrity: sha512-7++XhnsPlr1HDbor5amovPjOH6vsrFOCdp93iKXhFn6bcMUI6soodj3WWKfgEO6JosKU1W5n3uky3WW9RlRjTg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@biomejs/cli-win32-x64/-/cli-win32-x64-2.4.6.tgz} engines: {node: '>=14.21.3'} cpu: [x64] os: [win32] + '@bufbuild/protobuf@1.10.1': + resolution: {integrity: sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@bufbuild/protobuf/-/protobuf-1.10.1.tgz} + '@emoji-mart/data@1.2.1': resolution: {integrity: sha512-no2pQMWiBy6gpBEiqGeU77/bFejDqUTRY7KX+0+iur13op3bqUsXdnwoZs6Xb1zbv0gAj5VvS1PWoUUckSr5Dw==} @@ -322,157 +328,157 @@ packages: react: ^16.8 || ^17 || ^18 '@esbuild/aix-ppc64@0.27.3': - resolution: {integrity: sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==} + resolution: {integrity: sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz} engines: {node: '>=18'} cpu: [ppc64] os: [aix] '@esbuild/android-arm64@0.27.3': - resolution: {integrity: sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==} + resolution: {integrity: sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/android-arm64/-/android-arm64-0.27.3.tgz} engines: {node: '>=18'} cpu: [arm64] os: [android] '@esbuild/android-arm@0.27.3': - resolution: {integrity: sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==} + resolution: {integrity: sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/android-arm/-/android-arm-0.27.3.tgz} engines: {node: '>=18'} cpu: [arm] os: [android] '@esbuild/android-x64@0.27.3': - resolution: {integrity: sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==} + resolution: {integrity: sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/android-x64/-/android-x64-0.27.3.tgz} engines: {node: '>=18'} cpu: [x64] os: [android] '@esbuild/darwin-arm64@0.27.3': - resolution: {integrity: sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==} + resolution: {integrity: sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/darwin-arm64/-/darwin-arm64-0.27.3.tgz} engines: {node: '>=18'} cpu: [arm64] os: [darwin] '@esbuild/darwin-x64@0.27.3': - resolution: {integrity: sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==} + resolution: {integrity: sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/darwin-x64/-/darwin-x64-0.27.3.tgz} engines: {node: '>=18'} cpu: [x64] os: [darwin] '@esbuild/freebsd-arm64@0.27.3': - resolution: {integrity: sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==} + resolution: {integrity: sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.3.tgz} engines: {node: '>=18'} cpu: [arm64] os: [freebsd] '@esbuild/freebsd-x64@0.27.3': - resolution: {integrity: sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==} + resolution: {integrity: sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/freebsd-x64/-/freebsd-x64-0.27.3.tgz} engines: {node: '>=18'} cpu: [x64] os: [freebsd] '@esbuild/linux-arm64@0.27.3': - resolution: {integrity: sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==} + resolution: {integrity: sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/linux-arm64/-/linux-arm64-0.27.3.tgz} engines: {node: '>=18'} cpu: [arm64] os: [linux] '@esbuild/linux-arm@0.27.3': - resolution: {integrity: sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==} + resolution: {integrity: sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/linux-arm/-/linux-arm-0.27.3.tgz} engines: {node: '>=18'} cpu: [arm] os: [linux] '@esbuild/linux-ia32@0.27.3': - resolution: {integrity: sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==} + resolution: {integrity: sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/linux-ia32/-/linux-ia32-0.27.3.tgz} engines: {node: '>=18'} cpu: [ia32] os: [linux] '@esbuild/linux-loong64@0.27.3': - resolution: {integrity: sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==} + resolution: {integrity: sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/linux-loong64/-/linux-loong64-0.27.3.tgz} engines: {node: '>=18'} cpu: [loong64] os: [linux] '@esbuild/linux-mips64el@0.27.3': - resolution: {integrity: sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==} + resolution: {integrity: sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/linux-mips64el/-/linux-mips64el-0.27.3.tgz} engines: {node: '>=18'} cpu: [mips64el] os: [linux] '@esbuild/linux-ppc64@0.27.3': - resolution: {integrity: sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==} + resolution: {integrity: sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/linux-ppc64/-/linux-ppc64-0.27.3.tgz} engines: {node: '>=18'} cpu: [ppc64] os: [linux] '@esbuild/linux-riscv64@0.27.3': - resolution: {integrity: sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==} + resolution: {integrity: sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/linux-riscv64/-/linux-riscv64-0.27.3.tgz} engines: {node: '>=18'} cpu: [riscv64] os: [linux] '@esbuild/linux-s390x@0.27.3': - resolution: {integrity: sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==} + resolution: {integrity: sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/linux-s390x/-/linux-s390x-0.27.3.tgz} engines: {node: '>=18'} cpu: [s390x] os: [linux] '@esbuild/linux-x64@0.27.3': - resolution: {integrity: sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==} + resolution: {integrity: sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/linux-x64/-/linux-x64-0.27.3.tgz} engines: {node: '>=18'} cpu: [x64] os: [linux] '@esbuild/netbsd-arm64@0.27.3': - resolution: {integrity: sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==} + resolution: {integrity: sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.3.tgz} engines: {node: '>=18'} cpu: [arm64] os: [netbsd] '@esbuild/netbsd-x64@0.27.3': - resolution: {integrity: sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==} + resolution: {integrity: sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/netbsd-x64/-/netbsd-x64-0.27.3.tgz} engines: {node: '>=18'} cpu: [x64] os: [netbsd] '@esbuild/openbsd-arm64@0.27.3': - resolution: {integrity: sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==} + resolution: {integrity: sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.3.tgz} engines: {node: '>=18'} cpu: [arm64] os: [openbsd] '@esbuild/openbsd-x64@0.27.3': - resolution: {integrity: sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==} + resolution: {integrity: sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/openbsd-x64/-/openbsd-x64-0.27.3.tgz} engines: {node: '>=18'} cpu: [x64] os: [openbsd] '@esbuild/openharmony-arm64@0.27.3': - resolution: {integrity: sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==} + resolution: {integrity: sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.3.tgz} engines: {node: '>=18'} cpu: [arm64] os: [openharmony] '@esbuild/sunos-x64@0.27.3': - resolution: {integrity: sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==} + resolution: {integrity: sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/sunos-x64/-/sunos-x64-0.27.3.tgz} engines: {node: '>=18'} cpu: [x64] os: [sunos] '@esbuild/win32-arm64@0.27.3': - resolution: {integrity: sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==} + resolution: {integrity: sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/win32-arm64/-/win32-arm64-0.27.3.tgz} engines: {node: '>=18'} cpu: [arm64] os: [win32] '@esbuild/win32-ia32@0.27.3': - resolution: {integrity: sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==} + resolution: {integrity: sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/win32-ia32/-/win32-ia32-0.27.3.tgz} engines: {node: '>=18'} cpu: [ia32] os: [win32] '@esbuild/win32-x64@0.27.3': - resolution: {integrity: sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==} + resolution: {integrity: sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/win32-x64/-/win32-x64-0.27.3.tgz} engines: {node: '>=18'} cpu: [x64] os: [win32] @@ -508,6 +514,12 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@livekit/mutex@1.1.1': + resolution: {integrity: sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@livekit/mutex/-/mutex-1.1.1.tgz} + + '@livekit/protocol@1.44.0': + resolution: {integrity: sha512-/vfhDUGcUKO8Q43r6i+5FrDhl5oZjm/X3U4x2Iciqvgn5C8qbj+57YPcWSJ1kyIZm5Cm6AV2nAPjMm3ETD/iyg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@livekit/protocol/-/protocol-1.44.0.tgz} + '@noble/ciphers@2.1.1': resolution: {integrity: sha512-bysYuiVfhxNJuldNXlFEitTVdNnYUc+XNJZd7Qm2a5j1vZHgY+fazadNFWFaMK/2vye0JVlxV3gHmC0WDfAOQw==} engines: {node: '>= 20.19.0'} @@ -942,140 +954,140 @@ packages: resolution: {integrity: sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==} '@rollup/rollup-android-arm-eabi@4.59.0': - resolution: {integrity: sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg==} + resolution: {integrity: sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.59.0.tgz} cpu: [arm] os: [android] '@rollup/rollup-android-arm64@4.59.0': - resolution: {integrity: sha512-hZ+Zxj3SySm4A/DylsDKZAeVg0mvi++0PYVceVyX7hemkw7OreKdCvW2oQ3T1FMZvCaQXqOTHb8qmBShoqk69Q==} + resolution: {integrity: sha512-hZ+Zxj3SySm4A/DylsDKZAeVg0mvi++0PYVceVyX7hemkw7OreKdCvW2oQ3T1FMZvCaQXqOTHb8qmBShoqk69Q==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.59.0.tgz} cpu: [arm64] os: [android] '@rollup/rollup-darwin-arm64@4.59.0': - resolution: {integrity: sha512-W2Psnbh1J8ZJw0xKAd8zdNgF9HRLkdWwwdWqubSVk0pUuQkoHnv7rx4GiF9rT4t5DIZGAsConRE3AxCdJ4m8rg==} + resolution: {integrity: sha512-W2Psnbh1J8ZJw0xKAd8zdNgF9HRLkdWwwdWqubSVk0pUuQkoHnv7rx4GiF9rT4t5DIZGAsConRE3AxCdJ4m8rg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.59.0.tgz} cpu: [arm64] os: [darwin] '@rollup/rollup-darwin-x64@4.59.0': - resolution: {integrity: sha512-ZW2KkwlS4lwTv7ZVsYDiARfFCnSGhzYPdiOU4IM2fDbL+QGlyAbjgSFuqNRbSthybLbIJ915UtZBtmuLrQAT/w==} + resolution: {integrity: sha512-ZW2KkwlS4lwTv7ZVsYDiARfFCnSGhzYPdiOU4IM2fDbL+QGlyAbjgSFuqNRbSthybLbIJ915UtZBtmuLrQAT/w==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.59.0.tgz} cpu: [x64] os: [darwin] '@rollup/rollup-freebsd-arm64@4.59.0': - resolution: {integrity: sha512-EsKaJ5ytAu9jI3lonzn3BgG8iRBjV4LxZexygcQbpiU0wU0ATxhNVEpXKfUa0pS05gTcSDMKpn3Sx+QB9RlTTA==} + resolution: {integrity: sha512-EsKaJ5ytAu9jI3lonzn3BgG8iRBjV4LxZexygcQbpiU0wU0ATxhNVEpXKfUa0pS05gTcSDMKpn3Sx+QB9RlTTA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.59.0.tgz} cpu: [arm64] os: [freebsd] '@rollup/rollup-freebsd-x64@4.59.0': - resolution: {integrity: sha512-d3DuZi2KzTMjImrxoHIAODUZYoUUMsuUiY4SRRcJy6NJoZ6iIqWnJu9IScV9jXysyGMVuW+KNzZvBLOcpdl3Vg==} + resolution: {integrity: sha512-d3DuZi2KzTMjImrxoHIAODUZYoUUMsuUiY4SRRcJy6NJoZ6iIqWnJu9IScV9jXysyGMVuW+KNzZvBLOcpdl3Vg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.59.0.tgz} cpu: [x64] os: [freebsd] '@rollup/rollup-linux-arm-gnueabihf@4.59.0': - resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==} + resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.59.0.tgz} cpu: [arm] os: [linux] libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.59.0': - resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==} + resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.59.0.tgz} cpu: [arm] os: [linux] libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.59.0': - resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==} + resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.59.0.tgz} cpu: [arm64] os: [linux] libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.59.0': - resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==} + resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.59.0.tgz} cpu: [arm64] os: [linux] libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.59.0': - resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==} + resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.59.0.tgz} cpu: [loong64] os: [linux] libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.59.0': - resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==} + resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.59.0.tgz} cpu: [loong64] os: [linux] libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.59.0': - resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==} + resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.59.0.tgz} cpu: [ppc64] os: [linux] libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.59.0': - resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==} + resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.59.0.tgz} cpu: [ppc64] os: [linux] libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.59.0': - resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==} + resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.59.0.tgz} cpu: [riscv64] os: [linux] libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.59.0': - resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==} + resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.59.0.tgz} cpu: [riscv64] os: [linux] libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.59.0': - resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==} + resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.59.0.tgz} cpu: [s390x] os: [linux] libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.59.0': - resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==} + resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.59.0.tgz} cpu: [x64] os: [linux] libc: [glibc] '@rollup/rollup-linux-x64-musl@4.59.0': - resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==} + resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.59.0.tgz} cpu: [x64] os: [linux] libc: [musl] '@rollup/rollup-openbsd-x64@4.59.0': - resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==} + resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.59.0.tgz} cpu: [x64] os: [openbsd] '@rollup/rollup-openharmony-arm64@4.59.0': - resolution: {integrity: sha512-tt9KBJqaqp5i5HUZzoafHZX8b5Q2Fe7UjYERADll83O4fGqJ49O1FsL6LpdzVFQcpwvnyd0i+K/VSwu/o/nWlA==} + resolution: {integrity: sha512-tt9KBJqaqp5i5HUZzoafHZX8b5Q2Fe7UjYERADll83O4fGqJ49O1FsL6LpdzVFQcpwvnyd0i+K/VSwu/o/nWlA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.59.0.tgz} cpu: [arm64] os: [openharmony] '@rollup/rollup-win32-arm64-msvc@4.59.0': - resolution: {integrity: sha512-V5B6mG7OrGTwnxaNUzZTDTjDS7F75PO1ae6MJYdiMu60sq0CqN5CVeVsbhPxalupvTX8gXVSU9gq+Rx1/hvu6A==} + resolution: {integrity: sha512-V5B6mG7OrGTwnxaNUzZTDTjDS7F75PO1ae6MJYdiMu60sq0CqN5CVeVsbhPxalupvTX8gXVSU9gq+Rx1/hvu6A==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.59.0.tgz} cpu: [arm64] os: [win32] '@rollup/rollup-win32-ia32-msvc@4.59.0': - resolution: {integrity: sha512-UKFMHPuM9R0iBegwzKF4y0C4J9u8C6MEJgFuXTBerMk7EJ92GFVFYBfOZaSGLu6COf7FxpQNqhNS4c4icUPqxA==} + resolution: {integrity: sha512-UKFMHPuM9R0iBegwzKF4y0C4J9u8C6MEJgFuXTBerMk7EJ92GFVFYBfOZaSGLu6COf7FxpQNqhNS4c4icUPqxA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.59.0.tgz} cpu: [ia32] os: [win32] '@rollup/rollup-win32-x64-gnu@4.59.0': - resolution: {integrity: sha512-laBkYlSS1n2L8fSo1thDNGrCTQMmxjYY5G0WFWjFFYZkKPjsMBsgJfGf4TLxXrF6RyhI60L8TMOjBMvXiTcxeA==} + resolution: {integrity: sha512-laBkYlSS1n2L8fSo1thDNGrCTQMmxjYY5G0WFWjFFYZkKPjsMBsgJfGf4TLxXrF6RyhI60L8TMOjBMvXiTcxeA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.59.0.tgz} cpu: [x64] os: [win32] '@rollup/rollup-win32-x64-msvc@4.59.0': - resolution: {integrity: sha512-2HRCml6OztYXyJXAvdDXPKcawukWY2GpR5/nxKp4iBgiO3wcoEGkAaqctIbZcNB6KlUQBIqt8VYkNSj2397EfA==} + resolution: {integrity: sha512-2HRCml6OztYXyJXAvdDXPKcawukWY2GpR5/nxKp4iBgiO3wcoEGkAaqctIbZcNB6KlUQBIqt8VYkNSj2397EfA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.59.0.tgz} cpu: [x64] os: [win32] @@ -1191,72 +1203,72 @@ packages: resolution: {integrity: sha512-hKL/jWf293UDSUN09rR69hrToyIXBb8CjGaWC7gfinvnQrBVvnLr08FeFi38gxtugAVyVcTa5/FD/Xnkb1siBw==} '@tauri-apps/cli-darwin-arm64@2.10.1': - resolution: {integrity: sha512-Z2OjCXiZ+fbYZy7PmP3WRnOpM9+Fy+oonKDEmUE6MwN4IGaYqgceTjwHucc/kEEYZos5GICve35f7ZiizgqEnQ==} + resolution: {integrity: sha512-Z2OjCXiZ+fbYZy7PmP3WRnOpM9+Fy+oonKDEmUE6MwN4IGaYqgceTjwHucc/kEEYZos5GICve35f7ZiizgqEnQ==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-darwin-arm64/-/cli-darwin-arm64-2.10.1.tgz} engines: {node: '>= 10'} cpu: [arm64] os: [darwin] '@tauri-apps/cli-darwin-x64@2.10.1': - resolution: {integrity: sha512-V/irQVvjPMGOTQqNj55PnQPVuH4VJP8vZCN7ajnj+ZS8Kom1tEM2hR3qbbIRoS3dBKs5mbG8yg1WC+97dq17Pw==} + resolution: {integrity: sha512-V/irQVvjPMGOTQqNj55PnQPVuH4VJP8vZCN7ajnj+ZS8Kom1tEM2hR3qbbIRoS3dBKs5mbG8yg1WC+97dq17Pw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-darwin-x64/-/cli-darwin-x64-2.10.1.tgz} engines: {node: '>= 10'} cpu: [x64] os: [darwin] '@tauri-apps/cli-linux-arm-gnueabihf@2.10.1': - resolution: {integrity: sha512-Hyzwsb4VnCWKGfTw+wSt15Z2pLw2f0JdFBfq2vHBOBhvg7oi6uhKiF87hmbXOBXUZaGkyRDkCHsdzJcIfoJC2w==} + resolution: {integrity: sha512-Hyzwsb4VnCWKGfTw+wSt15Z2pLw2f0JdFBfq2vHBOBhvg7oi6uhKiF87hmbXOBXUZaGkyRDkCHsdzJcIfoJC2w==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-linux-arm-gnueabihf/-/cli-linux-arm-gnueabihf-2.10.1.tgz} engines: {node: '>= 10'} cpu: [arm] os: [linux] '@tauri-apps/cli-linux-arm64-gnu@2.10.1': - resolution: {integrity: sha512-OyOYs2t5GkBIvyWjA1+h4CZxTcdz1OZPCWAPz5DYEfB0cnWHERTnQ/SLayQzncrT0kwRoSfSz9KxenkyJoTelA==} + resolution: {integrity: sha512-OyOYs2t5GkBIvyWjA1+h4CZxTcdz1OZPCWAPz5DYEfB0cnWHERTnQ/SLayQzncrT0kwRoSfSz9KxenkyJoTelA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-linux-arm64-gnu/-/cli-linux-arm64-gnu-2.10.1.tgz} engines: {node: '>= 10'} cpu: [arm64] os: [linux] libc: [glibc] '@tauri-apps/cli-linux-arm64-musl@2.10.1': - resolution: {integrity: sha512-MIj78PDDGjkg3NqGptDOGgfXks7SYJwhiMh8SBoZS+vfdz7yP5jN18bNaLnDhsVIPARcAhE1TlsZe/8Yxo2zqg==} + resolution: {integrity: sha512-MIj78PDDGjkg3NqGptDOGgfXks7SYJwhiMh8SBoZS+vfdz7yP5jN18bNaLnDhsVIPARcAhE1TlsZe/8Yxo2zqg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.10.1.tgz} engines: {node: '>= 10'} cpu: [arm64] os: [linux] libc: [musl] '@tauri-apps/cli-linux-riscv64-gnu@2.10.1': - resolution: {integrity: sha512-X0lvOVUg8PCVaoEtEAnpxmnkwlE1gcMDTqfhbefICKDnOTJ5Est3qL0SrWxizDackIOKBcvtpejrSiVpuJI1kw==} + resolution: {integrity: sha512-X0lvOVUg8PCVaoEtEAnpxmnkwlE1gcMDTqfhbefICKDnOTJ5Est3qL0SrWxizDackIOKBcvtpejrSiVpuJI1kw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-linux-riscv64-gnu/-/cli-linux-riscv64-gnu-2.10.1.tgz} engines: {node: '>= 10'} cpu: [riscv64] os: [linux] libc: [glibc] '@tauri-apps/cli-linux-x64-gnu@2.10.1': - resolution: {integrity: sha512-2/12bEzsJS9fAKybxgicCDFxYD1WEI9kO+tlDwX5znWG2GwMBaiWcmhGlZ8fi+DMe9CXlcVarMTYc0L3REIRxw==} + resolution: {integrity: sha512-2/12bEzsJS9fAKybxgicCDFxYD1WEI9kO+tlDwX5znWG2GwMBaiWcmhGlZ8fi+DMe9CXlcVarMTYc0L3REIRxw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-linux-x64-gnu/-/cli-linux-x64-gnu-2.10.1.tgz} engines: {node: '>= 10'} cpu: [x64] os: [linux] libc: [glibc] '@tauri-apps/cli-linux-x64-musl@2.10.1': - resolution: {integrity: sha512-Y8J0ZzswPz50UcGOFuXGEMrxbjwKSPgXftx5qnkuMs2rmwQB5ssvLb6tn54wDSYxe7S6vlLob9vt0VKuNOaCIQ==} + resolution: {integrity: sha512-Y8J0ZzswPz50UcGOFuXGEMrxbjwKSPgXftx5qnkuMs2rmwQB5ssvLb6tn54wDSYxe7S6vlLob9vt0VKuNOaCIQ==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-linux-x64-musl/-/cli-linux-x64-musl-2.10.1.tgz} engines: {node: '>= 10'} cpu: [x64] os: [linux] libc: [musl] '@tauri-apps/cli-win32-arm64-msvc@2.10.1': - resolution: {integrity: sha512-iSt5B86jHYAPJa/IlYw++SXtFPGnWtFJriHn7X0NFBVunF6zu9+/zOn8OgqIWSl8RgzhLGXQEEtGBdR4wzpVgg==} + resolution: {integrity: sha512-iSt5B86jHYAPJa/IlYw++SXtFPGnWtFJriHn7X0NFBVunF6zu9+/zOn8OgqIWSl8RgzhLGXQEEtGBdR4wzpVgg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-win32-arm64-msvc/-/cli-win32-arm64-msvc-2.10.1.tgz} engines: {node: '>= 10'} cpu: [arm64] os: [win32] '@tauri-apps/cli-win32-ia32-msvc@2.10.1': - resolution: {integrity: sha512-gXyxgEzsFegmnWywYU5pEBURkcFN/Oo45EAwvZrHMh+zUSEAvO5E8TXsgPADYm31d1u7OQU3O3HsYfVBf2moHw==} + resolution: {integrity: sha512-gXyxgEzsFegmnWywYU5pEBURkcFN/Oo45EAwvZrHMh+zUSEAvO5E8TXsgPADYm31d1u7OQU3O3HsYfVBf2moHw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-win32-ia32-msvc/-/cli-win32-ia32-msvc-2.10.1.tgz} engines: {node: '>= 10'} cpu: [ia32] os: [win32] '@tauri-apps/cli-win32-x64-msvc@2.10.1': - resolution: {integrity: sha512-6Cn7YpPFwzChy0ERz6djKEmUehWrYlM+xTaNzGPgZocw3BD7OfwfWHKVWxXzdjEW2KfKkHddfdxK1XXTYqBRLg==} + resolution: {integrity: sha512-6Cn7YpPFwzChy0ERz6djKEmUehWrYlM+xTaNzGPgZocw3BD7OfwfWHKVWxXzdjEW2KfKkHddfdxK1XXTYqBRLg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-win32-x64-msvc/-/cli-win32-x64-msvc-2.10.1.tgz} engines: {node: '>= 10'} cpu: [x64] os: [win32] @@ -1293,6 +1305,9 @@ packages: '@types/debug@4.1.12': resolution: {integrity: sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==} + '@types/dom-mediacapture-record@1.0.22': + resolution: {integrity: sha512-mUMZLK3NvwRLcAAT9qmcK+9p7tpU2FHdDsntR3YI4+GY88XrgG4XiE7u1Q2LAN2/FZOz/tdMDC3GQCR4T8nFuw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@types/dom-mediacapture-record/-/dom-mediacapture-record-1.0.22.tgz} + '@types/estree-jsx@1.0.5': resolution: {integrity: sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==} @@ -1514,6 +1529,10 @@ packages: estree-util-is-identifier-name@3.0.0: resolution: {integrity: sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==} + events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/events/-/events-3.3.0.tgz} + engines: {node: '>=0.8.x'} + extend@3.0.2: resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} @@ -1541,12 +1560,12 @@ packages: resolution: {integrity: sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ==} fsevents@2.3.2: - resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/fsevents/-/fsevents-2.3.2.tgz} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] fsevents@2.3.3: - resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/fsevents/-/fsevents-2.3.3.tgz} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] @@ -1646,6 +1665,9 @@ packages: resolution: {integrity: sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==} hasBin: true + jose@6.2.2: + resolution: {integrity: sha512-d7kPDd34KO/YnzaDOlikGpOurfF0ByC2sEV4cANCtdqLlTfBlw2p14O/5d/zv40gJPbIQxfES3nSx1/oYNyuZQ==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/jose/-/jose-6.2.2.tgz} + js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -1666,9 +1688,18 @@ packages: lines-and-columns@1.2.4: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} + livekit-client@2.18.1: + resolution: {integrity: sha512-nGjuEEV1mVN01EcAMwGIwG3J1gpBMqwn2V4R6W/8zz9Rah1CaAohIm6AMLG7BdctQpyeh34dfAOLfDodIsWyYA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/livekit-client/-/livekit-client-2.18.1.tgz} + peerDependencies: + '@types/dom-mediacapture-record': ^1 + lodash@4.17.23: resolution: {integrity: sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==} + loglevel@1.9.2: + resolution: {integrity: sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/loglevel/-/loglevel-1.9.2.tgz} + engines: {node: '>= 0.6.0'} + longest-streak@3.1.0: resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==} @@ -2075,9 +2106,19 @@ packages: run-parallel@1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} + rxjs@7.8.2: + resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/rxjs/-/rxjs-7.8.2.tgz} + scheduler@0.27.0: resolution: {integrity: sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==} + sdp-transform@2.15.0: + resolution: {integrity: sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/sdp-transform/-/sdp-transform-2.15.0.tgz} + hasBin: true + + sdp@3.2.2: + resolution: {integrity: sha512-xZocWwfyp4hkbN4hLWxMjmv2Q8aNa9MhmOZ7L9aCZPT+dZsgRr6wZRrSYE3HTdyk/2pZKPSgqI7ns7Een1xMSA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/sdp/-/sdp-3.2.2.tgz} + semver@6.3.1: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true @@ -2173,13 +2214,16 @@ packages: resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} tslib@2.8.1: - resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/tslib/-/tslib-2.8.1.tgz} tsx@4.21.0: resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==} engines: {node: '>=18.0.0'} hasBin: true + typed-emitter@2.1.0: + resolution: {integrity: sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/typed-emitter/-/typed-emitter-2.1.0.tgz} + typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} @@ -2296,6 +2340,10 @@ packages: webpack-virtual-modules@0.6.2: resolution: {integrity: sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ==} + webrtc-adapter@9.0.4: + resolution: {integrity: sha512-5ZZY1+lGq8LEKuDlg9M2RPJHlH3R7OVwyHqMcUsLKCgd9Wvf+QrFTCItkXXYPmrJn8H6gRLXbSgxLLdexiqHxw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/webrtc-adapter/-/webrtc-adapter-9.0.4.tgz} + engines: {node: '>=6.0.0', npm: '>=3.10.0'} + yallist@3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} @@ -2471,6 +2519,8 @@ snapshots: '@biomejs/cli-win32-x64@2.4.6': optional: true + '@bufbuild/protobuf@1.10.1': {} + '@emoji-mart/data@1.2.1': {} '@emoji-mart/react@1.1.1(emoji-mart@5.6.0)(react@19.2.4)': @@ -2592,6 +2642,12 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@livekit/mutex@1.1.1': {} + + '@livekit/protocol@1.44.0': + dependencies: + '@bufbuild/protobuf': 1.10.1 + '@noble/ciphers@2.1.1': {} '@noble/curves@2.0.1': @@ -3306,6 +3362,8 @@ snapshots: dependencies: '@types/ms': 2.1.0 + '@types/dom-mediacapture-record@1.0.22': {} + '@types/estree-jsx@1.0.5': dependencies: '@types/estree': 1.0.8 @@ -3524,6 +3582,8 @@ snapshots: estree-util-is-identifier-name@3.0.0: {} + events@3.3.0: {} + extend@3.0.2: {} fast-glob@3.3.3: @@ -3659,6 +3719,8 @@ snapshots: jiti@1.21.7: {} + jose@6.2.2: {} + js-tokens@4.0.0: {} jsesc@3.1.0: {} @@ -3669,8 +3731,23 @@ snapshots: lines-and-columns@1.2.4: {} + livekit-client@2.18.1(@types/dom-mediacapture-record@1.0.22): + dependencies: + '@livekit/mutex': 1.1.1 + '@livekit/protocol': 1.44.0 + '@types/dom-mediacapture-record': 1.0.22 + events: 3.3.0 + jose: 6.2.2 + loglevel: 1.9.2 + sdp-transform: 2.15.0 + tslib: 2.8.1 + typed-emitter: 2.1.0 + webrtc-adapter: 9.0.4 + lodash@4.17.23: {} + loglevel@1.9.2: {} + longest-streak@3.1.0: {} loose-envify@1.4.0: @@ -4335,8 +4412,17 @@ snapshots: dependencies: queue-microtask: 1.2.3 + rxjs@7.8.2: + dependencies: + tslib: 2.8.1 + optional: true + scheduler@0.27.0: {} + sdp-transform@2.15.0: {} + + sdp@3.2.2: {} + semver@6.3.1: {} seroval-plugins@1.5.2(seroval@1.5.2): @@ -4459,6 +4545,10 @@ snapshots: optionalDependencies: fsevents: 2.3.3 + typed-emitter@2.1.0: + optionalDependencies: + rxjs: 7.8.2 + typescript@5.9.3: {} undici-types@7.18.2: {} @@ -4561,6 +4651,10 @@ snapshots: webpack-virtual-modules@0.6.2: {} + webrtc-adapter@9.0.4: + dependencies: + sdp: 3.2.2 + yallist@3.1.1: {} yaml@2.8.3: {} diff --git a/desktop/public/worklet.js b/desktop/public/worklet.js new file mode 100644 index 0000000000..f833814329 --- /dev/null +++ b/desktop/public/worklet.js @@ -0,0 +1,39 @@ +// AudioWorklet processor — runs in the AudioWorklet thread. +// Accumulates PCM Float32 samples and sends 100ms batches to the main thread. +class SttTapProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.buffer = new Float32Array(4800); // ~100ms at 48kHz + this.offset = 0; + } + + process(inputs) { + const input = inputs[0]?.[0]; // mono channel + if (!input) return true; + + // Accumulate samples + const remaining = this.buffer.length - this.offset; + const toCopy = Math.min(input.length, remaining); + this.buffer.set(input.subarray(0, toCopy), this.offset); + this.offset += toCopy; + + // Flush when buffer is full + if (this.offset >= this.buffer.length) { + // Transfer ownership for zero-copy + this.port.postMessage(this.buffer, [this.buffer.buffer]); + this.buffer = new Float32Array(4800); + this.offset = 0; + + // Handle leftover samples + if (toCopy < input.length) { + const leftover = input.subarray(toCopy); + this.buffer.set(leftover); + this.offset = leftover.length; + } + } + + return true; + } +} + +registerProcessor('stt-tap-processor', SttTapProcessor); diff --git a/desktop/src-tauri/Info.plist b/desktop/src-tauri/Info.plist index 41531bc3ad..b46d11a263 100644 --- a/desktop/src-tauri/Info.plist +++ b/desktop/src-tauri/Info.plist @@ -6,5 +6,7 @@ Sprout CFBundleName Sprout + NSMicrophoneUsageDescription + Sprout needs microphone access for voice huddles. diff --git a/desktop/src-tauri/src/app_state.rs b/desktop/src-tauri/src/app_state.rs index 7af53fd053..86e8f6afb5 100644 --- a/desktop/src-tauri/src/app_state.rs +++ b/desktop/src-tauri/src/app_state.rs @@ -3,6 +3,7 @@ use std::{collections::HashMap, io::Write, sync::Mutex}; use nostr::{Keys, ToBech32}; use tauri::{AppHandle, Manager}; +use crate::huddle::HuddleState; use crate::managed_agents::ManagedAgentProcess; pub struct AppState { @@ -12,6 +13,7 @@ pub struct AppState { pub session_token: Mutex>, pub managed_agents_store_lock: Mutex<()>, pub managed_agent_processes: Mutex>, + pub huddle_state: Mutex, } pub fn build_app_state() -> AppState { @@ -55,6 +57,7 @@ pub fn build_app_state() -> AppState { session_token: Mutex::new(None), managed_agents_store_lock: Mutex::new(()), managed_agent_processes: Mutex::new(HashMap::new()), + huddle_state: Mutex::new(HuddleState::default()), } } diff --git a/desktop/src-tauri/src/events.rs b/desktop/src-tauri/src/events.rs index efa18ebaa9..a0b4e2bc8a 100644 --- a/desktop/src-tauri/src/events.rs +++ b/desktop/src-tauri/src/events.rs @@ -357,6 +357,62 @@ pub fn build_profile( Ok(EventBuilder::new(Kind::Custom(0), content)) } +// ── Huddles ────────────────────────────────────────────────────────────────── + +/// Kind 48100 — huddle started advisory posted to the parent channel. +pub fn build_huddle_started( + parent_channel_id: &str, + ephemeral_channel_id: &str, + livekit_room: &str, +) -> Result { + let content = serde_json::json!({ + "ephemeral_channel_id": ephemeral_channel_id, + "livekit_room": livekit_room, + }) + .to_string(); + let tags = vec![tag(vec!["h", parent_channel_id])?]; + Ok(EventBuilder::new(Kind::Custom(48100), content).tags(tags)) +} + +/// Kind 48101 — participant joined a huddle, posted to the parent channel. +pub fn build_huddle_participant_joined( + parent_channel_id: &str, + ephemeral_channel_id: &str, +) -> Result { + let content = serde_json::json!({ + "ephemeral_channel_id": ephemeral_channel_id, + }) + .to_string(); + let tags = vec![tag(vec!["h", parent_channel_id])?]; + Ok(EventBuilder::new(Kind::Custom(48101), content).tags(tags)) +} + +/// Kind 48102 — participant left a huddle, posted to the parent channel. +pub fn build_huddle_participant_left( + parent_channel_id: &str, + ephemeral_channel_id: &str, +) -> Result { + let content = serde_json::json!({ + "ephemeral_channel_id": ephemeral_channel_id, + }) + .to_string(); + let tags = vec![tag(vec!["h", parent_channel_id])?]; + Ok(EventBuilder::new(Kind::Custom(48102), content).tags(tags)) +} + +/// Kind 48103 — huddle ended, posted to the parent channel. +pub fn build_huddle_ended( + parent_channel_id: &str, + ephemeral_channel_id: &str, +) -> Result { + let content = serde_json::json!({ + "ephemeral_channel_id": ephemeral_channel_id, + }) + .to_string(); + let tags = vec![tag(vec!["h", parent_channel_id])?]; + Ok(EventBuilder::new(Kind::Custom(48103), content).tags(tags)) +} + // ── Social notes ──────────────────────────────────────────────────────────── /// Kind 1 — NIP-01 short text note (global, no channel scope). diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs new file mode 100644 index 0000000000..95e424b05c --- /dev/null +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -0,0 +1,399 @@ +//! Huddle (voice/video) state machine and Tauri commands. +//! +//! Mental model: +//! parent channel → start_huddle → ephemeral channel + LiveKit token +//! other clients → join_huddle → LiveKit token +//! any client → leave_huddle → lifecycle event, clear local state +//! creator → end_huddle → archive ephemeral channel, clear state +//! +//! HuddleState is stored in AppState and serialized for get_huddle_state. + +use reqwest::Method; +use serde::{Deserialize, Serialize}; +use tauri::State; +use uuid::Uuid; + +use crate::{ + app_state::AppState, + events, + relay::{api_path, build_authed_request, send_json_request, submit_event}, +}; + +// ── State types ─────────────────────────────────────────────────────────────── + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum HuddlePhase { + Idle, + Creating, + Connecting, + Active, + Leaving, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HuddleState { + pub phase: HuddlePhase, + pub parent_channel_id: Option, + pub ephemeral_channel_id: Option, + pub livekit_token: Option, + pub livekit_url: Option, + pub livekit_room: Option, + /// Participant pubkey hex strings. + pub participants: Vec, +} + +impl Default for HuddleState { + fn default() -> Self { + Self { + phase: HuddlePhase::Idle, + parent_channel_id: None, + ephemeral_channel_id: None, + livekit_token: None, + livekit_url: None, + livekit_room: None, + participants: Vec::new(), + } + } +} + +// ── Response types ──────────────────────────────────────────────────────────── + +/// Returned by start_huddle and join_huddle. +#[derive(Debug, Serialize, Deserialize)] +pub struct HuddleJoinInfo { + pub ephemeral_channel_id: String, + pub livekit_token: String, + pub livekit_url: String, + pub livekit_room: String, +} + +/// Raw response from `POST /api/huddles/{channel_id}/token`. +#[derive(Debug, Deserialize)] +struct LiveKitTokenResponse { + pub token: String, + pub url: String, + pub room: String, +} + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +fn parse_channel_uuid(channel_id: &str) -> Result { + Uuid::parse_str(channel_id).map_err(|_| format!("invalid channel UUID: {channel_id}")) +} + +/// Fetch a LiveKit token from the relay for the given channel. +async fn fetch_livekit_token( + channel_id: &str, + state: &AppState, +) -> Result { + let path = api_path(&["huddles", channel_id, "token"]); + let request = build_authed_request(&state.http_client, Method::POST, &path, state)?; + send_json_request(request).await +} + +// ── Tauri commands ──────────────────────────────────────────────────────────── + +/// Start a new huddle in the given parent channel. +/// +/// Steps: +/// 1. Create an ephemeral channel (kind 9007, ttl=3600). +/// 2. Add each invited member to the ephemeral channel (kind 9000). +/// 3. Fetch a LiveKit token from the relay. +/// 4. Emit KIND_HUDDLE_STARTED to the parent channel (kind 48100) — only after +/// token is confirmed, so no phantom announcement on failure. +/// 5. Store state and return join info. +/// +/// If ANY step fails (including channel creation), the orphaned ephemeral +/// channel is archived (best-effort) and state is reset to Idle. +#[tauri::command] +pub async fn start_huddle( + parent_channel_id: String, + member_pubkeys: Vec, + state: State<'_, AppState>, +) -> Result { + // Transition to Creating. + { + let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + if hs.phase != HuddlePhase::Idle { + return Err(format!( + "cannot start huddle: already in phase {:?}", + hs.phase + )); + } + hs.phase = HuddlePhase::Creating; + hs.parent_channel_id = Some(parent_channel_id.clone()); + } + + let ephemeral_uuid = Uuid::new_v4(); + let ephemeral_channel_id = ephemeral_uuid.to_string(); + let short_id = &ephemeral_channel_id[..8]; + let channel_name = format!("huddle-{short_id}"); + + // All steps wrapped so we can roll back on ANY failure, including step 1. + // channel_was_created tracks whether we need to archive on rollback. + let mut channel_was_created = false; + + let result: Result = async { + // 1. Create ephemeral channel. + let create_builder = events::build_create_channel( + ephemeral_uuid, + &channel_name, + "private", + "stream", + None, + Some(3600), + )?; + submit_event(create_builder, &state).await?; + channel_was_created = true; + + // 2. Add members to the ephemeral channel (best-effort). + for pubkey in &member_pubkeys { + let add_builder = events::build_add_member(ephemeral_uuid, pubkey, None)?; + if let Err(e) = submit_event(add_builder, &state).await { + eprintln!("sprout-desktop: huddle add_member failed for {pubkey}: {e}"); + } + } + + // 3. Fetch LiveKit token BEFORE emitting HUDDLE_STARTED. + // This prevents a phantom announcement if the token fetch fails. + let lk = fetch_livekit_token(&ephemeral_channel_id, &state).await?; + + // 4. Emit HUDDLE_STARTED to parent channel — only now that token is confirmed. + let started_builder = events::build_huddle_started( + &parent_channel_id, + &ephemeral_channel_id, + &lk.room, + )?; + submit_event(started_builder, &state).await?; + + Ok(lk) + } + .await; + + match result { + Ok(lk) => { + // 5. Store active state. + { + let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + hs.phase = HuddlePhase::Active; + hs.ephemeral_channel_id = Some(ephemeral_channel_id.clone()); + hs.livekit_token = Some(lk.token.clone()); + hs.livekit_url = Some(lk.url.clone()); + hs.livekit_room = Some(lk.room.clone()); + hs.participants = member_pubkeys; + } + + Ok(HuddleJoinInfo { + ephemeral_channel_id, + livekit_token: lk.token, + livekit_url: lk.url, + livekit_room: lk.room, + }) + } + Err(e) => { + // Rollback: archive the orphaned ephemeral channel if it was created. + if channel_was_created { + if let Ok(archive_builder) = events::build_archive(ephemeral_uuid) { + if let Err(ae) = submit_event(archive_builder, &state).await { + eprintln!( + "sprout-desktop: rollback archive of {ephemeral_channel_id} failed: {ae}" + ); + } + } + } + // Reset state to Idle so the user can retry. + if let Ok(mut hs) = state.huddle_state.lock() { + *hs = HuddleState::default(); + } + Err(e) + } + } +} + +/// Join an existing huddle in the given parent channel. +/// +/// Steps: +/// 1. Fetch a LiveKit token from the relay for the ephemeral channel. +/// 2. Emit KIND_HUDDLE_PARTICIPANT_JOINED to the parent channel (best-effort). +/// 3. Store state and return join info. +#[tauri::command] +pub async fn join_huddle( + parent_channel_id: String, + ephemeral_channel_id: String, + livekit_room: String, + state: State<'_, AppState>, +) -> Result { + // Transition to Connecting. + { + let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + if hs.phase != HuddlePhase::Idle { + return Err(format!( + "cannot join huddle: already in phase {:?}", + hs.phase + )); + } + hs.phase = HuddlePhase::Connecting; + hs.parent_channel_id = Some(parent_channel_id.clone()); + hs.ephemeral_channel_id = Some(ephemeral_channel_id.clone()); + hs.livekit_room = Some(livekit_room.clone()); + } + + // 1. Fetch LiveKit token. On failure, reset state to Idle so user can retry. + let lk = match fetch_livekit_token(&ephemeral_channel_id, &state).await { + Ok(lk) => lk, + Err(e) => { + if let Ok(mut hs) = state.huddle_state.lock() { + *hs = HuddleState::default(); + } + return Err(e); + } + }; + + // 2. Emit PARTICIPANT_JOINED to parent channel (best-effort — don't fail the join). + if let Ok(joined_builder) = + events::build_huddle_participant_joined(&parent_channel_id, &ephemeral_channel_id) + { + if let Err(e) = submit_event(joined_builder, &state).await { + eprintln!("sprout-desktop: huddle_participant_joined event failed: {e}"); + } + } + + // 3. Store active state. + { + let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + hs.phase = HuddlePhase::Active; + hs.livekit_token = Some(lk.token.clone()); + hs.livekit_url = Some(lk.url.clone()); + hs.livekit_room = Some(lk.room.clone()); + } + + Ok(HuddleJoinInfo { + ephemeral_channel_id, + livekit_token: lk.token, + livekit_url: lk.url, + livekit_room: lk.room, + }) +} + +/// Leave the current huddle. +/// +/// Steps: +/// 1. Emit KIND_HUDDLE_PARTICIPANT_LEFT to the parent channel. +/// 2. Clear local huddle state. +#[tauri::command] +pub async fn leave_huddle(state: State<'_, AppState>) -> Result<(), String> { + let (parent_channel_id, ephemeral_channel_id) = { + let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + if hs.phase == HuddlePhase::Idle { + return Ok(()); // Nothing to leave. + } + hs.phase = HuddlePhase::Leaving; + ( + hs.parent_channel_id.clone().unwrap_or_default(), + hs.ephemeral_channel_id.clone().unwrap_or_default(), + ) + }; + + // Emit PARTICIPANT_LEFT (best-effort). + if !parent_channel_id.is_empty() && !ephemeral_channel_id.is_empty() { + if let Ok(left_builder) = + events::build_huddle_participant_left(&parent_channel_id, &ephemeral_channel_id) + { + if let Err(e) = submit_event(left_builder, &state).await { + eprintln!("sprout-desktop: huddle_participant_left event failed: {e}"); + } + } + } + + // Clear state. + { + let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + *hs = HuddleState::default(); + } + + Ok(()) +} + +/// End the current huddle (creator only). +/// +/// Steps: +/// 1. Emit KIND_HUDDLE_ENDED to the parent channel. +/// 2. Archive the ephemeral channel. +/// 3. Clear local huddle state. +#[tauri::command] +pub async fn end_huddle(state: State<'_, AppState>) -> Result<(), String> { + let (parent_channel_id, ephemeral_channel_id) = { + let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + if hs.phase == HuddlePhase::Idle { + return Ok(()); // Nothing to end. + } + hs.phase = HuddlePhase::Leaving; + ( + hs.parent_channel_id.clone().unwrap_or_default(), + hs.ephemeral_channel_id.clone().unwrap_or_default(), + ) + }; + + // Emit HUDDLE_ENDED (best-effort). + if !parent_channel_id.is_empty() && !ephemeral_channel_id.is_empty() { + if let Ok(ended_builder) = + events::build_huddle_ended(&parent_channel_id, &ephemeral_channel_id) + { + if let Err(e) = submit_event(ended_builder, &state).await { + eprintln!("sprout-desktop: huddle_ended event failed: {e}"); + } + } + } + + // Archive the ephemeral channel (best-effort). + if !ephemeral_channel_id.is_empty() { + if let Ok(uuid) = parse_channel_uuid(&ephemeral_channel_id) { + if let Ok(archive_builder) = events::build_archive(uuid) { + if let Err(e) = submit_event(archive_builder, &state).await { + eprintln!("sprout-desktop: huddle archive ephemeral channel failed: {e}"); + } + } + } + } + + // Clear state. + { + let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + *hs = HuddleState::default(); + } + + Ok(()) +} + +/// Return the current HuddleState (serialized for the frontend). +#[tauri::command] +pub fn get_huddle_state(state: State<'_, AppState>) -> Result { + let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + Ok(hs.clone()) +} + +/// Receive raw PCM audio bytes from the AudioWorklet. +/// Phase 1 stub — logs receipt. Phase 2 will feed to STT pipeline. +#[tauri::command] +pub fn push_audio_pcm(request: tauri::ipc::Request<'_>) -> Result<(), String> { + match request.body() { + tauri::ipc::InvokeBody::Raw(bytes) => { + // Phase 1: just acknowledge receipt. Phase 2 will process. + let sample_count = bytes.len() / 4; // f32 = 4 bytes + if sample_count > 0 { + // Log occasionally to avoid spam. + static COUNTER: std::sync::atomic::AtomicU64 = + std::sync::atomic::AtomicU64::new(0); + let count = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if count % 100 == 0 { + eprintln!( + "sprout-desktop: push_audio_pcm received {sample_count} samples (batch #{count})" + ); + } + } + Ok(()) + } + _ => Err("expected raw binary body".to_string()), + } +} diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index e8a07a3ab4..232627c8d9 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -1,6 +1,7 @@ mod app_state; mod commands; mod events; +mod huddle; mod managed_agents; mod migration; mod models; @@ -9,6 +10,7 @@ mod util; use app_state::{build_app_state, resolve_persisted_identity, AppState}; use commands::*; +use huddle::{end_huddle, get_huddle_state, join_huddle, leave_huddle, push_audio_pcm, start_huddle}; use managed_agents::{ ensure_nest, find_managed_agent_mut, kill_stale_tracked_processes, load_managed_agents, save_managed_agents, start_managed_agent_process, sync_managed_agent_processes, BackendKind, @@ -498,6 +500,12 @@ pub fn run() { get_contact_list, set_contact_list, get_notes_timeline, + start_huddle, + join_huddle, + leave_huddle, + end_huddle, + get_huddle_state, + push_audio_pcm, ]) .build(tauri::generate_context!()) .expect("error while building tauri application"); diff --git a/desktop/src/features/huddle/components/HuddleBar.tsx b/desktop/src/features/huddle/components/HuddleBar.tsx new file mode 100644 index 0000000000..ece1ae684e --- /dev/null +++ b/desktop/src/features/huddle/components/HuddleBar.tsx @@ -0,0 +1,126 @@ +import { invoke } from '@tauri-apps/api/core'; +import { Mic, MicOff, PhoneOff, Users } from 'lucide-react'; +import * as React from 'react'; + +import { cn } from '@/shared/lib/cn'; +import { Button } from '@/shared/ui/button'; +import { ParticipantList } from './ParticipantList'; + +// Shape returned by the `get_huddle_state` Tauri command +type HuddleState = { + phase: 'idle' | 'creating' | 'connecting' | 'active' | 'leaving'; + parent_channel_id: string | null; + ephemeral_channel_id: string | null; + livekit_token: string | null; + livekit_url: string | null; + livekit_room: string | null; + participants: string[]; // pubkey hex strings +}; + +type HuddleBarProps = { + /** MediaStreamTrack for local mic — used for mute toggle */ + localAudioTrack: MediaStreamTrack | null; + className?: string; +}; + +export function HuddleBar({ localAudioTrack, className }: HuddleBarProps) { + const [state, setState] = React.useState(null); + const [isMuted, setIsMuted] = React.useState(false); + const [isLeaving, setIsLeaving] = React.useState(false); + + // Poll huddle state — replace with event listener once Rust emits events + React.useEffect(() => { + let cancelled = false; + + async function poll() { + try { + const s = await invoke('get_huddle_state'); + if (!cancelled) setState(s); + } catch { + // Command not yet registered or no active huddle — ignore + if (!cancelled) setState(null); + } + } + + void poll(); + const id = window.setInterval(() => void poll(), 2_000); + + return () => { + cancelled = true; + window.clearInterval(id); + }; + }, []); + + // Sync mute state to the audio track + React.useEffect(() => { + if (localAudioTrack) { + localAudioTrack.enabled = !isMuted; + } + }, [isMuted, localAudioTrack]); + + if (!state || state.phase !== 'active') return null; + + async function handleLeave() { + if (isLeaving) return; + setIsLeaving(true); + try { + await invoke('leave_huddle'); + setState(null); + } catch (e) { + console.error('Failed to leave huddle:', e); + } finally { + setIsLeaving(false); + } + } + + return ( +
+ {/* Room label */} + + {state.livekit_room ?? 'Huddle'} + + + {/* Participant count */} +
+ + {state.participants.length} +
+ + {/* Participant avatars */} + {state.participants.length > 0 && ( + + )} + + {/* Mute toggle */} + + + {/* Leave button */} + +
+ ); +} diff --git a/desktop/src/features/huddle/components/ParticipantList.tsx b/desktop/src/features/huddle/components/ParticipantList.tsx new file mode 100644 index 0000000000..cb8b507ec8 --- /dev/null +++ b/desktop/src/features/huddle/components/ParticipantList.tsx @@ -0,0 +1,51 @@ +import { cn } from '@/shared/lib/cn'; + +// Legacy type kept for any callers that haven't migrated yet +export type HuddleParticipant = { + identity: string; + displayName: string; + isMuted: boolean; +}; + +type ParticipantListProps = { + /** Pubkey hex strings from the Rust huddle state */ + participants: string[]; + className?: string; +}; + +export function ParticipantList({ participants, className }: ParticipantListProps) { + if (participants.length === 0) return null; + + return ( +
+ {participants.map((pubkey) => ( + + ))} +
+ ); +} + +type ParticipantAvatarProps = { + pubkey: string; +}; + +function ParticipantAvatar({ pubkey }: ParticipantAvatarProps) { + // Use first 6 hex chars as a short identifier + const shortId = pubkey.slice(0, 6).toUpperCase(); + + // Derive a stable hue from the pubkey for a distinct avatar color + const hue = parseInt(pubkey.slice(0, 4), 16) % 360; + const style = { backgroundColor: `hsl(${hue}, 60%, 55%)`, color: '#fff' }; + + return ( +
+ {shortId} +
+ ); +} diff --git a/desktop/src/features/huddle/index.ts b/desktop/src/features/huddle/index.ts new file mode 100644 index 0000000000..a61a31a8b8 --- /dev/null +++ b/desktop/src/features/huddle/index.ts @@ -0,0 +1,6 @@ +export { connectToHuddle } from './lib/livekit'; +export type { HuddleConnection } from './lib/livekit'; +export { setupAudioWorklet } from './lib/audioWorklet'; +export { HuddleBar } from './components/HuddleBar'; +export { ParticipantList } from './components/ParticipantList'; +export type { HuddleParticipant } from './components/ParticipantList'; diff --git a/desktop/src/features/huddle/lib/audioWorklet.ts b/desktop/src/features/huddle/lib/audioWorklet.ts new file mode 100644 index 0000000000..1f83fbd779 --- /dev/null +++ b/desktop/src/features/huddle/lib/audioWorklet.ts @@ -0,0 +1,59 @@ +// Tauri internals surface — not part of the public @tauri-apps/api but +// available at runtime in the webview. We use it here for raw binary invoke +// (InvokeBody::Raw on the Rust side) which the typed wrapper doesn't support. +declare global { + interface Window { + __TAURI_INTERNALS__: { + invoke: (cmd: string, payload?: unknown) => Promise; + }; + } +} + +export async function setupAudioWorklet( + audioTrack: MediaStreamTrack, +): Promise<{ stop: () => void }> { + const audioContext = new AudioContext({ sampleRate: 48000 }); + + // Resume after user gesture (required by autoplay policy) + if (audioContext.state === 'suspended') { + await audioContext.resume(); + } + + // Load the worklet processor (must live in public/ for Vite to serve it) + await audioContext.audioWorklet.addModule('/worklet.js'); + + // Create source from the mic track + const source = audioContext.createMediaStreamSource( + new MediaStream([audioTrack]), + ); + + // Create worklet node + const workletNode = new AudioWorkletNode(audioContext, 'stt-tap-processor'); + + // Connect: mic → worklet (tap only — no playback) + source.connect(workletNode); + + // Forward PCM batches to Rust via raw binary invoke + workletNode.port.onmessage = async (event: MessageEvent) => { + const float32 = event.data; + try { + // Tauri v2 InvokeBody::Raw only accepts ArrayBuffer | Uint8Array. + // Create a zero-copy Uint8Array view over the same underlying buffer. + // Rust reinterprets the bytes as f32 on the other side. + await window.__TAURI_INTERNALS__.invoke( + 'push_audio_pcm', + new Uint8Array(float32.buffer, float32.byteOffset, float32.byteLength), + ); + } catch (e) { + console.error('Failed to send PCM to Rust:', e); + } + }; + + return { + stop: () => { + source.disconnect(); + workletNode.disconnect(); + void audioContext.close(); + }, + }; +} diff --git a/desktop/src/features/huddle/lib/livekit.ts b/desktop/src/features/huddle/lib/livekit.ts new file mode 100644 index 0000000000..0b263e9726 --- /dev/null +++ b/desktop/src/features/huddle/lib/livekit.ts @@ -0,0 +1,45 @@ +import { LocalAudioTrack, Room } from 'livekit-client'; + +export interface HuddleConnection { + room: Room; + localAudioTrack: MediaStreamTrack; + disconnect: () => Promise; +} + +export async function connectToHuddle( + url: string, + token: string, +): Promise { + const room = new Room(); + let stream: MediaStream | null = null; + + try { + stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const audioTrack = stream.getAudioTracks()[0]; + + await room.connect(url, token); + + try { + // false = don't let LiveKit manage the track lifecycle + const localTrack = new LocalAudioTrack(audioTrack, undefined, false); + await room.localParticipant.publishTrack(localTrack); + } catch (publishErr) { + // Publish failed after connect — disconnect room before propagating + room.disconnect(); + throw publishErr; + } + + return { + room, + localAudioTrack: audioTrack, + disconnect: async () => { + room.disconnect(); + stream?.getTracks().forEach((t) => t.stop()); + }, + }; + } catch (err) { + // Clean up mic stream on any failure (getUserMedia, connect, or publish) + stream?.getTracks().forEach((t) => t.stop()); + throw err; + } +} From a2304b9a27441a5d1ff9d081c677c56ac29d58ca Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Sat, 11 Apr 2026 12:45:23 -0400 Subject: [PATCH 02/41] =?UTF-8?q?feat(huddles):=20Phase=202=20=E2=80=94=20?= =?UTF-8?q?Speech-to-Text=20Pipeline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Human speech is transcribed and posted to the ephemeral channel. Speak in huddle → text appears in channel → agents can see it. STT Pipeline (stt.rs): - PCM f32 48kHz from AudioWorklet → rubato resample to 16kHz mono → earshot VAD → sherpa-onnx Moonshine → transcribed text - Dedicated std::thread (CPU-bound, not async) - Bounded audio queue (sync_channel, 50 slots, try_send drops on backpressure) - Shutdown flag + Drop impl joins worker thread cleanly - Final flush: buffered speech transcribed on shutdown - Merged decoder layout (v2) matches Moonshine tiny int8 model Model Download Manager (models.rs): - Background download of Moonshine tiny (~26MB) from sherpa-onnx releases - Atomic extraction: temp dir → verify → rename-backup swap (old model preserved on failure) - Platform-gated: #[cfg(unix)] tar extraction, clear error on non-Unix - Race-safe: status set to Downloading before spawn - OnceLock singleton, no unwrap() on mutex (poison recovery) Pipeline Integration (mod.rs): - push_audio_pcm feeds SttPipeline when active - Transcribed text posted as kind:9 with agent p-tags (read at post time, not snapshot) - Auto-start on huddle join/start if models ready (is_moonshine_ready) - Pipeline shutdown called before state clear on leave/end - recv_timeout replaces busy polling New dependencies: sherpa-onnx 1.12, earshot 1.0, rubato 2.0, audioadapter-buffers 3.0 Crossfired: Codex 3/10 → 8/10 → 10/10 APPROVE after 3 fix rounds. --- desktop/src-tauri/Cargo.lock | 209 +++++++++++++- desktop/src-tauri/Cargo.toml | 4 + desktop/src-tauri/src/huddle/mod.rs | 330 ++++++++++++++++++++-- desktop/src-tauri/src/huddle/models.rs | 351 +++++++++++++++++++++++ desktop/src-tauri/src/huddle/stt.rs | 367 +++++++++++++++++++++++++ desktop/src-tauri/src/lib.rs | 8 +- 6 files changed, 1249 insertions(+), 20 deletions(-) create mode 100644 desktop/src-tauri/src/huddle/models.rs create mode 100644 desktop/src-tauri/src/huddle/stt.rs diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index ae836764d5..e740b147f5 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -253,6 +253,43 @@ dependencies = [ "rand 0.9.2", ] +[[package]] +name = "audio-core" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ebbf82d06013f4c41fe71303feb980cddd78496d904d06be627972de51a24" + +[[package]] +name = "audioadapter" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91f87b70b051c5866680ad79f6743a42ccab264c009d1a71f4d33a3872ae60c8" +dependencies = [ + "audio-core", + "num-traits", +] + +[[package]] +name = "audioadapter-buffers" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9097d67933fb083d382ce980430afdb758aada60846010aee6be068c06cef0ca" +dependencies = [ + "audioadapter", + "audioadapter-sample", + "num-traits", +] + +[[package]] +name = "audioadapter-sample" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ab94f2bc04a14e1f49ee5f222f66460e8a1b51627bdfedf34eed394d747938" +dependencies = [ + "audio-core", + "num-traits", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -507,6 +544,16 @@ dependencies = [ "serde", ] +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + [[package]] name = "bzip2" version = "0.5.2" @@ -1176,6 +1223,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "earshot" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d393a8f23619412e0502df1b94cd148e34855a50a4143d365cc6336cc338b4f4" +dependencies = [ + "libm", +] + [[package]] name = "embed-resource" version = "3.0.8" @@ -2448,6 +2504,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "libredox" version = "0.1.15" @@ -2781,12 +2843,30 @@ dependencies = [ "zbus", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -3421,6 +3501,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -3700,6 +3789,15 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" +[[package]] +name = "realfft" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677" +dependencies = [ + "rustfft", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -3898,6 +3996,22 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rubato" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce96ead1a91f7895704a9f08ea5947dfc8bd7c1f2936a22295b655ec67e5c6ef" +dependencies = [ + "audioadapter", + "audioadapter-buffers", + "num-complex", + "num-integer", + "num-traits", + "realfft", + "visibility", + "windowfunctions", +] + [[package]] name = "rustc-hash" version = "2.1.2" @@ -3913,6 +4027,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rustix" version = "1.1.4" @@ -3933,6 +4061,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ "aws-lc-rs", + "log", "once_cell", "ring", "rustls-pki-types", @@ -4414,6 +4543,28 @@ dependencies = [ "digest 0.11.2", ] +[[package]] +name = "sherpa-onnx" +version = "1.12.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f69ec9bc7c245dc7b976e7662c8529b32236efb04ab9d53427af2c4b8393b0da" +dependencies = [ + "serde", + "serde_json", + "sherpa-onnx-sys", +] + +[[package]] +name = "sherpa-onnx-sys" +version = "1.12.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b78e254a3030040d015edcdc4adfa1ea4714d3284dba0034f120feccbbde4aff" +dependencies = [ + "bzip2 0.4.4", + "tar", + "ureq", +] + [[package]] name = "shlex" version = "1.3.0" @@ -4523,18 +4674,22 @@ name = "sprout" version = "0.1.0" dependencies = [ "atomic-write-file", + "audioadapter-buffers", "base64 0.22.1", "chrono", "dirs", + "earshot", "hex", "infer", "libc", "nostr 0.37.0", "png 0.18.1", "reqwest 0.13.2", + "rubato", "serde", "serde_json", "sha2 0.11.0", + "sherpa-onnx", "sprout-core", "tauri", "tauri-build", @@ -4572,6 +4727,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "string_cache" version = "0.8.9" @@ -5554,6 +5715,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "tray-icon" version = "0.21.3" @@ -5708,6 +5879,22 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64 0.22.1", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "url", + "webpki-roots 0.26.11", +] + [[package]] name = "url" version = "2.5.8" @@ -5769,6 +5956,17 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "visibility" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "vswhom" version = "0.1.0" @@ -6125,6 +6323,15 @@ dependencies = [ "windows-version", ] +[[package]] +name = "windowfunctions" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90628d739333b7c5d2ee0b70210b97b8cddc38440c682c96fd9e2c24c2db5f3a" +dependencies = [ + "num-traits", +] + [[package]] name = "windows" version = "0.61.3" @@ -6925,7 +7132,7 @@ checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" dependencies = [ "aes", "arbitrary", - "bzip2", + "bzip2 0.5.2", "constant_time_eq", "crc32fast", "crossbeam-utils", diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index b37c69a787..6184474ed3 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -49,6 +49,10 @@ tauri-plugin-notification = "2.3.3" uuid = { version = "1", features = ["v4"] } png = "0.18" zip = "2" +sherpa-onnx = "1.12" +earshot = "1.0" +rubato = "2.0" +audioadapter-buffers = "3.0" [dev-dependencies] tempfile = "3" diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index 95e424b05c..dd049f1a97 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -8,11 +8,17 @@ //! //! HuddleState is stored in AppState and serialized for get_huddle_state. +pub mod models; +pub mod stt; + use reqwest::Method; use serde::{Deserialize, Serialize}; +use std::sync::{Arc, Mutex}; use tauri::State; use uuid::Uuid; +use nostr::JsonUtil; + use crate::{ app_state::AppState, events, @@ -31,7 +37,7 @@ pub enum HuddlePhase { Leaving, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct HuddleState { pub phase: HuddlePhase, pub parent_channel_id: Option, @@ -39,8 +45,73 @@ pub struct HuddleState { pub livekit_token: Option, pub livekit_url: Option, pub livekit_room: Option, - /// Participant pubkey hex strings. + /// Participant pubkey hex strings (all members, including humans). pub participants: Vec, + /// Agent pubkeys only — used as p-tags on transcribed messages. + /// + /// Stored as `Arc>>` so the transcription task can clone + /// the `Arc` and read the current list at post time without holding the + /// outer `HuddleState` lock across an await point. + /// + /// Populated from `member_pubkeys` in `start_huddle` (the UI sends agent + /// pubkeys specifically). Joiners don't add agents — they were already + /// added by the creator. Serialized as a plain `Vec` for the + /// frontend via the custom `Serialize`/`Deserialize` impls below. + #[serde( + serialize_with = "serialize_agent_pubkeys", + deserialize_with = "deserialize_agent_pubkeys" + )] + pub agent_pubkeys: Arc>>, + /// Active STT pipeline — not serialized, not cloned. + #[serde(skip)] + pub stt_pipeline: Option>, +} + +fn serialize_agent_pubkeys( + v: &Arc>>, + s: S, +) -> Result +where + S: serde::Serializer, +{ + use serde::ser::SerializeSeq; + let guard = v.lock().unwrap_or_else(|e| e.into_inner()); + let mut seq = s.serialize_seq(Some(guard.len()))?; + for item in guard.iter() { + seq.serialize_element(item)?; + } + seq.end() +} + +fn deserialize_agent_pubkeys<'de, D>( + d: D, +) -> Result>>, D::Error> +where + D: serde::Deserializer<'de>, +{ + let v: Vec = serde::Deserialize::deserialize(d)?; + Ok(Arc::new(Mutex::new(v))) +} + +impl Clone for HuddleState { + fn clone(&self) -> Self { + let agent_pubkeys_snapshot = self + .agent_pubkeys + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(); + Self { + phase: self.phase.clone(), + parent_channel_id: self.parent_channel_id.clone(), + ephemeral_channel_id: self.ephemeral_channel_id.clone(), + livekit_token: self.livekit_token.clone(), + livekit_url: self.livekit_url.clone(), + livekit_room: self.livekit_room.clone(), + participants: self.participants.clone(), + agent_pubkeys: Arc::new(Mutex::new(agent_pubkeys_snapshot)), + stt_pipeline: None, // Never clone the pipeline handle. + } + } } impl Default for HuddleState { @@ -53,6 +124,8 @@ impl Default for HuddleState { livekit_url: None, livekit_room: None, participants: Vec::new(), + agent_pubkeys: Arc::new(Mutex::new(Vec::new())), + stt_pipeline: None, } } } @@ -92,6 +165,137 @@ async fn fetch_livekit_token( send_json_request(request).await } +/// Attempt to start the STT pipeline if models are present. +/// Silently skips if models are missing — huddle continues as voice-only. +/// +/// Fix 2: uses `models::is_moonshine_ready()` (checks all 4 expected files) +/// instead of an ad hoc `tokens.txt` existence check. +async fn maybe_start_stt_pipeline(state: &AppState, ephemeral_channel_id: &str) { + if !models::is_moonshine_ready() { + return; // Models not downloaded yet — voice-only mode. + } + let model_dir = match models::moonshine_model_dir() { + Some(d) => d, + None => return, + }; + + let pipeline = match stt::SttPipeline::new(model_dir) { + Ok(p) => Arc::new(p), + Err(e) => { + eprintln!("sprout-desktop: STT pipeline failed to start: {e}"); + return; + } + }; + + let channel_uuid = match parse_channel_uuid(ephemeral_channel_id) { + Ok(u) => u, + Err(_) => return, + }; + + // Clone the Arc>> BEFORE storing the pipeline, so we + // can pass it to the transcription task without holding the state lock. + let agent_pubkeys_arc = { + let hs = match state.huddle_state.lock() { + Ok(h) => h, + Err(_) => return, + }; + Arc::clone(&hs.agent_pubkeys) + }; + + // Store the pipeline. + { + let mut hs = match state.huddle_state.lock() { + Ok(h) => h, + Err(_) => return, + }; + hs.stt_pipeline = Some(Arc::clone(&pipeline)); + } + + spawn_transcription_task(pipeline, channel_uuid, agent_pubkeys_arc, state); +} + +/// Spawn a tokio task that reads text_rx and posts kind:9 events. +/// +/// Fix 1: `agent_pubkeys_arc` is an `Arc>>` cloned from +/// `HuddleState` — the task reads it at post time so p-tags are always +/// current, not a stale snapshot. +/// Fix 3: no `.unwrap()` on mutex — poisoned locks are recovered gracefully. +/// Fix 4: `recv_timeout` instead of `try_recv` + sleep — no busy-polling. +fn spawn_transcription_task( + pipeline: Arc, + channel_uuid: Uuid, + agent_pubkeys_arc: Arc>>, + state: &AppState, +) { + let http_client = state.http_client.clone(); + let keys = match state.keys.lock() { + Ok(k) => k.clone(), + Err(_) => return, + }; + let configured_api_token = state.configured_api_token.clone(); + + tauri::async_runtime::spawn(async move { + loop { + // Fix 3: recover from a poisoned mutex rather than panicking. + // Fix 4: recv_timeout blocks the thread efficiently; Disconnected + // means the pipeline worker has exited — stop the task. + let text = { + let rx = pipeline.text_rx.lock().unwrap_or_else(|e| e.into_inner()); + match rx.recv_timeout(std::time::Duration::from_millis(100)) { + Ok(t) => Some(t), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => None, + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => break, + } + }; + + let t = match text { + Some(t) if !t.is_empty() => t, + _ => continue, + }; + + // Fix 1: read current agent pubkeys at post time. + let agent_pubkeys: Vec = agent_pubkeys_arc + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(); + + let p_tags: Vec<&str> = agent_pubkeys.iter().map(|s| s.as_str()).collect(); + let builder = match events::build_message(channel_uuid, &t, None, &p_tags, &[]) { + Ok(b) => b, + Err(e) => { + eprintln!("sprout-desktop: STT build_message: {e}"); + continue; + } + }; + let event = match builder.sign_with_keys(&keys) { + Ok(e) => e, + Err(e) => { + eprintln!("sprout-desktop: STT sign event: {e}"); + continue; + } + }; + let event_json = event.as_json(); + let auth_header = match configured_api_token.as_deref() { + Some(token) => format!("Bearer {token}"), + None => format!("X-Pubkey {}", keys.public_key().to_hex()), + }; + let url = format!("{}/api/events", crate::relay::relay_api_base_url()); + let req = if auth_header.starts_with("Bearer ") { + http_client.post(&url).header("Authorization", &auth_header) + } else { + let pk = auth_header.strip_prefix("X-Pubkey ").unwrap_or(""); + http_client.post(&url).header("X-Pubkey", pk) + } + .header("Content-Type", "application/json") + .body(event_json); + + if let Err(e) = req.send().await { + eprintln!("sprout-desktop: STT kind:9 post failed: {e}"); + } + } + }); +} + // ── Tauri commands ──────────────────────────────────────────────────────────── /// Start a new huddle in the given parent channel. @@ -181,9 +385,16 @@ pub async fn start_huddle( hs.livekit_token = Some(lk.token.clone()); hs.livekit_url = Some(lk.url.clone()); hs.livekit_room = Some(lk.room.clone()); + // Fix 1: the UI sends agent pubkeys in member_pubkeys — store them + // separately so the transcription task can p-tag agents only. + *hs.agent_pubkeys.lock().unwrap_or_else(|e| e.into_inner()) = + member_pubkeys.clone(); hs.participants = member_pubkeys; } + // 6. Auto-start STT pipeline if models are ready. + maybe_start_stt_pipeline(&state, &ephemeral_channel_id).await; + Ok(HuddleJoinInfo { ephemeral_channel_id, livekit_token: lk.token, @@ -266,8 +477,12 @@ pub async fn join_huddle( hs.livekit_token = Some(lk.token.clone()); hs.livekit_url = Some(lk.url.clone()); hs.livekit_room = Some(lk.room.clone()); + // Note: agent_pubkeys stays empty for joiners — agents were added by the creator. } + // 4. Auto-start STT pipeline if models are ready. + maybe_start_stt_pipeline(&state, &ephemeral_channel_id).await; + Ok(HuddleJoinInfo { ephemeral_channel_id, livekit_token: lk.token, @@ -280,7 +495,8 @@ pub async fn join_huddle( /// /// Steps: /// 1. Emit KIND_HUDDLE_PARTICIPANT_LEFT to the parent channel. -/// 2. Clear local huddle state. +/// 2. Shut down the STT pipeline (Fix 5). +/// 3. Clear local huddle state. #[tauri::command] pub async fn leave_huddle(state: State<'_, AppState>) -> Result<(), String> { let (parent_channel_id, ephemeral_channel_id) = { @@ -306,6 +522,15 @@ pub async fn leave_huddle(state: State<'_, AppState>) -> Result<(), String> { } } + // Fix 5: signal the STT pipeline to stop before dropping state. + // The pipeline's Drop impl will join the worker thread for a clean exit. + { + let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + if let Some(ref pipeline) = hs.stt_pipeline { + pipeline.shutdown(); + } + } + // Clear state. { let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; @@ -320,7 +545,8 @@ pub async fn leave_huddle(state: State<'_, AppState>) -> Result<(), String> { /// Steps: /// 1. Emit KIND_HUDDLE_ENDED to the parent channel. /// 2. Archive the ephemeral channel. -/// 3. Clear local huddle state. +/// 3. Shut down the STT pipeline (Fix 5). +/// 4. Clear local huddle state. #[tauri::command] pub async fn end_huddle(state: State<'_, AppState>) -> Result<(), String> { let (parent_channel_id, ephemeral_channel_id) = { @@ -357,6 +583,15 @@ pub async fn end_huddle(state: State<'_, AppState>) -> Result<(), String> { } } + // Fix 5: signal the STT pipeline to stop before dropping state. + // The pipeline's Drop impl will join the worker thread for a clean exit. + { + let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + if let Some(ref pipeline) = hs.stt_pipeline { + pipeline.shutdown(); + } + } + // Clear state. { let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; @@ -373,23 +608,20 @@ pub fn get_huddle_state(state: State<'_, AppState>) -> Result) -> Result<(), String> { +pub fn push_audio_pcm( + request: tauri::ipc::Request<'_>, + state: State<'_, AppState>, +) -> Result<(), String> { match request.body() { tauri::ipc::InvokeBody::Raw(bytes) => { - // Phase 1: just acknowledge receipt. Phase 2 will process. - let sample_count = bytes.len() / 4; // f32 = 4 bytes - if sample_count > 0 { - // Log occasionally to avoid spam. - static COUNTER: std::sync::atomic::AtomicU64 = - std::sync::atomic::AtomicU64::new(0); - let count = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - if count % 100 == 0 { - eprintln!( - "sprout-desktop: push_audio_pcm received {sample_count} samples (batch #{count})" - ); + if let Ok(hs) = state.huddle_state.lock() { + if let Some(ref pipeline) = hs.stt_pipeline { + pipeline.push_audio(bytes.to_vec())?; } } Ok(()) @@ -397,3 +629,65 @@ pub fn push_audio_pcm(request: tauri::ipc::Request<'_>) -> Result<(), String> { _ => Err("expected raw binary body".to_string()), } } + +/// Start the STT pipeline for the active huddle. +/// +/// Creates the pipeline, stores it in HuddleState, and spawns a tokio task +/// that reads transcribed text and posts kind:9 events to the ephemeral +/// channel. +/// +/// No-op if models are not present — huddle continues as voice-only. +/// Safe to call multiple times: replaces the existing pipeline if already running. +#[tauri::command] +pub async fn start_stt_pipeline(state: State<'_, AppState>) -> Result<(), String> { + if !models::is_moonshine_ready() { + return Err("Moonshine model not ready".to_string()); + } + let model_dir = models::moonshine_model_dir() + .ok_or_else(|| "Moonshine model directory not found".to_string())?; + + let (ephemeral_channel_id, agent_pubkeys_arc) = { + let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + ( + hs.ephemeral_channel_id.clone(), + Arc::clone(&hs.agent_pubkeys), + ) + }; + + let ephemeral_channel_id = + ephemeral_channel_id.ok_or("no active huddle — start or join a huddle first")?; + let channel_uuid = parse_channel_uuid(&ephemeral_channel_id)?; + + let pipeline = Arc::new(stt::SttPipeline::new(model_dir)?); + + { + let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + hs.stt_pipeline = Some(Arc::clone(&pipeline)); + } + + spawn_transcription_task(pipeline, channel_uuid, agent_pubkeys_arc, &state); + Ok(()) +} + +/// Trigger a background download of voice models (Moonshine STT for Phase 2). +/// +/// Returns immediately — download runs in a tokio background task. +/// Poll `get_model_status` to track progress. +/// Safe to call multiple times: no-op if already downloading or ready. +#[tauri::command] +pub async fn download_voice_models(state: State<'_, AppState>) -> Result<(), String> { + let manager = models::global_model_manager() + .ok_or("model manager unavailable (home directory could not be resolved)")?; + manager.start_moonshine_download(state.http_client.clone()); + Ok(()) +} + +/// Return the current download status for all voice models. +#[tauri::command] +pub fn get_model_status(_state: State<'_, AppState>) -> Result { + let manager = models::global_model_manager() + .ok_or("model manager unavailable (home directory could not be resolved)")?; + Ok(models::VoiceModelStatus { + moonshine: manager.moonshine_status(), + }) +} diff --git a/desktop/src-tauri/src/huddle/models.rs b/desktop/src-tauri/src/huddle/models.rs new file mode 100644 index 0000000000..67e3d40f68 --- /dev/null +++ b/desktop/src-tauri/src/huddle/models.rs @@ -0,0 +1,351 @@ +//! Model download manager for STT (Moonshine) and TTS (Kokoro) models. +//! +//! Mental model: +//! app launch → start_moonshine_download (background) → ~/.sprout/models/moonshine-tiny/ +//! STT pipeline → is_moonshine_ready() → moonshine_model_dir() → run inference +//! +//! Models are downloaded once and cached. No versioning in MVP — presence of +//! all expected files is sufficient to consider the model ready. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, OnceLock}; + +use serde::{Deserialize, Serialize}; + +// ── Constants ───────────────────────────────────────────────────────────────── + +const MOONSHINE_DOWNLOAD_URL: &str = + "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/\ + sherpa-onnx-moonshine-tiny-en-int8.tar.bz2"; + +/// Subdirectory name produced by `tar xjf` on the archive. +const MOONSHINE_ARCHIVE_SUBDIR: &str = "sherpa-onnx-moonshine-tiny-en-int8"; + +/// Final directory name under `~/.sprout/models/`. +const MOONSHINE_MODEL_DIR_NAME: &str = "moonshine-tiny"; + +/// All files that must be present for the model to be considered ready. +const MOONSHINE_EXPECTED_FILES: &[&str] = &[ + "preprocessor.onnx", + "encoder.onnx", + "merged_decoder.onnx", + "tokens.txt", +]; + +// ── Status types ────────────────────────────────────────────────────────────── + +/// Download/readiness status for a single model. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ModelStatus { + NotDownloaded, + Downloading { progress_percent: u8 }, + Ready, + Error(String), +} + +/// Combined status for all voice models (returned to the frontend). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VoiceModelStatus { + pub moonshine: ModelStatus, +} + +// ── Platform-gated tar extraction ───────────────────────────────────────────── + +/// Extract a `.tar.bz2` archive into `dest_dir` using the system `tar`. +/// +/// Only available on Unix — on other platforms this returns an error at +/// compile time so the caller can handle it gracefully. +#[cfg(unix)] +fn extract_archive(archive_path: &Path, dest_dir: &Path) -> Result<(), String> { + let status = std::process::Command::new("tar") + .args([ + "xjf", + &archive_path.to_string_lossy(), + "-C", + &dest_dir.to_string_lossy(), + ]) + .status() + .map_err(|e| format!("tar execution failed: {e}"))?; + if !status.success() { + return Err(format!("tar exited with status {status}")); + } + Ok(()) +} + +#[cfg(not(unix))] +fn extract_archive(_archive_path: &Path, _dest_dir: &Path) -> Result<(), String> { + Err("Model download is not yet supported on this platform".to_string()) +} + +// ── ModelManager ────────────────────────────────────────────────────────────── + +/// Manages download and location of STT/TTS model files. +/// +/// Cheap to clone — the inner status is behind an `Arc>`. +#[derive(Clone)] +pub struct ModelManager { + /// `~/.sprout/models/` + models_dir: PathBuf, + moonshine_status: Arc>, +} + +impl ModelManager { + /// Create a new `ModelManager` rooted at `~/.sprout/models/`. + /// + /// Returns `None` if the home directory cannot be resolved. + pub fn new() -> Option { + let models_dir = dirs::home_dir()?.join(".sprout").join("models"); + Some(Self { + models_dir, + moonshine_status: Arc::new(Mutex::new(ModelStatus::NotDownloaded)), + }) + } + + /// Returns the path to the Moonshine model directory, or `None` if not ready. + pub fn moonshine_model_dir(&self) -> Option { + if self.is_moonshine_ready() { + Some(self.models_dir.join(MOONSHINE_MODEL_DIR_NAME)) + } else { + None + } + } + + /// Returns `true` if all expected Moonshine model files are present on disk. + pub fn is_moonshine_ready(&self) -> bool { + let dir = self.models_dir.join(MOONSHINE_MODEL_DIR_NAME); + MOONSHINE_EXPECTED_FILES + .iter() + .all(|f| dir.join(f).is_file()) + } + + /// Current Moonshine download status. + pub fn moonshine_status(&self) -> ModelStatus { + self.moonshine_status + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone() + } + + /// Trigger a background download of the Moonshine model. + /// + /// Returns immediately. Progress is tracked via `moonshine_status()`. + /// No-op if the model is already ready or a download is already running. + /// + /// Fix: status is set to `Downloading` synchronously *before* the task is + /// spawned, eliminating the race where two concurrent callers both see + /// `NotDownloaded` and each spawn a download. + pub fn start_moonshine_download(&self, http_client: reqwest::Client) { + // Fast path: already on disk — sync the status and return. + if self.is_moonshine_ready() { + *self.moonshine_status.lock().unwrap_or_else(|e| e.into_inner()) = + ModelStatus::Ready; + return; + } + + // Atomically check-and-set: if already Downloading or Ready, bail out. + // Setting the status here (before spawn) prevents a second caller from + // racing through this check while the first caller's task hasn't started. + { + let mut status = self + .moonshine_status + .lock() + .unwrap_or_else(|e| e.into_inner()); + match *status { + ModelStatus::Downloading { .. } | ModelStatus::Ready => return, + _ => {} + } + *status = ModelStatus::Downloading { progress_percent: 0 }; + } // lock released before spawn + + let manager = self.clone(); + tokio::spawn(async move { + if let Err(e) = manager.download_moonshine_model(http_client).await { + eprintln!("sprout-desktop: moonshine download failed: {e}"); + *manager + .moonshine_status + .lock() + .unwrap_or_else(|e2| e2.into_inner()) = ModelStatus::Error(e); + } + }); + } + + // ── Private ─────────────────────────────────────────────────────────────── + + fn set_status(&self, status: ModelStatus) { + *self + .moonshine_status + .lock() + .unwrap_or_else(|e| e.into_inner()) = status; + } + + /// Download, extract, and verify the Moonshine model archive. + /// + /// Extraction is atomic: we extract into a temp directory, verify all + /// expected files, then rename into place. A failed extraction leaves any + /// previously working model untouched. + async fn download_moonshine_model(&self, http_client: reqwest::Client) -> Result<(), String> { + // 1. Ensure models directory exists. + fs::create_dir_all(&self.models_dir) + .map_err(|e| format!("create models dir: {e}"))?; + + self.set_status(ModelStatus::Downloading { progress_percent: 0 }); + + let archive_path = self.models_dir.join("moonshine-tiny.tar.bz2"); + + eprintln!("sprout-desktop: downloading Moonshine model from {MOONSHINE_DOWNLOAD_URL}"); + + // 2. Fetch the archive. + let response = http_client + .get(MOONSHINE_DOWNLOAD_URL) + .send() + .await + .map_err(|e| format!("download request failed: {e}"))?; + + if !response.status().is_success() { + return Err(format!( + "download HTTP {}: {}", + response.status().as_u16(), + response.status().canonical_reason().unwrap_or("unknown"), + )); + } + + let content_length = response.content_length(); + + // 3. Stream bytes to disk with progress updates. + { + use tokio::io::AsyncWriteExt; + + let body = response + .bytes() + .await + .map_err(|e| format!("download stream error: {e}"))?; + + // Report progress based on content-length (if known). + if let Some(total) = content_length { + if total > 0 { + let pct = ((body.len() as u64 * 100) / total).min(89) as u8; + self.set_status(ModelStatus::Downloading { progress_percent: pct }); + } + } + + eprintln!("sprout-desktop: downloaded {} bytes, writing…", body.len()); + + let mut file = tokio::fs::File::create(&archive_path) + .await + .map_err(|e| format!("create archive file: {e}"))?; + file.write_all(&body) + .await + .map_err(|e| format!("write archive: {e}"))?; + file.flush() + .await + .map_err(|e| format!("flush archive: {e}"))?; + } + + self.set_status(ModelStatus::Downloading { progress_percent: 90 }); + + // 4. Extract into a temp directory so that a failure does not destroy + // any previously working model. + let temp_dir = self.models_dir.join("moonshine-tiny.tmp"); + let final_dir = self.models_dir.join(MOONSHINE_MODEL_DIR_NAME); + + // Clean up any leftover temp dir from a prior failed attempt. + if temp_dir.exists() { + fs::remove_dir_all(&temp_dir) + .map_err(|e| format!("remove stale temp dir: {e}"))?; + } + fs::create_dir_all(&temp_dir) + .map_err(|e| format!("create temp dir: {e}"))?; + + eprintln!("sprout-desktop: extracting Moonshine archive…"); + extract_archive(&archive_path, &temp_dir)?; + + // 5. Locate the extracted subdirectory inside the temp dir. + let extracted_subdir = temp_dir.join(MOONSHINE_ARCHIVE_SUBDIR); + if !extracted_subdir.is_dir() { + // Clean up before returning the error. + let _ = fs::remove_dir_all(&temp_dir); + return Err(format!( + "expected subdir '{}' not found after extraction", + MOONSHINE_ARCHIVE_SUBDIR, + )); + } + + // 6. Verify all expected files are present before touching the live dir. + let missing: Vec<&str> = MOONSHINE_EXPECTED_FILES + .iter() + .filter(|&&f| !extracted_subdir.join(f).is_file()) + .copied() + .collect(); + + if !missing.is_empty() { + let _ = fs::remove_dir_all(&temp_dir); + return Err(format!( + "model verification failed — missing: {}", + missing.join(", "), + )); + } + + // 7. Atomic swap: rename old out of the way first, then bring new in. + // This ensures a rename failure cannot destroy the previously working model. + let backup_dir = final_dir.with_extension("old"); + if final_dir.exists() { + // Remove any stale backup from a prior interrupted swap. + if backup_dir.exists() { + let _ = fs::remove_dir_all(&backup_dir); + } + fs::rename(&final_dir, &backup_dir) + .map_err(|e| format!("backup old model: {e}"))?; + } + + // Bring new model into place; restore backup on failure. + if let Err(e) = fs::rename(&extracted_subdir, &final_dir) { + if backup_dir.exists() { + let _ = fs::rename(&backup_dir, &final_dir); + } + return Err(format!("install new model: {e}")); + } + + // 8. Clean up backup, temp dir (now empty after rename), and archive. + let _ = fs::remove_dir_all(&backup_dir); + let _ = fs::remove_dir_all(&temp_dir); + let _ = fs::remove_file(&archive_path); + + eprintln!( + "sprout-desktop: Moonshine model ready at {}", + final_dir.display() + ); + + self.set_status(ModelStatus::Ready); + Ok(()) + } +} + +// ── Process-global singleton ────────────────────────────────────────────────── + +/// Process-global `ModelManager`. Initialized on first access. +/// +/// `None` only if the home directory cannot be resolved (extremely rare). +static GLOBAL_MODEL_MANAGER: OnceLock> = OnceLock::new(); + +/// Return a reference to the process-global `ModelManager`. +pub fn global_model_manager() -> Option<&'static ModelManager> { + GLOBAL_MODEL_MANAGER + .get_or_init(ModelManager::new) + .as_ref() +} + +// ── Standalone helpers (used by the STT pipeline) ──────────────────────────── + +/// Path to the Moonshine model directory, or `None` if not ready. +pub fn moonshine_model_dir() -> Option { + global_model_manager()?.moonshine_model_dir() +} + +/// `true` if all expected Moonshine model files are present on disk. +pub fn is_moonshine_ready() -> bool { + global_model_manager() + .map(|m| m.is_moonshine_ready()) + .unwrap_or(false) +} diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs new file mode 100644 index 0000000000..384f0b0d56 --- /dev/null +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -0,0 +1,367 @@ +//! Speech-to-Text pipeline for huddle voice transcription. +//! +//! Mental model: +//! +//! ```text +//! AudioWorklet (48 kHz f32 PCM) +//! → push_audio_pcm (Tauri cmd) +//! → SttPipeline::push_audio [bounded sync_channel] +//! → stt_worker thread +//! rubato: 48 kHz → 16 kHz mono +//! earshot VAD: accumulate speech frames +//! sherpa-onnx Moonshine: transcribe on silence +//! → text_rx [mpsc channel] +//! → tokio task (start_stt_pipeline) +//! builds kind:9 event → relay +//! ``` +//! +//! The worker runs on a dedicated `std::thread` (not async) because +//! sherpa-onnx is CPU-bound and not Send-safe across await points. + +use std::{ + path::PathBuf, + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc::{self, Receiver, SyncSender}, + Arc, Mutex, + }, + thread, + time::Duration, +}; + +// ── Public pipeline handle ──────────────────────────────────────────────────── + +/// Bounded audio queue capacity. +/// 100 ms batches at 48 kHz ≈ 19 KB each → 50 slots ≈ 5 s / ~1 MB max backlog. +const AUDIO_QUEUE_DEPTH: usize = 50; + +/// Handle to the running STT pipeline. +/// +/// Not Clone — wrap in `Arc` to share across threads. +#[derive(Debug)] +pub struct SttPipeline { + /// Send raw PCM bytes (f32 LE, 48 kHz mono) into the pipeline. + audio_tx: SyncSender>, + /// Receive transcribed text from the pipeline. + /// Wrapped in Mutex so it can be polled from a tokio task. + pub text_rx: Mutex>, + /// Signals the worker thread to stop. + shutdown: Arc, + /// Worker thread handle — taken on drop to join cleanly. + thread: Option>, +} + +impl SttPipeline { + /// Spawn the pipeline thread. + /// + /// Returns `Err` only if the thread cannot be spawned (OS error). + /// If model files are missing, the worker logs and exits cleanly — + /// the pipeline handle is still returned but will never produce text. + pub fn new(model_dir: PathBuf) -> Result { + let (audio_tx, audio_rx) = mpsc::sync_channel::>(AUDIO_QUEUE_DEPTH); + let (text_tx, text_rx) = mpsc::channel::(); + let shutdown = Arc::new(AtomicBool::new(false)); + + let shutdown_worker = Arc::clone(&shutdown); + let handle = thread::Builder::new() + .name("stt-worker".into()) + .spawn(move || stt_worker(model_dir, audio_rx, text_tx, shutdown_worker)) + .map_err(|e| format!("failed to spawn stt-worker thread: {e}"))?; + + Ok(Self { + audio_tx, + text_rx: Mutex::new(text_rx), + shutdown, + thread: Some(handle), + }) + } + + /// Signal the worker thread to stop. + pub fn shutdown(&self) { + self.shutdown.store(true, Ordering::Release); + } + + /// Feed raw PCM bytes into the pipeline. + /// + /// Non-blocking. Drops audio silently if the pipeline can't keep up — + /// better to lose frames than to stall the UI thread. + pub fn push_audio(&self, pcm_bytes: Vec) -> Result<(), String> { + // Warn on non-4-byte-aligned input (would silently truncate in bytes_to_f32). + if pcm_bytes.len() % 4 != 0 { + eprintln!( + "sprout-desktop: push_audio_pcm received non-aligned input ({} bytes)", + pcm_bytes.len() + ); + } + // Drop audio if the pipeline can't keep up — better than blocking the UI. + let _ = self.audio_tx.try_send(pcm_bytes); + Ok(()) + } +} + +impl Drop for SttPipeline { + fn drop(&mut self) { + // Signal the worker to stop. + self.shutdown.store(true, Ordering::Release); + // Dropping `audio_tx` (implicitly when self is dropped after this fn) + // unblocks the worker's recv_timeout loop. Join to ensure clean exit. + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +// ── Worker thread ───────────────────────────────────────────────────────────── + +/// How many 16 kHz samples of silence before we flush to STT. +/// 300 ms × 16 000 Hz / 256 samples-per-frame ≈ 19 frames. +const SILENCE_FLUSH_FRAMES: usize = 19; + +/// earshot requires exactly 256 samples per frame at 16 kHz. +const VAD_FRAME_SAMPLES: usize = 256; + +/// VAD probability threshold — above this is considered speech. +const VAD_THRESHOLD: f32 = 0.5; + +/// How long the worker waits on the audio channel before checking the shutdown flag. +const RECV_TIMEOUT: Duration = Duration::from_millis(50); + +fn stt_worker( + model_dir: PathBuf, + audio_rx: Receiver>, + text_tx: mpsc::Sender, + shutdown: Arc, +) { + // ── 1. Initialise rubato resampler (48 kHz → 16 kHz, mono) ─────────────── + use rubato::{Fft, FixedSync, Resampler}; + + let mut resampler = match Fft::::new(48_000, 16_000, 1024, 2, 1, FixedSync::Input) { + Ok(r) => r, + Err(e) => { + eprintln!("sprout-desktop: STT resampler init failed: {e}"); + return; + } + }; + let chunk_in = resampler.input_frames_next(); + + // ── 2. Initialise earshot VAD ───────────────────────────────────────────── + use earshot::{DefaultPredictor, Detector}; + let mut vad = Detector::new(DefaultPredictor::new()); + + // ── 3. Initialise sherpa-onnx recognizer ───────────────────────────────── + use sherpa_onnx::{ + OfflineMoonshineModelConfig, OfflineRecognizer, OfflineRecognizerConfig, + }; + + let tokens_path = model_dir.join("tokens.txt"); + if !tokens_path.exists() { + eprintln!( + "sprout-desktop: STT models not found at {} — STT disabled", + model_dir.display() + ); + // Drain the channel so push_audio doesn't block the sender. + drain_channel(audio_rx, &shutdown); + return; + } + + let model_dir_str = model_dir.to_string_lossy().into_owned(); + + let mut cfg = OfflineRecognizerConfig::default(); + cfg.model_config.moonshine = OfflineMoonshineModelConfig { + preprocessor: Some(format!("{model_dir_str}/preprocessor.onnx")), + encoder: Some(format!("{model_dir_str}/encoder.onnx")), + uncached_decoder: None, // v1 layout only — not used with tiny int8 + cached_decoder: None, // v1 layout only — not used with tiny int8 + merged_decoder: Some(format!("{model_dir_str}/merged_decoder.onnx")), // v2 (tiny int8) + }; + cfg.model_config.tokens = Some(tokens_path.to_string_lossy().into_owned()); + cfg.model_config.num_threads = 1; + cfg.model_config.model_type = Some("moonshine".into()); + + let recognizer = match OfflineRecognizer::create(&cfg) { + Some(r) => r, + None => { + eprintln!("sprout-desktop: OfflineRecognizer::create returned None — STT disabled"); + drain_channel(audio_rx, &shutdown); + return; + } + }; + + // ── 4. Processing state ─────────────────────────────────────────────────── + // Leftover 48 kHz samples that didn't fill a full resampler chunk. + let mut input_buf_48k: Vec = Vec::with_capacity(chunk_in * 2); + // Leftover 16 kHz samples that didn't fill a full VAD frame. + let mut leftover_16k: Vec = Vec::new(); + // Accumulated speech frames (16 kHz). + let mut speech_buf: Vec = Vec::new(); + // Consecutive silence frame count. + let mut silence_frames: usize = 0; + // Whether we're currently in a speech segment. + let mut in_speech = false; + + // ── 5. Main loop ────────────────────────────────────────────────────────── + loop { + // Check shutdown flag before blocking. + if shutdown.load(Ordering::Acquire) { + break; + } + + // Use recv_timeout so we can periodically check the shutdown flag. + let bytes = match audio_rx.recv_timeout(RECV_TIMEOUT) { + Ok(b) => b, + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(mpsc::RecvTimeoutError::Disconnected) => break, // Sender dropped. + }; + + // Drain any additional pending messages to batch-process. + let mut batch = vec![bytes]; + while let Ok(b) = audio_rx.try_recv() { + batch.push(b); + } + + for bytes in batch { + // Convert raw bytes to f32 samples (little-endian). + let samples_48k = bytes_to_f32(&bytes); + input_buf_48k.extend_from_slice(&samples_48k); + + // Resample in chunk_in-sized blocks. + while input_buf_48k.len() >= chunk_in { + let chunk: Vec = input_buf_48k.drain(..chunk_in).collect(); + let resampled = resample_chunk(&mut resampler, &chunk); + process_16k_samples( + &resampled, + &mut leftover_16k, + &mut vad, + &mut speech_buf, + &mut silence_frames, + &mut in_speech, + &recognizer, + &text_tx, + ); + } + } + } + + // ── 6. Final flush ──────────────────────────────────────────────────────── + // Transcribe any speech buffered at shutdown so the last utterance isn't lost. + if !speech_buf.is_empty() { + flush_to_stt(&speech_buf, &recognizer, &text_tx); + } +} + +/// Resample a mono 48 kHz chunk to 16 kHz using rubato. +/// Returns the resampled samples (may be empty on error). +fn resample_chunk( + resampler: &mut rubato::Fft, + chunk_48k: &[f32], +) -> Vec { + use audioadapter_buffers::direct::InterleavedSlice; + use rubato::Resampler; + + // rubato expects interleaved layout even for mono. + let input = match InterleavedSlice::new(chunk_48k, 1, chunk_48k.len()) { + Ok(a) => a, + Err(e) => { + eprintln!("sprout-desktop: STT resample input error: {e}"); + return Vec::new(); + } + }; + + match resampler.process(&input, 0, None) { + Ok(out) => out.take_data(), + Err(e) => { + eprintln!("sprout-desktop: STT resample error: {e}"); + Vec::new() + } + } +} + +/// Feed 16 kHz samples through the VAD and accumulate speech. +/// Flushes to STT when silence exceeds threshold. +fn process_16k_samples( + samples: &[f32], + leftover: &mut Vec, + vad: &mut earshot::Detector, + speech_buf: &mut Vec, + silence_frames: &mut usize, + in_speech: &mut bool, + recognizer: &sherpa_onnx::OfflineRecognizer, + text_tx: &mpsc::Sender, +) { + leftover.extend_from_slice(samples); + + while leftover.len() >= VAD_FRAME_SAMPLES { + let frame: Vec = leftover.drain(..VAD_FRAME_SAMPLES).collect(); + let prob = vad.predict_f32(&frame); + let is_speech = prob > VAD_THRESHOLD; + + if is_speech { + *silence_frames = 0; + *in_speech = true; + speech_buf.extend_from_slice(&frame); + } else { + if *in_speech { + // Still accumulate during brief silence gaps. + speech_buf.extend_from_slice(&frame); + *silence_frames += 1; + + if *silence_frames >= SILENCE_FLUSH_FRAMES { + // End of utterance — transcribe. + flush_to_stt(speech_buf, recognizer, text_tx); + speech_buf.clear(); + *silence_frames = 0; + *in_speech = false; + } + } + // If not in speech, just discard the frame. + } + } +} + +/// Run sherpa-onnx on the accumulated speech buffer and send the text. +fn flush_to_stt( + speech_buf: &[f32], + recognizer: &sherpa_onnx::OfflineRecognizer, + text_tx: &mpsc::Sender, +) { + if speech_buf.is_empty() { + return; + } + + let stream = recognizer.create_stream(); + stream.accept_waveform(16_000, speech_buf); + recognizer.decode(&stream); + + let text = stream + .get_result() + .map(|r| r.text.trim().to_string()) + .unwrap_or_default(); + + if !text.is_empty() { + if let Err(e) = text_tx.send(text) { + eprintln!("sprout-desktop: STT text channel closed: {e}"); + } + } +} + +/// Convert raw bytes (f32 LE) to f32 samples. +/// Caller should ensure `bytes.len() % 4 == 0`; extra bytes are silently truncated. +fn bytes_to_f32(bytes: &[u8]) -> Vec { + bytes + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect() +} + +/// Drain and discard all pending messages on the channel until shutdown or disconnect. +fn drain_channel(rx: Receiver>, shutdown: &AtomicBool) { + loop { + if shutdown.load(Ordering::Acquire) { + break; + } + match rx.recv_timeout(Duration::from_millis(100)) { + Ok(_) => continue, + Err(_) => break, + } + } +} diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 232627c8d9..60fbd0d8d6 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -10,7 +10,10 @@ mod util; use app_state::{build_app_state, resolve_persisted_identity, AppState}; use commands::*; -use huddle::{end_huddle, get_huddle_state, join_huddle, leave_huddle, push_audio_pcm, start_huddle}; +use huddle::{ + download_voice_models, end_huddle, get_huddle_state, get_model_status, join_huddle, + leave_huddle, push_audio_pcm, start_huddle, start_stt_pipeline, +}; use managed_agents::{ ensure_nest, find_managed_agent_mut, kill_stale_tracked_processes, load_managed_agents, save_managed_agents, start_managed_agent_process, sync_managed_agent_processes, BackendKind, @@ -506,6 +509,9 @@ pub fn run() { end_huddle, get_huddle_state, push_audio_pcm, + start_stt_pipeline, + download_voice_models, + get_model_status, ]) .build(tauri::generate_context!()) .expect("error while building tauri application"); From 65aefed3d70aaa238b025ccad0ae00679c477930 Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Sat, 11 Apr 2026 13:44:38 -0400 Subject: [PATCH 03/41] =?UTF-8?q?feat(huddles):=20Phase=203=20=E2=80=94=20?= =?UTF-8?q?Agent=20Integration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agents participate in huddles via text. They hear all human speech, respond when relevant, and get interrupted by new speech. Agent Enrollment (agents.rs): - add_agent_to_huddle: dual channel add (ephemeral + parent) - Ephemeral add is required; parent add is best-effort - Returns structured AgentAddResult with parent_error detail - Only successfully enrolled agents get p-tagged on transcriptions - Voice-mode guidelines posted as kind:9 [System] message (not kind:40099) HuddleManager Integration (mod.rs): - add_agent_to_huddle Tauri command with active phase check - start_huddle tracks successful_agents (failed adds not enrolled) - Voice-mode system message posted after HUDDLE_STARTED - Agent pubkeys stored in Arc>> for live p-tag reads Agent Add UI: - '+' button on HuddleBar opens AddAgentDialog - Dialog fetches list_managed_agents, filters to running agents only - Structured error handling: hard failures shown as red, parent_error as amber warning - Dialog stays open on warning so user can see the message Design decisions: - No separate ACP process spawn needed — existing managed agent auto-subscribes via kind:9000 membership notification - ACP system prompt injection deferred to post-MVP (requires SubscriptionRule changes) - Client does NOT mint kind:40099 (relay-signed only) — uses kind:9 instead Crossfired: Codex 4/10 → 8/10 → 10/10 APPROVE after 3 rounds. --- desktop/src-tauri/src/huddle/agents.rs | 90 ++++++++++++ desktop/src-tauri/src/huddle/mod.rs | 97 +++++++++++-- desktop/src-tauri/src/lib.rs | 5 +- .../huddle/components/AddAgentDialog.tsx | 131 ++++++++++++++++++ .../features/huddle/components/HuddleBar.tsx | 40 +++++- 5 files changed, 351 insertions(+), 12 deletions(-) create mode 100644 desktop/src-tauri/src/huddle/agents.rs create mode 100644 desktop/src/features/huddle/components/AddAgentDialog.tsx diff --git a/desktop/src-tauri/src/huddle/agents.rs b/desktop/src-tauri/src/huddle/agents.rs new file mode 100644 index 0000000000..46588aefe3 --- /dev/null +++ b/desktop/src-tauri/src/huddle/agents.rs @@ -0,0 +1,90 @@ +//! Agent enrollment for huddles. +//! +//! Mental model: +//! add_agent_to_huddle → kind:9000 to ephemeral channel +//! → kind:9000 to parent channel (best-effort) +//! +//! ACP spawning is NOT needed here: the running agent process auto-subscribes +//! when it receives the kind:9000 membership notification. Huddle-specific +//! env vars (interrupt mode, custom system prompt) are a post-MVP enhancement. + +use serde::Serialize; +use uuid::Uuid; + +use crate::{app_state::AppState, events, relay::submit_event}; + +// ── Constants ───────────────────────────────────────────────────────────────── + +/// Voice-mode guidelines posted as a kind:9 message (with [System] prefix) +/// to the ephemeral channel at huddle start. Instructs agents on voice-mode +/// etiquette: TTS constraints, brevity rules, self-selection. +pub const VOICE_MODE_GUIDELINES: &str = "\ +You are in a live voice huddle. Your text is read aloud via TTS. +You will be interrupted by new messages whenever a human speaks — this is normal. + +Rules: +- Only respond if the message is relevant to you or directed at you. + If it's not for you, respond with just \".\" or stay silent. +- Keep responses under 2 sentences. This is a conversation, not an essay. +- Spell out numbers: \"eleven thirty\" not \"11:30\". +- No markdown, code blocks, or bullet lists — they sound terrible as speech. +- To share code or data, say \"I'll post that in the main channel\" and use it. +- You have access to Sprout tools — you can join channels, search messages, + and take actions. Use them proactively when asked."; + +// ── Agent enrollment ────────────────────────────────────────────────────────── + +/// Result of adding an agent to a huddle. +/// +/// `ephemeral_added` is always true on success (the function returns Err if +/// the ephemeral add fails). `parent_added` reflects whether the parent-channel +/// add succeeded; `parent_error` carries the error string when it didn't. +#[derive(Debug, Serialize)] +pub struct AgentAddResult { + /// Whether the agent was added to the ephemeral channel (required). + pub ephemeral_added: bool, + /// Whether the agent was also added to the parent channel (best-effort). + pub parent_added: bool, + /// Error from the parent-channel add, if it failed. + pub parent_error: Option, +} + +/// Add an agent to both the ephemeral and parent huddle channels. +/// +/// Returns `Err` only if the ephemeral-channel add fails (policy rejection or +/// network error). The parent-channel add is best-effort: failure is captured +/// in `AgentAddResult::parent_error` rather than propagated. +/// +/// The running ACP process for this agent will auto-subscribe to the new +/// channel when it receives the kind:9000 membership notification. +pub async fn add_agent_to_huddle( + ephemeral_channel_id: Uuid, + parent_channel_id: Uuid, + agent_pubkey: &str, + state: &AppState, +) -> Result { + // 1. Add agent to ephemeral channel (required — fail hard on rejection). + let add_eph = events::build_add_member(ephemeral_channel_id, agent_pubkey, None)?; + submit_event(add_eph, state).await?; + + // 2. Add agent to parent channel — so agent has full context. + // Best-effort: capture the error but don't propagate it. + let (parent_added, parent_error) = { + let add_parent = events::build_add_member(parent_channel_id, agent_pubkey, None)?; + match submit_event(add_parent, state).await { + Ok(_) => (true, None), + Err(e) => { + eprintln!( + "sprout-desktop: add agent to parent channel failed (may already be member): {e}" + ); + (false, Some(e)) + } + } + }; + + Ok(AgentAddResult { + ephemeral_added: true, + parent_added, + parent_error, + }) +} diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index dd049f1a97..34ba5ab2a4 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -8,6 +8,7 @@ //! //! HuddleState is stored in AppState and serialized for get_huddle_state. +pub mod agents; pub mod models; pub mod stt; @@ -338,7 +339,7 @@ pub async fn start_huddle( // channel_was_created tracks whether we need to archive on rollback. let mut channel_was_created = false; - let result: Result = async { + let result: Result<(LiveKitTokenResponse, Vec), String> = async { // 1. Create ephemeral channel. let create_builder = events::build_create_channel( ephemeral_uuid, @@ -351,11 +352,16 @@ pub async fn start_huddle( submit_event(create_builder, &state).await?; channel_was_created = true; - // 2. Add members to the ephemeral channel (best-effort). + // 2. Add members to the ephemeral channel; only keep successfully enrolled ones. + let mut successful_agents: Vec = Vec::new(); for pubkey in &member_pubkeys { let add_builder = events::build_add_member(ephemeral_uuid, pubkey, None)?; - if let Err(e) = submit_event(add_builder, &state).await { - eprintln!("sprout-desktop: huddle add_member failed for {pubkey}: {e}"); + match submit_event(add_builder, &state).await { + Ok(_) => successful_agents.push(pubkey.clone()), + Err(e) => { + eprintln!("sprout-desktop: huddle add_member failed for {pubkey}: {e}"); + // Intentionally not added — policy rejected this agent. + } } } @@ -371,12 +377,27 @@ pub async fn start_huddle( )?; submit_event(started_builder, &state).await?; - Ok(lk) + // 5. Post voice-mode guidelines as a regular kind:9 message. + // We do NOT use kind:40099 — that is relay-signed; the client must not mint it. + // Best-effort: don't fail the huddle if this fails. + if let Ok(msg_builder) = events::build_message( + ephemeral_uuid, + &format!("[System] {}", agents::VOICE_MODE_GUIDELINES), + None, + &[], + &[], + ) { + if let Err(e) = submit_event(msg_builder, &state).await { + eprintln!("sprout-desktop: voice-mode guidelines message failed: {e}"); + } + } + + Ok((lk, successful_agents)) } .await; match result { - Ok(lk) => { + Ok((lk, successful_agents)) => { // 5. Store active state. { let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; @@ -385,10 +406,9 @@ pub async fn start_huddle( hs.livekit_token = Some(lk.token.clone()); hs.livekit_url = Some(lk.url.clone()); hs.livekit_room = Some(lk.room.clone()); - // Fix 1: the UI sends agent pubkeys in member_pubkeys — store them - // separately so the transcription task can p-tag agents only. + // Only store agents that were successfully enrolled (Fix 1). *hs.agent_pubkeys.lock().unwrap_or_else(|e| e.into_inner()) = - member_pubkeys.clone(); + successful_agents.clone(); hs.participants = member_pubkeys; } @@ -691,3 +711,62 @@ pub fn get_model_status(_state: State<'_, AppState>) -> Result, +) -> Result { + let (eph_id, parent_id) = { + let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + if hs.phase != HuddlePhase::Active { + return Err("no active huddle".to_string()); + } + let eph = hs + .ephemeral_channel_id + .clone() + .ok_or("no ephemeral channel")?; + let parent = hs + .parent_channel_id + .clone() + .ok_or("no parent channel")?; + (eph, parent) + }; + + let eph_uuid = Uuid::parse_str(&eph_id).map_err(|e| e.to_string())?; + let parent_uuid = Uuid::parse_str(&parent_id).map_err(|e| e.to_string())?; + + // Returns Err only if the ephemeral add fails — parent failure is in the result. + let result = agents::add_agent_to_huddle(eph_uuid, parent_uuid, &agent_pubkey, &state).await?; + + // Ephemeral add succeeded — safe to register for p-tagging. + // Clone the Arc first so we can drop the outer HuddleState lock before + // acquiring the inner pubkeys lock (avoids the E0597 borrow-checker error). + { + let agent_pubkeys_arc = { + let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + Arc::clone(&hs.agent_pubkeys) + }; + let mut pubkeys = agent_pubkeys_arc + .lock() + .unwrap_or_else(|e| e.into_inner()); + if !pubkeys.contains(&agent_pubkey) { + pubkeys.push(agent_pubkey); + } + } + + Ok(result) +} diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 60fbd0d8d6..0764498cd0 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -11,8 +11,8 @@ mod util; use app_state::{build_app_state, resolve_persisted_identity, AppState}; use commands::*; use huddle::{ - download_voice_models, end_huddle, get_huddle_state, get_model_status, join_huddle, - leave_huddle, push_audio_pcm, start_huddle, start_stt_pipeline, + add_agent_to_huddle, download_voice_models, end_huddle, get_huddle_state, get_model_status, + join_huddle, leave_huddle, push_audio_pcm, start_huddle, start_stt_pipeline, }; use managed_agents::{ ensure_nest, find_managed_agent_mut, kill_stale_tracked_processes, load_managed_agents, @@ -512,6 +512,7 @@ pub fn run() { start_stt_pipeline, download_voice_models, get_model_status, + add_agent_to_huddle, ]) .build(tauri::generate_context!()) .expect("error while building tauri application"); diff --git a/desktop/src/features/huddle/components/AddAgentDialog.tsx b/desktop/src/features/huddle/components/AddAgentDialog.tsx new file mode 100644 index 0000000000..5b1fa935bb --- /dev/null +++ b/desktop/src/features/huddle/components/AddAgentDialog.tsx @@ -0,0 +1,131 @@ +import { invoke } from '@tauri-apps/api/core'; +import { Bot } from 'lucide-react'; +import * as React from 'react'; + +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, +} from '@/shared/ui/dialog'; + +type ManagedAgentSummary = { + pubkey: string; + name: string; + status: string; +}; + +type AgentAddResult = { + ephemeral_added: boolean; + parent_added: boolean; + parent_error: string | null; +}; + +type AddAgentDialogProps = { + onClose: () => void; + onAdd: (pubkey: string) => Promise; +}; + +export function AddAgentDialog({ onClose, onAdd }: AddAgentDialogProps) { + const [agents, setAgents] = React.useState([]); + const [loading, setLoading] = React.useState(true); + const [adding, setAdding] = React.useState(null); + const [error, setError] = React.useState(null); + const [warning, setWarning] = React.useState(null); + + React.useEffect(() => { + invoke('list_managed_agents') + .then(setAgents) + .catch((e: unknown) => { + console.error('Failed to load agents:', e); + setError('Could not load agents.'); + }) + .finally(() => setLoading(false)); + }, []); + + // Only show agents that are currently running. + const runningAgents = agents.filter((a) => a.status === 'running'); + + async function handleAdd(pubkey: string) { + if (adding) return; + setAdding(pubkey); + setError(null); + setWarning(null); + try { + const result = await onAdd(pubkey); + if (result.parent_error) { + // Agent was added to the ephemeral channel but parent channel add failed. + // Show as a warning — don't close the dialog so the user can see it. + setWarning(`Added to huddle, but parent channel failed: ${result.parent_error}`); + } else { + onClose(); + } + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e); + setError(`Failed to add agent: ${msg}`); + console.error('Failed to add agent to huddle:', e); + } finally { + setAdding(null); + } + } + + return ( + { if (!open) onClose(); }} open> + + + Add Agent to Huddle + + + {error && ( +

+ {error} +

+ )} + + {warning && ( +
+ {warning} + +
+ )} + + {loading ? ( +

+ Loading agents… +

+ ) : runningAgents.length === 0 ? ( +

+ No running agents found. +

+ ) : ( +
    + {runningAgents.map((agent) => ( +
  • + +
    • + ))} +
+ )} + + + ); +} + +export type { AgentAddResult }; diff --git a/desktop/src/features/huddle/components/HuddleBar.tsx b/desktop/src/features/huddle/components/HuddleBar.tsx index ece1ae684e..f44b6831ff 100644 --- a/desktop/src/features/huddle/components/HuddleBar.tsx +++ b/desktop/src/features/huddle/components/HuddleBar.tsx @@ -1,9 +1,10 @@ import { invoke } from '@tauri-apps/api/core'; -import { Mic, MicOff, PhoneOff, Users } from 'lucide-react'; +import { Mic, MicOff, PhoneOff, Plus, Users } from 'lucide-react'; import * as React from 'react'; import { cn } from '@/shared/lib/cn'; import { Button } from '@/shared/ui/button'; +import { AddAgentDialog, type AgentAddResult } from './AddAgentDialog'; import { ParticipantList } from './ParticipantList'; // Shape returned by the `get_huddle_state` Tauri command @@ -27,6 +28,8 @@ export function HuddleBar({ localAudioTrack, className }: HuddleBarProps) { const [state, setState] = React.useState(null); const [isMuted, setIsMuted] = React.useState(false); const [isLeaving, setIsLeaving] = React.useState(false); + const [showAddAgent, setShowAddAgent] = React.useState(false); + const [agentAddError, setAgentAddError] = React.useState(null); // Poll huddle state — replace with event listener once Rust emits events React.useEffect(() => { @@ -98,6 +101,41 @@ export function HuddleBar({ localAudioTrack, className }: HuddleBarProps) { )} + {/* Add agent button */} + + + {agentAddError && ( + + {agentAddError} + + )} + + {showAddAgent && ( + setShowAddAgent(false)} + onAdd={async (pubkey: string): Promise => { + setAgentAddError(null); + try { + return await invoke('add_agent_to_huddle', { + agentPubkey: pubkey, + }); + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e); + setAgentAddError(`Failed to add agent: ${msg}`); + throw e; // Re-throw so AddAgentDialog shows its inline error. + } + }} + /> + )} + {/* Mute toggle */} + {/* TTS toggle */} + + {/* Leave button */} {/* TTS toggle */} {/* Leave button */} diff --git a/desktop/src/features/huddle/components/ParticipantList.tsx b/desktop/src/features/huddle/components/ParticipantList.tsx index cb8b507ec8..32d2e64a53 100644 --- a/desktop/src/features/huddle/components/ParticipantList.tsx +++ b/desktop/src/features/huddle/components/ParticipantList.tsx @@ -1,4 +1,4 @@ -import { cn } from '@/shared/lib/cn'; +import { cn } from "@/shared/lib/cn"; // Legacy type kept for any callers that haven't migrated yet export type HuddleParticipant = { @@ -13,11 +13,14 @@ type ParticipantListProps = { className?: string; }; -export function ParticipantList({ participants, className }: ParticipantListProps) { +export function ParticipantList({ + participants, + className, +}: ParticipantListProps) { if (participants.length === 0) return null; return ( -
+
{participants.map((pubkey) => ( ))} @@ -35,12 +38,12 @@ function ParticipantAvatar({ pubkey }: ParticipantAvatarProps) { // Derive a stable hue from the pubkey for a distinct avatar color const hue = parseInt(pubkey.slice(0, 4), 16) % 360; - const style = { backgroundColor: `hsl(${hue}, 60%, 55%)`, color: '#fff' }; + const style = { backgroundColor: `hsl(${hue}, 60%, 55%)`, color: "#fff" }; return (
Date: Sat, 11 Apr 2026 14:35:20 -0400 Subject: [PATCH 06/41] feat(huddles): add headphones button to channel header bar Adds a Headphones icon button next to the workflow Zap button in ChannelMembersBar. Clicking starts a huddle for the current channel via the start_huddle Tauri command. --- .../channels/ui/ChannelMembersBar.tsx | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/desktop/src/features/channels/ui/ChannelMembersBar.tsx b/desktop/src/features/channels/ui/ChannelMembersBar.tsx index 5c9c779244..49abf0aa71 100644 --- a/desktop/src/features/channels/ui/ChannelMembersBar.tsx +++ b/desktop/src/features/channels/ui/ChannelMembersBar.tsx @@ -1,5 +1,6 @@ -import { Plus, Settings2, Users, Zap } from "lucide-react"; +import { Headphones, Plus, Settings2, Users, Zap } from "lucide-react"; import * as React from "react"; +import { invoke } from "@tauri-apps/api/core"; import { useAcpProvidersQuery, @@ -38,6 +39,7 @@ export function ChannelMembersBar({ }: ChannelMembersBarProps) { const [isAddBotOpen, setIsAddBotOpen] = React.useState(false); const [isCreateWorkflowOpen, setIsCreateWorkflowOpen] = React.useState(false); + const [isStartingHuddle, setIsStartingHuddle] = React.useState(false); const membersQuery = useChannelMembersQuery(channel.id); const providersQuery = useAcpProvidersQuery(); const backendProvidersQuery = useBackendProvidersQuery(); @@ -186,6 +188,32 @@ export function ChannelMembersBar({ + + - -
- { - const createdChannel = await createChannelMutation.mutateAsync({ - name, + +
+ + + +
+ { - const createdForum = await createForumMutation.mutateAsync({ - name, + }) => { + const createdChannel = await createChannelMutation.mutateAsync({ + name, + description, + channelType: "stream", + visibility, + ttlSeconds, + }); + + await goChannel(createdChannel.id); + }} + onCreateForum={async ({ description, - channelType: "forum", + name, visibility, ttlSeconds, - }); - - await goChannel(createdForum.id); - }} - onHideDm={handleHideDm} - onOpenBrowseChannels={handleOpenBrowseChannels} - onOpenBrowseForums={handleOpenBrowseForums} - onOpenDm={async ({ pubkeys }) => { - const directMessage = await openDmMutation.mutateAsync({ - pubkeys, - }); - await goChannel(directMessage.id); - }} - onOpenSearch={handleOpenSearch} - onSelectAgents={() => { - void goAgents(); - }} - onSelectChannel={(channelId) => { - void goChannel(channelId); - }} - onSelectHome={() => { - void goHome(); - }} - onSelectPulse={() => { - void goPulse(); - }} - onSelectSettings={handleOpenSettings} - onSelectWorkflows={() => { - void goWorkflows(); - }} - onSetPresenceStatus={(status) => presenceSession.setStatus(status)} - profile={profileQuery.data} - selectedChannelId={selectedChannelId} - selectedView={selectedView} - unreadChannelIds={unreadChannelIds} - /> - - - - - - { - setIsChannelManagementOpen(false); - void goHome({ replace: true }); - }} - onOpenSearchResult={handleOpenSearchResult} - onSearchOpenChange={setIsSearchOpen} - onSelectChannel={(channelId) => { - void goChannel(channelId); - }} - /> - - - - {settingsOpen ? ( - - - - ) : null} - + }) => { + const createdForum = await createForumMutation.mutateAsync({ + name, + description, + channelType: "forum", + visibility, + ttlSeconds, + }); + + await goChannel(createdForum.id); + }} + onHideDm={handleHideDm} + onOpenBrowseChannels={handleOpenBrowseChannels} + onOpenBrowseForums={handleOpenBrowseForums} + onOpenDm={async ({ pubkeys }) => { + const directMessage = await openDmMutation.mutateAsync({ + pubkeys, + }); + await goChannel(directMessage.id); + }} + onOpenSearch={handleOpenSearch} + onSelectAgents={() => { + void goAgents(); + }} + onSelectChannel={(channelId) => { + void goChannel(channelId); + }} + onSelectHome={() => { + void goHome(); + }} + onSelectPulse={() => { + void goPulse(); + }} + onSelectSettings={handleOpenSettings} + onSelectWorkflows={() => { + void goWorkflows(); + }} + onSetPresenceStatus={(status) => + presenceSession.setStatus(status) + } + profile={profileQuery.data} + selectedChannelId={selectedChannelId} + selectedView={selectedView} + unreadChannelIds={unreadChannelIds} + /> + + + + + + { + setIsChannelManagementOpen(false); + void goHome({ replace: true }); + }} + onOpenSearchResult={handleOpenSearchResult} + onSearchOpenChange={setIsSearchOpen} + onSelectChannel={(channelId) => { + void goChannel(channelId); + }} + /> + + + + {settingsOpen ? ( + + + + ) : null} + diff --git a/desktop/src/features/huddle/lib/livekit.ts b/desktop/src/features/huddle/lib/livekit.ts index 0e3818c027..4c22d02edb 100644 --- a/desktop/src/features/huddle/lib/livekit.ts +++ b/desktop/src/features/huddle/lib/livekit.ts @@ -34,12 +34,16 @@ export async function connectToHuddle( localAudioTrack: audioTrack, disconnect: async () => { room.disconnect(); - stream?.getTracks().forEach((t) => t.stop()); + stream?.getTracks().forEach((t) => { + t.stop(); + }); }, }; } catch (err) { // Clean up mic stream on any failure (getUserMedia, connect, or publish) - stream?.getTracks().forEach((t) => t.stop()); + stream?.getTracks().forEach((t) => { + t.stop(); + }); throw err; } } From f22d3d78ac2f2408f0385f8a625a73d327d04644 Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Sun, 12 Apr 2026 15:45:18 -0400 Subject: [PATCH 26/41] =?UTF-8?q?fix(huddles):=20crossfire=20hardening=20?= =?UTF-8?q?=E2=80=94=2020+=20fixes=20across=20voice=20pipeline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comprehensive quality pass on the huddles implementation, driven by iterative crossfire review (codex CLI + opus subagents, 11 rounds). ## Correctness - Fix barge-in startup order: TTS starts before STT so tts_cancel is available - Shared tts_cancel in HuddleState: survives pipeline restarts and TTS toggle - Pipeline replacement leak: shutdown old STT pipeline before replacing - Participant state: only successfully enrolled agents shown - Sentence batching: join with space, not period-space (fixes garbled TTS) - Agent prompt re-delivery: guidelines re-posted when agents join mid-huddle - TTS duplicate-start prevention: re-check before storing new pipeline - Two-phase activation: Connected → Active lifecycle with confirm_huddle_active - Agent enrollment with role=bot: relay membership API correctly identifies agents ## Voice Quality - Silence threshold: 300ms → 450ms (reduces sentence fragmentation) - Barge-in debounce: 5 consecutive VAD frames (~80ms) required during TTS - STT state reset: clean segment state across TTS transitions and cooldown - STT hot-start: auto-starts when models finish downloading mid-huddle ## Agent Identity (authoritative, fail-closed) - Relay membership API: fetch_agent_pubkeys_from_relay with role=bot filter - get_huddle_agent_pubkeys Tauri command for frontend - Backend periodic refresh in check_pipeline_hotstart (every 5s) - Frontend periodic refresh (every 10s) with fail-closed semantics - Result-based error propagation: fetch failures keep TTS mute - Joiner hydration: join_huddle fetches agent list from relay ## Frontend - Startup atomicity: ephemeralChannelId set only after full setup - EOSE-based replay boundary with timestamp belt-and-suspenders - Agent-only TTS filter: only bot-role pubkeys spoken, fail-closed - AudioWorklet fire-and-forget: no main-thread backpressure - Cleanup consolidation: single cleanupFailedStart helper - leaveHuddle returns boolean: bar stays visible if backend cleanup fails - HuddleBar respects Connected phase in poll-failure fallback ## Performance and Quality - LazyLock regex in supertonic.rs: 12 patterns compiled once - tokio::fs for all async file ops in model downloads - Model-shape validation: bounds-check dims before indexing - Async transcription task: tokio::sync::mpsc, no Tokio thread blocking - Dead code removed, stale comments fixed, rustfmt + biome applied --- desktop/scripts/check-file-sizes.mjs | 5 +- desktop/src-tauri/src/huddle/agents.rs | 4 +- desktop/src-tauri/src/huddle/mod.rs | 297 +++++++++++++++--- desktop/src-tauri/src/huddle/models.rs | 86 +++-- desktop/src-tauri/src/huddle/stt.rs | 81 +++-- desktop/src-tauri/src/huddle/supertonic.rs | 73 +++-- desktop/src-tauri/src/huddle/tts.rs | 23 +- desktop/src-tauri/src/lib.rs | 8 +- desktop/src/features/huddle/HuddleContext.tsx | 223 ++++++++----- .../features/huddle/components/HuddleBar.tsx | 34 +- .../huddle/components/ParticipantList.tsx | 7 - desktop/src/features/huddle/index.ts | 1 - .../src/features/huddle/lib/audioWorklet.ts | 22 +- 13 files changed, 625 insertions(+), 239 deletions(-) diff --git a/desktop/scripts/check-file-sizes.mjs b/desktop/scripts/check-file-sizes.mjs index d26d348878..791efe74e8 100644 --- a/desktop/scripts/check-file-sizes.mjs +++ b/desktop/scripts/check-file-sizes.mjs @@ -42,7 +42,7 @@ const overrides = new Map([ ["src/features/tokens/ui/TokenSettingsCard.tsx", 800], ["src/shared/api/relayClientSession.ts", 790], // durable websocket session manager with reconnect/replay/recovery state + sendTypingIndicator + fetchChannelHistoryBefore ["src/shared/api/tauri.ts", 1100], // remote agent provider API bindings + canvas API functions - ["src-tauri/src/lib.rs", 560], // sprout-media:// proxy + Range headers + Sprout nest init (ensure_nest) in setup() + ["src-tauri/src/lib.rs", 590], // sprout-media:// proxy + Range headers + Sprout nest init (ensure_nest) in setup() + huddle command registration ["src-tauri/src/commands/media.rs", 720], // ffmpeg video transcode + poster frame extraction + run_ffmpeg_with_timeout (find_ffmpeg, is_video_file, transcode_to_mp4, extract_poster_frame, transcode_and_extract_poster) + spawn_blocking wrappers + tests ["src-tauri/src/commands/agents.rs", 860], // remote agent lifecycle routing (local + provider branches) + scope enforcement + mcp_toolsets field; rustfmt adds line breaks around long tuple/closure blocks ["src-tauri/src/managed_agents/runtime.rs", 650], // KNOWN_AGENT_BINARIES const + process_belongs_to_us FFI (macOS proc_name + Linux /proc/comm) + terminate_process + start/stop/sync lifecycle @@ -55,6 +55,9 @@ const overrides = new Map([ ["src/features/agents/ui/CreateAgentDialog.tsx", 685], // provider selector + config form + schema-typed config coercion + required field validation + locked scopes ["src/features/channels/ui/AddChannelBotDialog.tsx", 640], // provider mode: Run on selector, trust warning, probe effect, single-agent enforcement, provider warnings display ["src/shared/api/types.ts", 535], // persona provider/model fields + forum types + workflow type re-exports + ephemeral channel TTL fields + mcpToolsets + ["src-tauri/src/huddle/mod.rs", 1200], // huddle state machine + 12 Tauri commands + STT/TTS pipeline lifecycle + relay membership fetch + two-phase activation; split planned post-MVP + ["src-tauri/src/huddle/models.rs", 650], // model download manager for Moonshine STT + Supertonic TTS with streaming downloads + atomic swap + hot-start signaling + ["src-tauri/src/huddle/supertonic.rs", 780], // Supertonic 4-ONNX-session TTS engine wrapper + Unicode text processor + LazyLock regex patterns + text chunking ]); async function walkFiles(directory) { diff --git a/desktop/src-tauri/src/huddle/agents.rs b/desktop/src-tauri/src/huddle/agents.rs index c9343ea833..4c7fe918e3 100644 --- a/desktop/src-tauri/src/huddle/agents.rs +++ b/desktop/src-tauri/src/huddle/agents.rs @@ -71,13 +71,13 @@ pub async fn add_agent_to_huddle( state: &AppState, ) -> Result { // 1. Add agent to ephemeral channel (required — fail hard on rejection). - let add_eph = events::build_add_member(ephemeral_channel_id, agent_pubkey, None)?; + let add_eph = events::build_add_member(ephemeral_channel_id, agent_pubkey, Some("bot"))?; submit_event(add_eph, state).await?; // 2. Add agent to parent channel — so agent has full context. // Best-effort: capture the error but don't propagate it. let (parent_added, parent_error) = { - let add_parent = events::build_add_member(parent_channel_id, agent_pubkey, None)?; + let add_parent = events::build_add_member(parent_channel_id, agent_pubkey, Some("bot"))?; match submit_event(add_parent, state).await { Ok(_) => (true, None), Err(e) => { diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index 946af12a14..2dfc4af9ed 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -37,6 +37,7 @@ pub enum HuddlePhase { Idle, Creating, Connecting, + Connected, // Backend ready, waiting for frontend media confirmation. Active, Leaving, } @@ -78,6 +79,11 @@ pub struct HuddleState { /// Shared with the STT pipeline for barge-in / echo gating. #[serde(skip)] pub tts_active: Arc, + /// Shared barge-in cancel flag. Set by STT when it detects speech during TTS. + /// Read by TTS to stop playback. Lives in HuddleState so it survives pipeline + /// restarts — both STT and TTS reference the same flag for the entire huddle. + #[serde(skip)] + pub tts_cancel: Arc, } fn serialize_agent_pubkeys(v: &Arc>>, s: S) -> Result @@ -121,6 +127,7 @@ impl Clone for HuddleState { tts_pipeline: None, // Never clone the pipeline handle. tts_enabled: self.tts_enabled, tts_active: Arc::clone(&self.tts_active), + tts_cancel: Arc::clone(&self.tts_cancel), } } } @@ -140,6 +147,7 @@ impl Default for HuddleState { tts_pipeline: None, tts_enabled: true, tts_active: Arc::new(AtomicBool::new(false)), + tts_cancel: Arc::new(AtomicBool::new(false)), } } } @@ -179,6 +187,48 @@ async fn fetch_livekit_token( send_json_request(request).await } +/// Fetch agent (bot-role) pubkeys from the relay's channel membership API. +/// +/// Returns `Err` on any relay/network failure so callers can distinguish a +/// successful empty list from a failed lookup. Callers that want best-effort +/// behaviour should use `.unwrap_or_default()`. +async fn fetch_agent_pubkeys_from_relay( + channel_id: &str, + state: &AppState, +) -> Result, String> { + #[derive(Deserialize)] + struct Member { + pubkey: String, + role: Option, + } + #[derive(Deserialize)] + struct MembersResponse { + members: Vec, + } + + let path = api_path(&["channels", channel_id, "members"]); + let request = match build_authed_request(&state.http_client, Method::GET, &path, state) { + Ok(r) => r, + Err(e) => { + eprintln!("sprout-desktop: fetch agent pubkeys failed (build request): {e}"); + return Err(e); + } + }; + + match send_json_request::(request).await { + Ok(resp) => Ok(resp + .members + .into_iter() + .filter(|m| m.role.as_deref() == Some("bot")) + .map(|m| m.pubkey) + .collect()), + Err(e) => { + eprintln!("sprout-desktop: fetch agent pubkeys failed: {e}"); + Err(e) + } + } +} + /// Attempt to start the STT pipeline if models are present. /// Silently skips if models are missing — huddle continues as voice-only. /// @@ -203,22 +253,25 @@ async fn maybe_start_stt_pipeline(state: &AppState, ephemeral_channel_id: &str) Arc::clone(&hs.tts_active) }; - // Grab the TTS cancel flag so STT can trigger barge-in. + // Grab the shared barge-in cancel flag from HuddleState. + // This flag lives for the entire huddle session — no stale references + // if TTS is restarted (set_tts_enabled toggle or hot-start). let tts_cancel = { let hs = match state.huddle_state.lock() { Ok(h) => h, Err(_) => return, }; - hs.tts_pipeline.as_ref().map(|p| Arc::clone(&p.cancel)) + Some(Arc::clone(&hs.tts_cancel)) }; - let pipeline = match stt::SttPipeline::new(model_dir, tts_active, tts_cancel) { - Ok(p) => Arc::new(p), + let (pipeline, text_rx) = match stt::SttPipeline::new(model_dir, tts_active, tts_cancel) { + Ok(p) => p, Err(e) => { eprintln!("sprout-desktop: STT pipeline failed to start: {e}"); return; } }; + let pipeline = Arc::new(pipeline); let channel_uuid = match parse_channel_uuid(ephemeral_channel_id) { Ok(u) => u, @@ -235,16 +288,19 @@ async fn maybe_start_stt_pipeline(state: &AppState, ephemeral_channel_id: &str) Arc::clone(&hs.agent_pubkeys) }; - // Store the pipeline. + // Shut down existing STT pipeline before replacing (prevents leaked transcription tasks). { let mut hs = match state.huddle_state.lock() { Ok(h) => h, Err(_) => return, }; + if let Some(ref old) = hs.stt_pipeline { + old.shutdown(); + } hs.stt_pipeline = Some(Arc::clone(&pipeline)); } - spawn_transcription_task(pipeline, channel_uuid, agent_pubkeys_arc, state); + spawn_transcription_task(text_rx, channel_uuid, agent_pubkeys_arc, state); } /// Attempt to start the TTS pipeline if Supertonic models are present and TTS is enabled. @@ -253,24 +309,40 @@ async fn maybe_start_tts_pipeline(state: &AppState) { if !models::is_supertonic_ready() { return; // Supertonic not downloaded yet — TTS unavailable. } + + // Don't create a duplicate pipeline if one is already running. + { + let hs = match state.huddle_state.lock() { + Ok(h) => h, + Err(_) => return, + }; + if hs.tts_pipeline.is_some() { + return; + } + } + let model_dir = match models::supertonic_model_dir() { Some(d) => d, None => return, }; - let (tts_active, tts_enabled) = { + let (tts_active, tts_enabled, tts_cancel) = { let hs = match state.huddle_state.lock() { Ok(h) => h, Err(_) => return, }; - (Arc::clone(&hs.tts_active), hs.tts_enabled) + ( + Arc::clone(&hs.tts_active), + hs.tts_enabled, + Arc::clone(&hs.tts_cancel), + ) }; if !tts_enabled { return; } - let pipeline = match tts::TtsPipeline::new(model_dir, tts_active) { + let pipeline = match tts::TtsPipeline::new(model_dir, tts_active, tts_cancel) { Ok(p) => Arc::new(p), Err(e) => { eprintln!("sprout-desktop: TTS pipeline failed to start: {e}"); @@ -283,6 +355,11 @@ async fn maybe_start_tts_pipeline(state: &AppState) { Ok(h) => h, Err(_) => return, }; + // Re-check: another call may have created a pipeline while we were building ours. + if hs.tts_pipeline.is_some() { + // Drop the one we just created — the existing one wins. + return; + } hs.tts_pipeline = Some(pipeline); } } @@ -293,9 +370,10 @@ async fn maybe_start_tts_pipeline(state: &AppState) { /// `HuddleState` — the task reads it at post time so p-tags are always /// current, not a stale snapshot. /// Fix 3: no `.unwrap()` on mutex — poisoned locks are recovered gracefully. -/// Fix 4: `recv_timeout` instead of `try_recv` + sleep — no busy-polling. +/// Fix 4: `text_rx` is a `tokio::sync::mpsc::Receiver` — fully async `.recv().await` +/// never blocks a Tokio worker thread (unlike std `recv_timeout`). fn spawn_transcription_task( - pipeline: Arc, + mut text_rx: tokio::sync::mpsc::Receiver, channel_uuid: Uuid, agent_pubkeys_arc: Arc>>, state: &AppState, @@ -308,23 +386,12 @@ fn spawn_transcription_task( let configured_api_token = state.configured_api_token.clone(); tauri::async_runtime::spawn(async move { - loop { - // Fix 3: recover from a poisoned mutex rather than panicking. - // Fix 4: recv_timeout blocks the thread efficiently; Disconnected - // means the pipeline worker has exited — stop the task. - let text = { - let rx = pipeline.text_rx.lock().unwrap_or_else(|e| e.into_inner()); - match rx.recv_timeout(std::time::Duration::from_millis(100)) { - Ok(t) => Some(t), - Err(std::sync::mpsc::RecvTimeoutError::Timeout) => None, - Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => break, - } - }; - - let t = match text { - Some(t) if !t.is_empty() => t, - _ => continue, - }; + // recv().await yields (not blocks) until text arrives or sender is dropped. + // When the STT worker exits and drops its Sender, recv() returns None → loop ends. + while let Some(t) = text_rx.recv().await { + if t.is_empty() { + continue; + } // Fix 1: read current agent pubkeys at post time. let agent_pubkeys: Vec = agent_pubkeys_arc @@ -427,7 +494,7 @@ pub async fn start_huddle( // 2. Add members to the ephemeral channel; only keep successfully enrolled ones. let mut successful_agents: Vec = Vec::new(); for pubkey in &member_pubkeys { - let add_builder = events::build_add_member(ephemeral_uuid, pubkey, None)?; + let add_builder = events::build_add_member(ephemeral_uuid, pubkey, Some("bot"))?; match submit_event(add_builder, &state).await { Ok(_) => successful_agents.push(pubkey.clone()), Err(e) => { @@ -471,7 +538,7 @@ pub async fn start_huddle( // 5. Store active state. { let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; - hs.phase = HuddlePhase::Active; + hs.phase = HuddlePhase::Connected; hs.ephemeral_channel_id = Some(ephemeral_channel_id.clone()); hs.livekit_token = Some(lk.token.clone()); hs.livekit_url = Some(lk.url.clone()); @@ -479,13 +546,15 @@ pub async fn start_huddle( // Only store agents that were successfully enrolled (Fix 1). *hs.agent_pubkeys.lock().unwrap_or_else(|e| e.into_inner()) = successful_agents.clone(); - // Include the current user + any invited members as participants. + // Include the current user + successfully enrolled agents as participants. + // Use successful_agents (not member_pubkeys) so failed enrollments + // are not reflected in the participant list. let own_pubkey = state .keys .lock() .map(|k| k.public_key().to_hex()) .unwrap_or_default(); - let mut participants = member_pubkeys; + let mut participants = successful_agents.clone(); if !own_pubkey.is_empty() && !participants.contains(&own_pubkey) { participants.insert(0, own_pubkey); } @@ -498,9 +567,11 @@ pub async fn start_huddle( mgr.start_supertonic_download(state.http_client.clone()); } - // 7. Auto-start STT and TTS pipelines if models are already ready. - maybe_start_stt_pipeline(&state, &ephemeral_channel_id).await; + // 7. Auto-start TTS first, then STT. STT captures `tts_cancel` from + // `hs.tts_pipeline` at init time — TTS must exist before STT starts + // or barge-in never works. maybe_start_tts_pipeline(&state).await; + maybe_start_stt_pipeline(&state, &ephemeral_channel_id).await; Ok(HuddleJoinInfo { ephemeral_channel_id, @@ -580,22 +651,45 @@ pub async fn join_huddle( // 3. Store active state. { let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; - hs.phase = HuddlePhase::Active; + hs.phase = HuddlePhase::Connected; hs.livekit_token = Some(lk.token.clone()); hs.livekit_url = Some(lk.url.clone()); hs.livekit_room = Some(lk.room.clone()); - // Note: agent_pubkeys stays empty for joiners — agents were added by the creator. + // agent_pubkeys is hydrated after this block via fetch_agent_pubkeys_from_relay. + + // Include at least the current user in the participant list. + // Full participant sync from relay membership is a post-MVP enhancement. + let own_pubkey = state + .keys + .lock() + .map(|k| k.public_key().to_hex()) + .unwrap_or_default(); + if !own_pubkey.is_empty() { + hs.participants = vec![own_pubkey]; + } + } + + // 4. Hydrate agent_pubkeys from relay membership so joiners can: + // (a) p-tag agents on STT transcripts, and + // (b) filter agent messages for TTS on the frontend. + // Must happen before maybe_start_stt_pipeline — the transcription task reads agent_pubkeys. + // Best-effort for joiners — don't fail the join on a transient fetch error. + // On Ok: always write (even empty — huddle may have no agents yet). + // On Err: leave default empty list (periodic refresh will retry). + if let Ok(agents) = fetch_agent_pubkeys_from_relay(&ephemeral_channel_id, &state).await { + let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + *hs.agent_pubkeys.lock().unwrap_or_else(|e| e.into_inner()) = agents; } - // 4. Ensure voice models are downloading (idempotent). + // 5. Ensure voice models are downloading (idempotent). if let Some(mgr) = models::global_model_manager() { mgr.start_moonshine_download(state.http_client.clone()); mgr.start_supertonic_download(state.http_client.clone()); } - // 5. Auto-start STT and TTS pipelines if models are already ready. - maybe_start_stt_pipeline(&state, &ephemeral_channel_id).await; + // 6. Auto-start TTS first, then STT (same ordering rationale as start_huddle). maybe_start_tts_pipeline(&state).await; + maybe_start_stt_pipeline(&state, &ephemeral_channel_id).await; Ok(HuddleJoinInfo { ephemeral_channel_id, @@ -720,6 +814,21 @@ pub async fn end_huddle(state: State<'_, AppState>) -> Result<(), String> { Ok(()) } +/// Confirm that the frontend has established LiveKit + AudioWorklet. +/// Transitions from Connected → Active. No-op if already Active. +#[tauri::command] +pub async fn confirm_huddle_active(state: State<'_, AppState>) -> Result<(), String> { + let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + match hs.phase { + HuddlePhase::Connected => { + hs.phase = HuddlePhase::Active; + Ok(()) + } + HuddlePhase::Active => Ok(()), // Already active — idempotent. + ref other => Err(format!("cannot confirm active: phase is {:?}", other)), + } +} + /// Return the current HuddleState (serialized for the frontend). #[tauri::command] pub fn get_huddle_state(state: State<'_, AppState>) -> Result { @@ -727,6 +836,24 @@ pub fn get_huddle_state(state: State<'_, AppState>) -> Result) -> Result, String> { + let eph_id = { + let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + hs.ephemeral_channel_id.clone() + }; + match eph_id { + Some(id) => fetch_agent_pubkeys_from_relay(&id, &state).await, + None => Ok(Vec::new()), + } +} + /// Receive raw PCM audio bytes from the AudioWorklet and feed the STT pipeline. /// /// Expects a raw binary body of f32 LE samples at 48 kHz mono. @@ -749,6 +876,61 @@ pub fn push_audio_pcm( } } +/// Hot-start: check if voice models just finished downloading during an active +/// huddle and start the corresponding pipelines. +/// +/// Called by the frontend on a timer or after model status changes. No-op if +/// the huddle is not active or pipelines are already running. +#[tauri::command] +pub async fn check_pipeline_hotstart(state: State<'_, AppState>) -> Result<(), String> { + let (is_active, has_stt, has_tts, ephemeral_channel_id) = { + let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + ( + matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active), + hs.stt_pipeline.is_some(), + hs.tts_pipeline.is_some(), + hs.ephemeral_channel_id.clone(), + ) + }; + + if !is_active { + return Ok(()); + } + + // Check if models just became ready (one-shot flags). + let moonshine_ready = models::global_model_manager() + .map(|m| m.take_moonshine_ready()) + .unwrap_or(false); + let supertonic_ready = models::global_model_manager() + .map(|m| m.take_supertonic_ready()) + .unwrap_or(false); + + // Start TTS first (so STT can capture tts_cancel). + if !has_tts && (supertonic_ready || models::is_supertonic_ready()) { + maybe_start_tts_pipeline(&state).await; + } + + if !has_stt && (moonshine_ready || models::is_moonshine_ready()) { + if let Some(eph_id) = &ephemeral_channel_id { + maybe_start_stt_pipeline(&state, eph_id).await; + } + } + + // Periodically refresh agent_pubkeys from relay membership. + // This catches mid-huddle agent additions/removals by other participants, + // keeping STT p-tags authoritative throughout the session. + // On Ok: always replace (even with empty — agents may have been removed). + // On Err: preserve the existing list (transient failure shouldn't zero it). + if let Some(eph_id) = &ephemeral_channel_id { + if let Ok(fresh_agents) = fetch_agent_pubkeys_from_relay(eph_id, &state).await { + let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + *hs.agent_pubkeys.lock().unwrap_or_else(|e| e.into_inner()) = fresh_agents; + } + } + + Ok(()) +} + /// Start the STT pipeline for the active huddle. /// /// Creates the pipeline, stores it in HuddleState, and spawns a tokio task @@ -779,18 +961,22 @@ pub async fn start_stt_pipeline(state: State<'_, AppState>) -> Result<(), String let (tts_active, tts_cancel) = { let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; - let cancel = hs.tts_pipeline.as_ref().map(|p| Arc::clone(&p.cancel)); - (Arc::clone(&hs.tts_active), cancel) + // Read from HuddleState.tts_cancel — stable for the entire huddle session. + (Arc::clone(&hs.tts_active), Some(Arc::clone(&hs.tts_cancel))) }; - let pipeline = Arc::new(stt::SttPipeline::new(model_dir, tts_active, tts_cancel)?); + let (pipeline, text_rx) = stt::SttPipeline::new(model_dir, tts_active, tts_cancel)?; + let pipeline = Arc::new(pipeline); { let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + if let Some(ref old) = hs.stt_pipeline { + old.shutdown(); + } hs.stt_pipeline = Some(Arc::clone(&pipeline)); } - spawn_transcription_task(pipeline, channel_uuid, agent_pubkeys_arc, &state); + spawn_transcription_task(text_rx, channel_uuid, agent_pubkeys_arc, &state); Ok(()) } @@ -822,7 +1008,7 @@ pub fn get_model_status(_state: State<'_, AppState>) -> Result) -> Result<(), String> { { @@ -843,7 +1029,7 @@ pub async fn set_tts_enabled(enabled: bool, state: State<'_, AppState>) -> Resul let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; hs.phase.clone() }; - if phase == HuddlePhase::Active { + if matches!(phase, HuddlePhase::Connected | HuddlePhase::Active) { maybe_start_tts_pipeline(&state).await; } } @@ -862,7 +1048,9 @@ pub async fn set_tts_enabled(enabled: bool, state: State<'_, AppState>) -> Resul pub async fn speak_agent_message(text: String, state: State<'_, AppState>) -> Result<(), String> { let needs_pipeline = { let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; - hs.tts_enabled && hs.tts_pipeline.is_none() && hs.phase == HuddlePhase::Active + hs.tts_enabled + && hs.tts_pipeline.is_none() + && matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active) }; // Lazy-start: models may have finished downloading after the huddle began. @@ -899,7 +1087,7 @@ pub async fn add_agent_to_huddle( ) -> Result { let (eph_id, parent_id) = { let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; - if hs.phase != HuddlePhase::Active { + if !matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active) { return Err("no active huddle".to_string()); } let eph = hs @@ -930,6 +1118,21 @@ pub async fn add_agent_to_huddle( } } + // Re-post voice-mode guidelines so the newly-added agent sees them. + // The agent auto-subscribes via membership notification; by the time it + // processes the subscription, this message will be in the channel history. + { + let (eph_uuid, parent_id_for_guidelines) = (eph_uuid, parent_id.clone()); + let guidelines = agents::voice_mode_guidelines(&parent_id_for_guidelines); + if let Ok(msg_builder) = + events::build_message(eph_uuid, &format!("[System] {guidelines}"), None, &[], &[]) + { + if let Err(e) = submit_event(msg_builder, &state).await { + eprintln!("sprout-desktop: voice-mode guidelines re-post for agent failed: {e}"); + } + } + } + // Also add the agent to the visible participants list. { let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; diff --git a/desktop/src-tauri/src/huddle/models.rs b/desktop/src-tauri/src/huddle/models.rs index fab4219e08..adc7957b99 100644 --- a/desktop/src-tauri/src/huddle/models.rs +++ b/desktop/src-tauri/src/huddle/models.rs @@ -9,8 +9,8 @@ //! Models are downloaded once and cached. No versioning in MVP — presence of //! all expected files is sufficient to consider the model ready. -use std::fs; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; use serde::{Deserialize, Serialize}; @@ -107,6 +107,11 @@ pub struct ModelManager { models_dir: PathBuf, moonshine_status: Arc>, supertonic_status: Arc>, + /// Set to `true` when Moonshine download completes during an active huddle. + /// Polled by the huddle system to auto-start STT. + moonshine_just_ready: Arc, + /// Set to `true` when Supertonic download completes during an active huddle. + supertonic_just_ready: Arc, } impl ModelManager { @@ -119,6 +124,8 @@ impl ModelManager { models_dir, moonshine_status: Arc::new(Mutex::new(ModelStatus::NotDownloaded)), supertonic_status: Arc::new(Mutex::new(ModelStatus::NotDownloaded)), + moonshine_just_ready: Arc::new(AtomicBool::new(false)), + supertonic_just_ready: Arc::new(AtomicBool::new(false)), }) } @@ -149,6 +156,11 @@ impl ModelManager { .clone() } + /// Returns true (once) if Moonshine just became ready. Resets the flag. + pub fn take_moonshine_ready(&self) -> bool { + self.moonshine_just_ready.swap(false, Ordering::AcqRel) + } + // ── Supertonic ──────────────────────────────────────────────────────────── /// Returns the path to the Supertonic model directory, or `None` if not ready. @@ -176,6 +188,11 @@ impl ModelManager { .clone() } + /// Returns true (once) if Supertonic just became ready. Resets the flag. + pub fn take_supertonic_ready(&self) -> bool { + self.supertonic_just_ready.swap(false, Ordering::AcqRel) + } + /// Trigger a background download of the Supertonic TTS model (~253 MB total). /// /// Returns immediately. Progress is tracked via `supertonic_status()`. @@ -272,7 +289,9 @@ impl ModelManager { /// Download, extract, and verify the Moonshine model archive. async fn download_moonshine_model(&self, http_client: reqwest::Client) -> Result<(), String> { - fs::create_dir_all(&self.models_dir).map_err(|e| format!("create models dir: {e}"))?; + tokio::fs::create_dir_all(&self.models_dir) + .await + .map_err(|e| format!("create models dir: {e}"))?; self.set_moonshine_status(ModelStatus::Downloading { progress_percent: 0, @@ -336,16 +355,24 @@ impl ModelManager { let final_dir = self.models_dir.join(MOONSHINE_MODEL_DIR_NAME); if temp_dir.exists() { - fs::remove_dir_all(&temp_dir).map_err(|e| format!("remove stale temp dir: {e}"))?; + tokio::fs::remove_dir_all(&temp_dir) + .await + .map_err(|e| format!("remove stale temp dir: {e}"))?; } - fs::create_dir_all(&temp_dir).map_err(|e| format!("create temp dir: {e}"))?; + tokio::fs::create_dir_all(&temp_dir) + .await + .map_err(|e| format!("create temp dir: {e}"))?; eprintln!("sprout-desktop: extracting Moonshine archive…"); - extract_archive(&archive_path, &temp_dir)?; + let archive_path_clone = archive_path.clone(); + let temp_dir_clone = temp_dir.clone(); + tokio::task::spawn_blocking(move || extract_archive(&archive_path_clone, &temp_dir_clone)) + .await + .map_err(|e| format!("tar task panicked: {e}"))??; let extracted_subdir = temp_dir.join(MOONSHINE_ARCHIVE_SUBDIR); if !extracted_subdir.is_dir() { - let _ = fs::remove_dir_all(&temp_dir); + let _ = tokio::fs::remove_dir_all(&temp_dir).await; return Err(format!( "expected subdir '{}' not found after extraction", MOONSHINE_ARCHIVE_SUBDIR, @@ -359,7 +386,7 @@ impl ModelManager { .collect(); if !missing.is_empty() { - let _ = fs::remove_dir_all(&temp_dir); + let _ = tokio::fs::remove_dir_all(&temp_dir).await; return Err(format!( "model verification failed — missing: {}", missing.join(", "), @@ -369,21 +396,23 @@ impl ModelManager { let backup_dir = final_dir.with_extension("old"); if final_dir.exists() { if backup_dir.exists() { - let _ = fs::remove_dir_all(&backup_dir); + let _ = tokio::fs::remove_dir_all(&backup_dir).await; } - fs::rename(&final_dir, &backup_dir).map_err(|e| format!("backup old model: {e}"))?; + tokio::fs::rename(&final_dir, &backup_dir) + .await + .map_err(|e| format!("backup old model: {e}"))?; } - if let Err(e) = fs::rename(&extracted_subdir, &final_dir) { + if let Err(e) = tokio::fs::rename(&extracted_subdir, &final_dir).await { if backup_dir.exists() { - let _ = fs::rename(&backup_dir, &final_dir); + let _ = tokio::fs::rename(&backup_dir, &final_dir).await; } return Err(format!("install new model: {e}")); } - let _ = fs::remove_dir_all(&backup_dir); - let _ = fs::remove_dir_all(&temp_dir); - let _ = fs::remove_file(&archive_path); + let _ = tokio::fs::remove_dir_all(&backup_dir).await; + let _ = tokio::fs::remove_dir_all(&temp_dir).await; + let _ = tokio::fs::remove_file(&archive_path).await; eprintln!( "sprout-desktop: Moonshine model ready at {}", @@ -391,6 +420,7 @@ impl ModelManager { ); self.set_moonshine_status(ModelStatus::Ready); + self.moonshine_just_ready.store(true, Ordering::Release); Ok(()) } @@ -399,7 +429,9 @@ impl ModelManager { /// Downloads 7 files into `~/.sprout/models/supertonic/`. /// Files are written to a temp directory first, then moved atomically. async fn download_supertonic_model(&self, http_client: reqwest::Client) -> Result<(), String> { - fs::create_dir_all(&self.models_dir).map_err(|e| format!("create models dir: {e}"))?; + tokio::fs::create_dir_all(&self.models_dir) + .await + .map_err(|e| format!("create models dir: {e}"))?; self.set_supertonic_status(ModelStatus::Downloading { progress_percent: 0, @@ -409,9 +441,13 @@ impl ModelManager { let temp_dir = self.models_dir.join("supertonic.tmp"); if temp_dir.exists() { - fs::remove_dir_all(&temp_dir).map_err(|e| format!("remove stale temp dir: {e}"))?; + tokio::fs::remove_dir_all(&temp_dir) + .await + .map_err(|e| format!("remove stale temp dir: {e}"))?; } - fs::create_dir_all(&temp_dir).map_err(|e| format!("create temp dir: {e}"))?; + tokio::fs::create_dir_all(&temp_dir) + .await + .map_err(|e| format!("create temp dir: {e}"))?; // (url_suffix, local_filename) let downloads: &[(&str, &str)] = &[ @@ -437,7 +473,7 @@ impl ModelManager { .map_err(|e| format!("download {filename} request failed: {e}"))?; if !response.status().is_success() { - let _ = fs::remove_dir_all(&temp_dir); + let _ = tokio::fs::remove_dir_all(&temp_dir).await; return Err(format!( "download {filename} HTTP {}: {}", response.status().as_u16(), @@ -487,7 +523,7 @@ impl ModelManager { .collect(); if !missing.is_empty() { - let _ = fs::remove_dir_all(&temp_dir); + let _ = tokio::fs::remove_dir_all(&temp_dir).await; return Err(format!( "supertonic model verification failed — missing: {}", missing.join(", "), @@ -498,20 +534,21 @@ impl ModelManager { let backup_dir = final_dir.with_extension("old"); if final_dir.exists() { if backup_dir.exists() { - let _ = fs::remove_dir_all(&backup_dir); + let _ = tokio::fs::remove_dir_all(&backup_dir).await; } - fs::rename(&final_dir, &backup_dir) + tokio::fs::rename(&final_dir, &backup_dir) + .await .map_err(|e| format!("backup old supertonic model: {e}"))?; } - if let Err(e) = fs::rename(&temp_dir, &final_dir) { + if let Err(e) = tokio::fs::rename(&temp_dir, &final_dir).await { if backup_dir.exists() { - let _ = fs::rename(&backup_dir, &final_dir); + let _ = tokio::fs::rename(&backup_dir, &final_dir).await; } return Err(format!("install new supertonic model: {e}")); } - let _ = fs::remove_dir_all(&backup_dir); + let _ = tokio::fs::remove_dir_all(&backup_dir).await; eprintln!( "sprout-desktop: Supertonic model ready at {}", @@ -519,6 +556,7 @@ impl ModelManager { ); self.set_supertonic_status(ModelStatus::Ready); + self.supertonic_just_ready.store(true, Ordering::Release); Ok(()) } } diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index ab9bf88095..d3dcfaf038 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -23,12 +23,14 @@ use std::{ sync::{ atomic::{AtomicBool, Ordering}, mpsc::{self, Receiver, SyncSender}, - Arc, Mutex, + Arc, }, thread, time::Duration, }; +use tokio::sync::mpsc as tokio_mpsc; + // ── Public pipeline handle ──────────────────────────────────────────────────── /// Bounded audio queue capacity. @@ -38,13 +40,14 @@ const AUDIO_QUEUE_DEPTH: usize = 50; /// Handle to the running STT pipeline. /// /// Not Clone — wrap in `Arc` to share across threads. +/// +/// The text receiver (`tokio::sync::mpsc::Receiver`) is returned +/// separately from `new()` so the caller can move it directly into an async +/// task without holding a Mutex across await points. #[derive(Debug)] pub struct SttPipeline { /// Send raw PCM bytes (f32 LE, 48 kHz mono) into the pipeline. audio_tx: SyncSender>, - /// Receive transcribed text from the pipeline. - /// Wrapped in Mutex so it can be polled from a tokio task. - pub text_rx: Mutex>, /// Signals the worker thread to stop. shutdown: Arc, /// Worker thread handle — taken on drop to join cleanly. @@ -73,13 +76,18 @@ impl SttPipeline { /// Returns `Err` only if the thread cannot be spawned (OS error). /// If model files are missing, the worker logs and exits cleanly — /// the pipeline handle is still returned but will never produce text. + /// + /// The `tokio::sync::mpsc::Receiver` is returned separately so the + /// caller can move it directly into an async task. This avoids holding a + /// `Mutex` across await points (which would block a Tokio worker + /// thread on every `recv_timeout` call). pub fn new( model_dir: PathBuf, tts_active: Arc, tts_cancel: Option>, - ) -> Result { + ) -> Result<(Self, tokio_mpsc::Receiver), String> { let (audio_tx, audio_rx) = mpsc::sync_channel::>(AUDIO_QUEUE_DEPTH); - let (text_tx, text_rx) = mpsc::channel::(); + let (text_tx, text_rx) = tokio_mpsc::channel::(64); let shutdown = Arc::new(AtomicBool::new(false)); let shutdown_worker = Arc::clone(&shutdown); @@ -98,13 +106,13 @@ impl SttPipeline { }) .map_err(|e| format!("failed to spawn stt-worker thread: {e}"))?; - Ok(Self { + let pipeline = Self { audio_tx, - text_rx: Mutex::new(text_rx), shutdown, thread: Some(handle), tts_cancel, - }) + }; + Ok((pipeline, text_rx)) } /// Signal the worker thread to stop. @@ -145,8 +153,12 @@ impl Drop for SttPipeline { // ── Worker thread ───────────────────────────────────────────────────────────── /// How many 16 kHz samples of silence before we flush to STT. -/// 300 ms × 16 000 Hz / 256 samples-per-frame ≈ 19 frames. -const SILENCE_FLUSH_FRAMES: usize = 19; +/// 450 ms × 16 000 Hz / 256 samples-per-frame ≈ 28 frames. +const SILENCE_FLUSH_FRAMES: usize = 28; + +/// Consecutive VAD speech frames required before triggering barge-in during TTS. +/// 5 frames × 256 samples / 16 kHz ≈ 80 ms — filters out coughs and transients. +const BARGE_IN_DEBOUNCE_FRAMES: usize = 5; /// earshot requires exactly 256 samples per frame at 16 kHz. const VAD_FRAME_SAMPLES: usize = 256; @@ -164,7 +176,7 @@ const TTS_COOLDOWN: Duration = Duration::from_millis(200); fn stt_worker( model_dir: PathBuf, audio_rx: Receiver>, - text_tx: mpsc::Sender, + text_tx: tokio_mpsc::Sender, shutdown: Arc, tts_active: Arc, tts_cancel: Option>, @@ -233,6 +245,8 @@ fn stt_worker( let mut silence_frames: usize = 0; // Whether we're currently in a speech segment. let mut in_speech = false; + // Consecutive speech frames seen during TTS — used for barge-in debounce. + let mut barge_in_frames: usize = 0; // Timestamp when TTS last stopped — used for the 200 ms cooldown. let mut tts_stopped_at: Option = None; @@ -281,6 +295,7 @@ fn stt_worker( &mut speech_buf, &mut silence_frames, &mut in_speech, + &mut barge_in_frames, &recognizer, &text_tx, &tts_active, @@ -337,8 +352,9 @@ fn process_16k_samples( speech_buf: &mut Vec, silence_frames: &mut usize, in_speech: &mut bool, + barge_in_frames: &mut usize, recognizer: &sherpa_onnx::OfflineRecognizer, - text_tx: &mpsc::Sender, + text_tx: &tokio_mpsc::Sender, tts_active: &Arc, tts_cancel: Option<&AtomicBool>, tts_stopped_at: &mut Option, @@ -354,17 +370,26 @@ fn process_16k_samples( // Fix 2 + Fix 4: While TTS is playing, detect barge-in but skip accumulation. if tts_playing { - if is_speech && !*in_speech { - // Speech onset during TTS → barge-in: cancel TTS immediately. - *in_speech = true; - if let Some(cancel) = tts_cancel { - cancel.store(true, Ordering::Release); + // Reset segment state — STT is not tracking speech during TTS. + // The barge-in logic below will set in_speech=true only when the + // debounce threshold is met, preventing stale continuation after TTS stops. + *in_speech = false; + + if is_speech { + *barge_in_frames += 1; + if *barge_in_frames >= BARGE_IN_DEBOUNCE_FRAMES { + // Sustained speech during TTS → barge-in: cancel TTS. + *in_speech = true; + if let Some(cancel) = tts_cancel { + cancel.store(true, Ordering::Release); + } } - } else if !is_speech { - *in_speech = false; + } else { + *barge_in_frames = 0; } - // Don't accumulate — skip to next frame (echo prevention). + // Don't accumulate during TTS — clean slate for when TTS stops. speech_buf.clear(); + *silence_frames = 0; continue; } @@ -376,10 +401,15 @@ fn process_16k_samples( *in_speech = false; } speech_buf.clear(); + *silence_frames = 0; + *barge_in_frames = 0; continue; } else { - // Cooldown expired — clear the timer. + // Cooldown expired — clear the timer and reset all segment state. *tts_stopped_at = None; + *in_speech = false; + *silence_frames = 0; + *barge_in_frames = 0; } } @@ -405,10 +435,13 @@ fn process_16k_samples( } /// Run sherpa-onnx on the accumulated speech buffer and send the text. +/// +/// Uses `blocking_send` because this runs on a `std::thread` (not async). +/// The tokio channel's `blocking_send` is safe to call from sync contexts. fn flush_to_stt( speech_buf: &[f32], recognizer: &sherpa_onnx::OfflineRecognizer, - text_tx: &mpsc::Sender, + text_tx: &tokio_mpsc::Sender, ) { if speech_buf.is_empty() { return; @@ -424,7 +457,7 @@ fn flush_to_stt( .unwrap_or_default(); if !text.is_empty() { - if let Err(e) = text_tx.send(text) { + if let Err(e) = text_tx.blocking_send(text) { eprintln!("sprout-desktop: STT text channel closed: {e}"); } } diff --git a/desktop/src-tauri/src/huddle/supertonic.rs b/desktop/src-tauri/src/huddle/supertonic.rs index 96e288775f..facc6318a8 100644 --- a/desktop/src-tauri/src/huddle/supertonic.rs +++ b/desktop/src-tauri/src/huddle/supertonic.rs @@ -14,6 +14,7 @@ use serde::{Deserialize, Serialize}; use std::fs::File; use std::io::BufReader; use std::path::Path; +use std::sync::LazyLock; use unicode_normalization::UnicodeNormalization; use ort::{session::Session, value::Value}; @@ -86,6 +87,20 @@ pub(crate) fn load_voice_style>(path: P) -> Result let ttl_dims = &data.style_ttl.dims; let dp_dims = &data.style_dp.dims; + // Validate dimensions — model JSON must have [batch, dim1, dim2] shape. + if ttl_dims.len() < 3 { + return Err(format!( + "voice style ttl dims too short: expected 3, got {}", + ttl_dims.len() + )); + } + if dp_dims.len() < 3 { + return Err(format!( + "voice style dp dims too short: expected 3, got {}", + dp_dims.len() + )); + } + // dims = [1, dim1, dim2] — batch dimension is always 1 for a single voice. let (ttl_d1, ttl_d2) = (ttl_dims[1], ttl_dims[2]); let (dp_d1, dp_d2) = (dp_dims[1], dp_dims[2]); @@ -161,16 +176,33 @@ impl UnicodeProcessor { } } +// ── Compiled regex patterns (one-time init) ─────────────────────────────── +static RE_EMOJI: LazyLock = LazyLock::new(|| { + Regex::new( + r"[\x{1F600}-\x{1F64F}\x{1F300}-\x{1F5FF}\x{1F680}-\x{1F6FF}\x{1F700}-\x{1F77F}\x{1F780}-\x{1F7FF}\x{1F800}-\x{1F8FF}\x{1F900}-\x{1F9FF}\x{1FA00}-\x{1FA6F}\x{1FA70}-\x{1FAFF}\x{2600}-\x{26FF}\x{2700}-\x{27BF}\x{1F1E6}-\x{1F1FF}]+" + ).unwrap() +}); + +static RE_SPACE_COMMA: LazyLock = LazyLock::new(|| Regex::new(r" ,").unwrap()); +static RE_SPACE_DOT: LazyLock = LazyLock::new(|| Regex::new(r" \.").unwrap()); +static RE_SPACE_BANG: LazyLock = LazyLock::new(|| Regex::new(r" !").unwrap()); +static RE_SPACE_QUESTION: LazyLock = LazyLock::new(|| Regex::new(r" \?").unwrap()); +static RE_SPACE_SEMI: LazyLock = LazyLock::new(|| Regex::new(r" ;").unwrap()); +static RE_SPACE_COLON: LazyLock = LazyLock::new(|| Regex::new(r" :").unwrap()); +static RE_SPACE_APOS: LazyLock = LazyLock::new(|| Regex::new(r" '").unwrap()); +static RE_WHITESPACE: LazyLock = LazyLock::new(|| Regex::new(r"\s+").unwrap()); +static RE_ENDS_PUNC: LazyLock = + LazyLock::new(|| Regex::new(r#"[.!?;:,'")\]}…。」』】〉》›»]$"#).unwrap()); +static RE_PARAGRAPH: LazyLock = LazyLock::new(|| Regex::new(r"\n\s*\n").unwrap()); +static RE_SENTENCE_SPLIT: LazyLock = LazyLock::new(|| Regex::new(r"([.!?])\s+").unwrap()); + // ── Text preprocessing ──────────────────────────────────────────────────────── fn preprocess_text(text: &str, lang: &str) -> Result { let mut s: String = text.nfkd().collect(); // Strip emojis. - let emoji_re = Regex::new( - r"[\x{1F600}-\x{1F64F}\x{1F300}-\x{1F5FF}\x{1F680}-\x{1F6FF}\x{1F700}-\x{1F77F}\x{1F780}-\x{1F7FF}\x{1F800}-\x{1F8FF}\x{1F900}-\x{1F9FF}\x{1FA00}-\x{1FA6F}\x{1FA70}-\x{1FAFF}\x{2600}-\x{26FF}\x{2700}-\x{27BF}\x{1F1E6}-\x{1F1FF}]+" - ).unwrap(); - s = emoji_re.replace_all(&s, "").to_string(); + s = RE_EMOJI.replace_all(&s, "").to_string(); // Character replacements. for (from, to) in &[ @@ -208,17 +240,13 @@ fn preprocess_text(text: &str, lang: &str) -> Result { } // Fix spacing around punctuation. - for (pat, rep) in &[ - (r" ,", ","), - (r" \.", "."), - (r" !", "!"), - (r" \?", "?"), - (r" ;", ";"), - (r" :", ":"), - (r" '", "'"), - ] { - s = Regex::new(pat).unwrap().replace_all(&s, *rep).to_string(); - } + s = RE_SPACE_COMMA.replace_all(&s, ",").to_string(); + s = RE_SPACE_DOT.replace_all(&s, ".").to_string(); + s = RE_SPACE_BANG.replace_all(&s, "!").to_string(); + s = RE_SPACE_QUESTION.replace_all(&s, "?").to_string(); + s = RE_SPACE_SEMI.replace_all(&s, ";").to_string(); + s = RE_SPACE_COLON.replace_all(&s, ":").to_string(); + s = RE_SPACE_APOS.replace_all(&s, "'").to_string(); // Collapse duplicate quote pairs. while s.contains("\"\"") { @@ -232,15 +260,12 @@ fn preprocess_text(text: &str, lang: &str) -> Result { } // Collapse whitespace. - s = Regex::new(r"\s+").unwrap().replace_all(&s, " ").to_string(); + s = RE_WHITESPACE.replace_all(&s, " ").to_string(); s = s.trim().to_string(); // Ensure terminal punctuation. - if !s.is_empty() { - let ends_re = Regex::new(r#"[.!?;:,'")\]}…。」』】〉》›»]$"#).unwrap(); - if !ends_re.is_match(&s) { - s.push('.'); - } + if !s.is_empty() && !RE_ENDS_PUNC.is_match(&s) { + s.push('.'); } if !AVAILABLE_LANGS.contains(&lang) { @@ -337,10 +362,9 @@ pub(crate) fn chunk_text(text: &str, max_len: Option) -> Vec { return vec![String::new()]; } - let para_re = Regex::new(r"\n\s*\n").unwrap(); let mut chunks: Vec = Vec::new(); - for para in para_re.split(text) { + for para in RE_PARAGRAPH.split(text) { let para = para.trim(); if para.is_empty() { continue; @@ -438,8 +462,7 @@ pub(crate) fn chunk_text(text: &str, max_len: Option) -> Vec { } fn split_sentences(text: &str) -> Vec { - let re = Regex::new(r"([.!?])\s+").unwrap(); - let matches: Vec<_> = re.find_iter(text).collect(); + let matches: Vec<_> = RE_SENTENCE_SPLIT.find_iter(text).collect(); if matches.is_empty() { return vec![text.to_string()]; } diff --git a/desktop/src-tauri/src/huddle/tts.rs b/desktop/src-tauri/src/huddle/tts.rs index 807b5b4a32..8c7f312fe6 100644 --- a/desktop/src-tauri/src/huddle/tts.rs +++ b/desktop/src-tauri/src/huddle/tts.rs @@ -105,19 +105,28 @@ impl TtsPipeline { /// /// `tts_active` is set to `true` while audio is playing and `false` when idle. /// Pass the same `Arc` to the STT pipeline to gate microphone input. - pub fn new(model_dir: PathBuf, tts_active: Arc) -> Result { - Self::new_with_voice(model_dir, tts_active, supertonic::DEFAULT_VOICE) + /// + /// `cancel` is the shared barge-in flag from `HuddleState.tts_cancel`. Pass the + /// same `Arc` to the STT pipeline so both sides reference the same flag for the + /// entire huddle session — no stale references after pipeline restarts. + pub fn new( + model_dir: PathBuf, + tts_active: Arc, + cancel: Arc, + ) -> Result { + Self::new_with_voice(model_dir, tts_active, cancel, supertonic::DEFAULT_VOICE) } /// Spawn the TTS pipeline thread with a specific voice name (e.g. `"F1"`, `"M3"`). pub fn new_with_voice( model_dir: PathBuf, tts_active: Arc, + cancel: Arc, voice: &str, ) -> Result { let (text_tx, text_rx) = mpsc::sync_channel::(TEXT_QUEUE_DEPTH); let shutdown = Arc::new(AtomicBool::new(false)); - let cancel = Arc::new(AtomicBool::new(false)); + // cancel is passed in from HuddleState.tts_cancel — shared with STT for barge-in. let shutdown_worker = Arc::clone(&shutdown); let cancel_worker = Arc::clone(&cancel); @@ -331,14 +340,14 @@ fn tts_worker( /// Synthesize a batch of sentences in a single engine call. /// -/// Sentences are joined with ". " so Supertonic sees full context for better -/// prosody. `silence_secs=INTER_SENTENCE_SILENCE` lets the engine insert -/// natural pauses between sentences internally. +/// Sentences are joined with a space so Supertonic sees full context for +/// better prosody. `silence_secs=INTER_SENTENCE_SILENCE` lets the engine +/// insert natural pauses between sentences internally. /// /// After synthesis: volume boost (×VOLUME_BOOST, clamped) + 8ms fade in/out /// to eliminate clicks at batch boundaries. fn synth_batch(engine: &mut TextToSpeech, sentences: &[String], style: &Style) -> Option> { - let text = sentences.join(". "); + let text = sentences.join(" "); match engine.call( &text, "en", diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 66f6847cad..a9dd9f6d79 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -11,8 +11,9 @@ mod util; use app_state::{build_app_state, resolve_persisted_identity, AppState}; use commands::*; use huddle::{ - add_agent_to_huddle, download_voice_models, end_huddle, get_huddle_state, get_model_status, - join_huddle, leave_huddle, push_audio_pcm, set_tts_enabled, speak_agent_message, start_huddle, + add_agent_to_huddle, check_pipeline_hotstart, confirm_huddle_active, download_voice_models, + end_huddle, get_huddle_agent_pubkeys, get_huddle_state, get_model_status, join_huddle, + leave_huddle, push_audio_pcm, set_tts_enabled, speak_agent_message, start_huddle, start_stt_pipeline, }; use managed_agents::{ @@ -516,6 +517,9 @@ pub fn run() { set_tts_enabled, speak_agent_message, add_agent_to_huddle, + check_pipeline_hotstart, + confirm_huddle_active, + get_huddle_agent_pubkeys, ]) .build(tauri::generate_context!()) .expect("error while building tauri application"); diff --git a/desktop/src/features/huddle/HuddleContext.tsx b/desktop/src/features/huddle/HuddleContext.tsx index 617ec6ec65..0d940018f7 100644 --- a/desktop/src/features/huddle/HuddleContext.tsx +++ b/desktop/src/features/huddle/HuddleContext.tsx @@ -26,8 +26,9 @@ interface HuddleContextValue { parentChannelId: string, memberPubkeys: string[], ) => Promise; - /** Leave the current huddle — disconnects LiveKit, stops worklet, calls Rust leave_huddle */ - leaveHuddle: () => Promise; + /** Leave the current huddle — disconnects LiveKit, stops worklet, calls Rust leave_huddle. + * Returns true if backend cleanup succeeded, false if it failed (caller may retry). */ + leaveHuddle: () => Promise; } const HuddleContext = React.createContext(null); @@ -50,13 +51,8 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { >(null); /** Self pubkey — fetched once, used to filter out own messages from TTS */ const selfPubkeyRef = React.useRef(null); - const analyserRef = React.useRef<{ - ctx: AudioContext; - analyser: AnalyserNode; - raf: number; - } | null>(null); - const leaveHuddle = React.useCallback(async () => { + const leaveHuddle = React.useCallback(async (): Promise => { // Invalidate any in-flight startHuddle so it bails after its next await tokenRef.current += 1; @@ -87,10 +83,44 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { rustActiveRef.current = false; } catch { // Leave rustActiveRef true so a subsequent leaveHuddle() retries Rust cleanup + return false; // Signal that backend cleanup failed } } + return true; // Backend cleanup succeeded (or was not needed) }, []); + /** Clean up a partially-established huddle. Best-effort on every step. */ + const cleanupFailedStart = React.useCallback( + async ( + conn: HuddleConnection | null, + worklet: { stop: () => void } | null, + ) => { + try { + worklet?.stop(); + } catch { + /* best-effort */ + } + try { + if (conn) await conn.disconnect(); + } catch { + /* best-effort */ + } + connectionRef.current = null; + setLocalAudioTrack(null); + setMicConnected(false); + setEphemeralChannelId(null); + if (rustActiveRef.current) { + try { + await invoke("leave_huddle"); + rustActiveRef.current = false; + } catch { + /* leave rustActiveRef true so leaveHuddle retries */ + } + } + }, + [], + ); + const startHuddle = React.useCallback( async (parentChannelId: string, memberPubkeys: string[]) => { // Synchronous concurrency guard — belt-and-suspenders alongside isStarting state @@ -109,7 +139,7 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { memberPubkeys, }); rustActiveRef.current = true; - setEphemeralChannelId(joinInfo.ephemeral_channel_id); + // Do NOT set ephemeralChannelId yet — wait until fully established (LiveKit + Worklet) // Fetch self pubkey once for TTS filtering if (!selfPubkeyRef.current) { @@ -123,12 +153,7 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { // Bail if superseded (leaveHuddle or another startHuddle was called) if (tokenRef.current !== myToken) { - try { - await invoke("leave_huddle"); - rustActiveRef.current = false; - } catch { - /* leave rustActiveRef true so leaveHuddle retries */ - } + await cleanupFailedStart(null, null); return; } @@ -140,17 +165,7 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { // Bail if superseded after async connect if (tokenRef.current !== myToken) { - try { - await connection.disconnect(); - } catch { - /* best-effort */ - } - try { - await invoke("leave_huddle"); - rustActiveRef.current = false; - } catch { - /* leave rustActiveRef true so leaveHuddle retries */ - } + await cleanupFailedStart(connection, null); return; } @@ -163,46 +178,23 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { // Bail if superseded after async worklet setup if (tokenRef.current !== myToken) { - try { - worklet.stop(); - } catch { - /* best-effort */ - } - try { - await connection.disconnect(); - } catch { - /* best-effort */ - } - connectionRef.current = null; - setLocalAudioTrack(null); - try { - await invoke("leave_huddle"); - rustActiveRef.current = false; - } catch { - /* leave rustActiveRef true so leaveHuddle retries */ - } + await cleanupFailedStart(connection, worklet); return; } workletRef.current = worklet; + // Step 4: Huddle fully established — now safe to set ephemeralChannelId + // This triggers TTS subscription and hot-start polling effects + setEphemeralChannelId(joinInfo.ephemeral_channel_id); + + // Confirm to backend that media is established — transitions Connected → Active. + await invoke("confirm_huddle_active"); } catch (e) { - // Clean up the LOCAL connection captured above, not whatever is in the ref - try { - if (connection) await connection.disconnect(); - } catch { - /* best-effort */ - } - connectionRef.current = null; - setLocalAudioTrack(null); - // Tell Rust to reset from Creating/Active back to Idle and archive orphaned channel - if (rustActiveRef.current) { - try { - await invoke("leave_huddle"); - rustActiveRef.current = false; - } catch { - /* leave rustActiveRef true so leaveHuddle retries */ - } - } + // Pass workletRef.current — it may have been assigned before the error + // (e.g. confirm_huddle_active rejects after worklet setup succeeded). + const w = workletRef.current; + workletRef.current = null; + await cleanupFailedStart(connection, w); console.error("Failed to start huddle:", e); throw e; } finally { @@ -210,38 +202,96 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { busyRef.current = false; } }, - [], + [cleanupFailedStart], ); - // TTS subscription — pipe agent messages from ephemeral channel to speak_agent_message + // TTS subscription — pipe AGENT messages from ephemeral channel to speak_agent_message. + // Human STT transcripts are also kind:9 in this channel, so we must filter them out + // using an authoritative agent list fetched from the relay membership API. React.useEffect(() => { if (!ephemeralChannelId) return; let disposed = false; let cleanup: (() => void) | null = null; - // Track subscription start time — only speak messages that arrive AFTER we connect. - // subscribeToChannel replays recent history; we don't want to speak old messages. - const subscribeTime = Math.floor(Date.now() / 1000); + + // ── Agent identity (authoritative, fail-closed) ─────────────────────── + // + // Fetch the ephemeral channel's member list from the relay REST API and + // identify agents by their "bot" role. This is authoritative — it works + // for both creators and joiners, and reflects mid-huddle agent additions. + // + // FAIL-CLOSED: agentsLoaded starts false. Until the fetch succeeds and + // populates agentPubkeys, NO messages are spoken. An empty set after a + // successful fetch means "no agents in the huddle" → still mute. + let agentsLoaded = false; + const agentPubkeys = new Set(); + + async function loadAgentPubkeys() { + try { + const pubkeys = await invoke("get_huddle_agent_pubkeys"); + agentPubkeys.clear(); + for (const pk of pubkeys) agentPubkeys.add(pk); + agentsLoaded = true; + } catch (e) { + // Fail-closed on ALL failures, including refresh after prior success. + // Clear the set and mark as not loaded — TTS goes mute until the + // next successful refresh. Stale membership must never authorize speech. + agentPubkeys.clear(); + agentsLoaded = false; + console.error("[huddle] Failed to load agent pubkeys:", e); + } + } + + // Initial load + periodic refresh (catches mid-huddle agent additions). + void loadAgentPubkeys(); + const agentRefreshId = window.setInterval(() => { + void loadAgentPubkeys(); + }, 10_000); + + // ── EOSE-based replay boundary with timestamp belt-and-suspenders ───── + let subscriptionReady = false; + const seenEventIds = new Set(); + const subscribeTimeSecs = Math.floor(Date.now() / 1000); + + /** Speak an event if it passes all filters. */ + function maybeSpeakEvent(pubkey: string, content: string) { + // Fail-closed: don't speak until agent list is loaded. + if (!agentsLoaded) return; + // Only speak agent messages — skip human STT transcripts. + if (!agentPubkeys.has(pubkey)) return; + if (pubkey === selfPubkeyRef.current) return; + if (!content.trim()) return; + if (content.startsWith("[System]")) return; + invoke("speak_agent_message", { text: content }).catch(() => { + /* best-effort */ + }); + } relayClient .subscribeToChannel(ephemeralChannelId, (event) => { if (disposed) return; - // Only kind:9 (chat messages) if (event.kind !== 9) return; - // Skip historical messages (arrived before we subscribed) - if (event.created_at < subscribeTime) return; - // Skip own messages - if (event.pubkey === selfPubkeyRef.current) return; - // Skip empty/whitespace-only content - if (!event.content.trim()) return; - // Skip [System] messages - if (event.content.startsWith("[System]")) return; - - invoke("speak_agent_message", { text: event.content }).catch(() => { - /* best-effort */ - }); + + // Dedup by event ID (covers reconnect replay). + if (seenEventIds.has(event.id)) return; + seenEventIds.add(event.id); + + if (!subscriptionReady) { + // Before EOSE: track as history, don't speak. + return; + } + + // After EOSE: belt-and-suspenders — also reject events with + // created_at before our subscription started. This catches late + // history that arrives after the 250ms fallback fires. + if (event.created_at < subscribeTimeSecs) return; + + maybeSpeakEvent(event.pubkey, event.content); }) .then((dispose) => { + // subscribeToChannel resolves after EOSE (or relay's 250ms fallback). + subscriptionReady = true; + if (disposed) { void dispose(); return; @@ -255,9 +305,21 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { return () => { disposed = true; cleanup?.(); + window.clearInterval(agentRefreshId); }; }, [ephemeralChannelId]); + // Pipeline hot-start — check if voice models finished downloading mid-huddle + React.useEffect(() => { + if (!ephemeralChannelId) return; + const id = window.setInterval(() => { + invoke("check_pipeline_hotstart").catch(() => { + /* best-effort */ + }); + }, 5_000); + return () => window.clearInterval(id); + }, [ephemeralChannelId]); + // Mic level analyser — drives the voice activity indicator React.useEffect(() => { if (!localAudioTrack) { @@ -285,13 +347,10 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { } raf = requestAnimationFrame(tick); - analyserRef.current = { ctx, analyser, raf }; - return () => { cancelAnimationFrame(raf); source.disconnect(); void ctx.close(); - analyserRef.current = null; }; }, [localAudioTrack]); diff --git a/desktop/src/features/huddle/components/HuddleBar.tsx b/desktop/src/features/huddle/components/HuddleBar.tsx index dd896c78e5..4810d39a54 100644 --- a/desktop/src/features/huddle/components/HuddleBar.tsx +++ b/desktop/src/features/huddle/components/HuddleBar.tsx @@ -18,13 +18,20 @@ import { ParticipantList } from "./ParticipantList"; // Shape returned by the `get_huddle_state` Tauri command type HuddleState = { - phase: "idle" | "creating" | "connecting" | "active" | "leaving"; + phase: + | "idle" + | "creating" + | "connecting" + | "connected" + | "active" + | "leaving"; parent_channel_id: string | null; ephemeral_channel_id: string | null; livekit_token: string | null; livekit_url: string | null; livekit_room: string | null; participants: string[]; // pubkey hex strings + tts_enabled: boolean; }; type HuddleBarProps = { @@ -35,7 +42,9 @@ export function HuddleBar({ className }: HuddleBarProps) { const { localAudioTrack, leaveHuddle, micConnected, micLevel } = useHuddle(); const [state, setState] = React.useState(null); const [isMuted, setIsMuted] = React.useState(false); - const [ttsEnabled, setTtsEnabled] = React.useState(true); + // Derive TTS enabled from backend state (single source of truth). + // Fall back to true if state hasn't loaded yet. + const ttsEnabled = state?.tts_enabled ?? true; const [isLeaving, setIsLeaving] = React.useState(false); const [showAddAgent, setShowAddAgent] = React.useState(false); const [agentAddError, setAgentAddError] = React.useState(null); @@ -52,7 +61,11 @@ export function HuddleBar({ className }: HuddleBarProps) { // Only clear state if we never had an active huddle. // Transient errors shouldn't remove the control bar. if (!cancelled) { - setState((prev) => (prev?.phase === "active" ? prev : null)); + setState((prev) => + prev?.phase === "active" || prev?.phase === "connected" + ? prev + : null, + ); } } } @@ -73,14 +86,19 @@ export function HuddleBar({ className }: HuddleBarProps) { } }, [isMuted, localAudioTrack]); - if (!state || state.phase !== "active") return null; + if (!state || (state.phase !== "active" && state.phase !== "connected")) + return null; async function handleLeave() { if (isLeaving) return; setIsLeaving(true); try { - await leaveHuddle(); - setState(null); + const backendClean = await leaveHuddle(); + if (backendClean) { + setState(null); + } + // If backend cleanup failed, keep the bar visible so the user can retry. + // leaveHuddle retains rustActiveRef=true for the next attempt. } catch (e) { console.error("Failed to leave huddle:", e); } finally { @@ -187,7 +205,9 @@ export function HuddleBar({ className }: HuddleBarProps) { const next = !ttsEnabled; try { await invoke("set_tts_enabled", { enabled: next }); - setTtsEnabled(next); + // Refresh state immediately so the UI reflects the change + const s = await invoke("get_huddle_state"); + setState(s); } catch (e) { console.error("Failed to toggle TTS:", e); } diff --git a/desktop/src/features/huddle/components/ParticipantList.tsx b/desktop/src/features/huddle/components/ParticipantList.tsx index 32d2e64a53..5f895288bc 100644 --- a/desktop/src/features/huddle/components/ParticipantList.tsx +++ b/desktop/src/features/huddle/components/ParticipantList.tsx @@ -1,12 +1,5 @@ import { cn } from "@/shared/lib/cn"; -// Legacy type kept for any callers that haven't migrated yet -export type HuddleParticipant = { - identity: string; - displayName: string; - isMuted: boolean; -}; - type ParticipantListProps = { /** Pubkey hex strings from the Rust huddle state */ participants: string[]; diff --git a/desktop/src/features/huddle/index.ts b/desktop/src/features/huddle/index.ts index 59de6357c4..e6f1280ad8 100644 --- a/desktop/src/features/huddle/index.ts +++ b/desktop/src/features/huddle/index.ts @@ -4,4 +4,3 @@ export type { HuddleConnection } from "./lib/livekit"; export { setupAudioWorklet } from "./lib/audioWorklet"; export { HuddleBar } from "./components/HuddleBar"; export { ParticipantList } from "./components/ParticipantList"; -export type { HuddleParticipant } from "./components/ParticipantList"; diff --git a/desktop/src/features/huddle/lib/audioWorklet.ts b/desktop/src/features/huddle/lib/audioWorklet.ts index 9e496d9922..6fe194d06d 100644 --- a/desktop/src/features/huddle/lib/audioWorklet.ts +++ b/desktop/src/features/huddle/lib/audioWorklet.ts @@ -34,19 +34,21 @@ export async function setupAudioWorklet( source.connect(workletNode); // Forward PCM batches to Rust via raw binary invoke - workletNode.port.onmessage = async (event: MessageEvent) => { + workletNode.port.onmessage = (event: MessageEvent) => { const float32 = event.data; - try { - // Tauri v2 InvokeBody::Raw only accepts ArrayBuffer | Uint8Array. - // Create a zero-copy Uint8Array view over the same underlying buffer. - // Rust reinterprets the bytes as f32 on the other side. - await window.__TAURI_INTERNALS__.invoke( + // Fire-and-forget — Rust side uses try_send which drops on backpressure. + // No await: prevents main-thread backpressure from slow Rust processing. + // Tauri v2 InvokeBody::Raw only accepts ArrayBuffer | Uint8Array. + // Create a zero-copy Uint8Array view over the same underlying buffer. + // Rust reinterprets the bytes as f32 on the other side. + window.__TAURI_INTERNALS__ + .invoke( "push_audio_pcm", new Uint8Array(float32.buffer, float32.byteOffset, float32.byteLength), - ); - } catch (e) { - console.error("Failed to send PCM to Rust:", e); - } + ) + .catch(() => { + /* silently drop — Rust handles backpressure */ + }); }; return { From f3798d712112ab8534cf5d044df3d12292110805 Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Sun, 12 Apr 2026 19:39:31 -0400 Subject: [PATCH 27/41] =?UTF-8?q?fix(huddles):=20crossfire=20quality=20pas?= =?UTF-8?q?s=20=E2=80=94=20safety,=20DRY,=20UX=20hardening?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-round crossfire review (Opus + Codex CLI, 8 full review passes, 17 parallel worker delegates) identified and fixed 25+ issues across the huddle voice pipeline. Safety & correctness: - Session generation guard: Arc on HuddleState prevents stale transcription tasks from posting kind:9 after leave/end - Speech buffer capped at 30s (prevents OOM in noisy environments) - Raw IPC payload bounded at 100KB per batch - TTS input bounded at 2000 chars with Unicode-safe truncation - LiveKit token/URL hidden from polling via #[serde(skip)] - Model downloads: SHA-256 verification with pinned hashes, Rust-native tar+bzip2 extraction with pre-validation (path traversal, symlinks), streaming to disk, size limits, version manifest for cache invalidation - Pubkey format validation (64 hex chars) at Tauri boundary - Max 20 agents per huddle enforced on both start and incremental add - UUID validation on all huddle event builders - PCM alignment rejection (not just warning) on non-4-byte-aligned input Architecture & DRY: - teardown_huddle() helper eliminates duplicated leave/end shutdown code - start_stt_pipeline delegates to maybe_start_stt_pipeline - split_sentences consolidated in preprocessing.rs (deleted from tts.rs and supertonic.rs, net -85 lines) - int_to_words extended to 0-999,999 - Agent refresh throttled to 15s with success-gated timestamp Lifecycle: - Creator enforcement: is_creator field on HuddleState, end_huddle rejects non-creators, HuddleBar shows End/Leave conditionally - Failed startup cleanup calls end_huddle (not leave_huddle) to prevent orphaned ephemeral channels - Participant state hydrated from relay immediately on start/join UX & AX: - Agent prompt tightened: silent when not addressed, no dot responses, no repeat after interruption - TTS subscription buffers pre-EOSE events, replays live ones (fixes silent drop of fast agent responses) - AddAgentDialog filters already-added agents - Participant display shows 'In huddle' instead of misleading count - Room label shows 'Huddle' instead of raw LiveKit room name - TTS backpressure logged (Rust + frontend) - AudioWorklet IPC wrapped behind invokeRawBinary() abstraction - ASCII lifecycle docs added to HuddleContext, audioWorklet, livekit - supertonic.rs header fixed (44.1kHz not 24kHz) - worklet.js documents intentional partial-buffer drop on disconnect - ParticipantList NaN-safe hue derivation with gray fallback - tts.rs expect() calls replaced with match+break 18 files changed, +984 -370 --- desktop/public/worklet.js | 5 + desktop/scripts/check-file-sizes.mjs | 5 +- desktop/src-tauri/Cargo.lock | 2 + desktop/src-tauri/Cargo.toml | 2 + desktop/src-tauri/src/events.rs | 14 + desktop/src-tauri/src/huddle/agents.rs | 29 +- desktop/src-tauri/src/huddle/mod.rs | 397 ++++++++++++------ desktop/src-tauri/src/huddle/models.rs | 349 ++++++++++++--- desktop/src-tauri/src/huddle/preprocessing.rs | 173 +++++++- desktop/src-tauri/src/huddle/stt.rs | 28 +- desktop/src-tauri/src/huddle/supertonic.rs | 43 +- desktop/src-tauri/src/huddle/tts.rs | 73 +--- desktop/src/features/huddle/HuddleContext.tsx | 110 ++++- .../huddle/components/AddAgentDialog.tsx | 17 +- .../features/huddle/components/HuddleBar.tsx | 65 ++- .../huddle/components/ParticipantList.tsx | 11 +- .../src/features/huddle/lib/audioWorklet.ts | 49 ++- desktop/src/features/huddle/lib/livekit.ts | 15 + 18 files changed, 1015 insertions(+), 372 deletions(-) diff --git a/desktop/public/worklet.js b/desktop/public/worklet.js index c889a729a8..2a874f8ccd 100644 --- a/desktop/public/worklet.js +++ b/desktop/public/worklet.js @@ -1,5 +1,10 @@ // AudioWorklet processor — runs in the AudioWorklet thread. // Accumulates PCM Float32 samples and sends 100ms batches to the main thread. +// +// Note: when the worklet is disconnected, any partial buffer (< 4800 samples) +// is silently dropped. This means the last ~100ms of speech may be lost on +// huddle leave. This is acceptable — the STT pipeline's silence-flush threshold +// (450ms) means the last utterance was already transcribed before disconnect. class SttTapProcessor extends AudioWorkletProcessor { constructor() { super(); diff --git a/desktop/scripts/check-file-sizes.mjs b/desktop/scripts/check-file-sizes.mjs index 791efe74e8..d96f07b5e0 100644 --- a/desktop/scripts/check-file-sizes.mjs +++ b/desktop/scripts/check-file-sizes.mjs @@ -55,8 +55,9 @@ const overrides = new Map([ ["src/features/agents/ui/CreateAgentDialog.tsx", 685], // provider selector + config form + schema-typed config coercion + required field validation + locked scopes ["src/features/channels/ui/AddChannelBotDialog.tsx", 640], // provider mode: Run on selector, trust warning, probe effect, single-agent enforcement, provider warnings display ["src/shared/api/types.ts", 535], // persona provider/model fields + forum types + workflow type re-exports + ephemeral channel TTL fields + mcpToolsets - ["src-tauri/src/huddle/mod.rs", 1200], // huddle state machine + 12 Tauri commands + STT/TTS pipeline lifecycle + relay membership fetch + two-phase activation; split planned post-MVP - ["src-tauri/src/huddle/models.rs", 650], // model download manager for Moonshine STT + Supertonic TTS with streaming downloads + atomic swap + hot-start signaling + ["src-tauri/src/huddle/mod.rs", 1300], // huddle state machine + 14 Tauri commands + STT/TTS pipeline lifecycle + relay membership fetch + session generation guard + creator enforcement + input validation; split planned post-MVP + ["src-tauri/src/huddle/models.rs", 850], // model download manager for Moonshine STT + Supertonic TTS with streaming downloads + SHA-256 verification + Rust-native tar extraction + version manifest + atomic swap + hot-start signaling + ["src-tauri/src/huddle/preprocessing.rs", 620], // TTS text preprocessing pipeline + unified split_sentences (consolidated from tts.rs + supertonic.rs) + int_to_words 0-999999 + 18 unit tests ["src-tauri/src/huddle/supertonic.rs", 780], // Supertonic 4-ONNX-session TTS engine wrapper + Unicode text processor + LazyLock regex patterns + text chunking ]); diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index b5c9456846..ece145bb43 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -5151,6 +5151,7 @@ dependencies = [ "atomic-write-file", "audioadapter-buffers", "base64 0.22.1", + "bzip2 0.5.2", "chrono", "dirs", "earshot", @@ -5172,6 +5173,7 @@ dependencies = [ "sha2 0.11.0", "sherpa-onnx", "sprout-core", + "tar", "tauri", "tauri-build", "tauri-plugin-dialog", diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index 06b3a3705a..ef76260a54 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -44,6 +44,8 @@ url = "2" sprout-core = { path = "../../crates/sprout-core" } base64 = "0.22" sha2 = "0.11" +tar = "0.4" +bzip2 = "0.5" chrono = { version = "0.4", features = ["serde"] } tauri-plugin-notification = "2.3.3" uuid = { version = "1", features = ["v4"] } diff --git a/desktop/src-tauri/src/events.rs b/desktop/src-tauri/src/events.rs index a0b4e2bc8a..5c105e2bce 100644 --- a/desktop/src-tauri/src/events.rs +++ b/desktop/src-tauri/src/events.rs @@ -359,12 +359,20 @@ pub fn build_profile( // ── Huddles ────────────────────────────────────────────────────────────────── +/// Validate that a string is a valid UUID (defense-in-depth for `&str` channel IDs). +fn validate_channel_id(id: &str) -> Result<(), String> { + uuid::Uuid::parse_str(id).map_err(|_| format!("invalid channel UUID: {id}"))?; + Ok(()) +} + /// Kind 48100 — huddle started advisory posted to the parent channel. pub fn build_huddle_started( parent_channel_id: &str, ephemeral_channel_id: &str, livekit_room: &str, ) -> Result { + validate_channel_id(parent_channel_id)?; + validate_channel_id(ephemeral_channel_id)?; let content = serde_json::json!({ "ephemeral_channel_id": ephemeral_channel_id, "livekit_room": livekit_room, @@ -379,6 +387,8 @@ pub fn build_huddle_participant_joined( parent_channel_id: &str, ephemeral_channel_id: &str, ) -> Result { + validate_channel_id(parent_channel_id)?; + validate_channel_id(ephemeral_channel_id)?; let content = serde_json::json!({ "ephemeral_channel_id": ephemeral_channel_id, }) @@ -392,6 +402,8 @@ pub fn build_huddle_participant_left( parent_channel_id: &str, ephemeral_channel_id: &str, ) -> Result { + validate_channel_id(parent_channel_id)?; + validate_channel_id(ephemeral_channel_id)?; let content = serde_json::json!({ "ephemeral_channel_id": ephemeral_channel_id, }) @@ -405,6 +417,8 @@ pub fn build_huddle_ended( parent_channel_id: &str, ephemeral_channel_id: &str, ) -> Result { + validate_channel_id(parent_channel_id)?; + validate_channel_id(ephemeral_channel_id)?; let content = serde_json::json!({ "ephemeral_channel_id": ephemeral_channel_id, }) diff --git a/desktop/src-tauri/src/huddle/agents.rs b/desktop/src-tauri/src/huddle/agents.rs index 4c7fe918e3..12160f668f 100644 --- a/desktop/src-tauri/src/huddle/agents.rs +++ b/desktop/src-tauri/src/huddle/agents.rs @@ -23,19 +23,17 @@ use crate::{app_state::AppState, events, relay::submit_event}; pub fn voice_mode_guidelines(parent_channel_id: &str) -> String { format!( "\ -You are in a live voice huddle. Your text is read aloud via TTS. -This huddle is attached to channel {parent_channel_id} — that's the main channel. -You will be interrupted by new messages whenever a human speaks — this is normal. +You are in a live voice huddle. Your responses are read aloud via text-to-speech. +This huddle is attached to channel {parent_channel_id} (the main channel). +You will be interrupted whenever a human speaks — this is normal, do not repeat yourself. Rules: -- Only respond if the message is relevant to you or directed at you. - If it's not for you, respond with just \".\" or stay silent. -- Keep responses under 2 sentences. This is a conversation, not an essay. -- Spell out numbers: \"eleven thirty\" not \"11:30\". -- No markdown, code blocks, or bullet lists — they sound terrible as speech. -- To share code or data, say \"I'll post that in the main channel\" and use it. -- You have access to Sprout tools — you can join channels, search messages, - and take actions. Use them proactively when asked." +- ONLY respond if addressed directly or the topic is clearly relevant to you. + If not for you, stay completely silent — do not respond at all. +- Maximum 2 sentences. This is a conversation, not a monologue. +- Speak naturally: \"eleven thirty\" not \"11:30\", no markdown, no code blocks, no lists. +- To share code or structured data, say \"I'll post that in the main channel\" and do so. +- Use your Sprout tools proactively — search messages, join channels, take actions when asked." ) } @@ -43,9 +41,12 @@ Rules: /// Result of adding an agent to a huddle. /// -/// `ephemeral_added` is always true on success (the function returns Err if -/// the ephemeral add fails). `parent_added` reflects whether the parent-channel -/// add succeeded; `parent_error` carries the error string when it didn't. +/// `ephemeral_added` is always `true` when this struct is returned (the +/// function returns `Err` if the ephemeral add fails). Retained for +/// forward compatibility with batch-add operations. +/// +/// `parent_added` reflects whether the parent-channel add succeeded; +/// `parent_error` carries the error string when it didn't. #[derive(Debug, Serialize)] pub struct AgentAddResult { /// Whether the agent was added to the ephemeral channel (required). diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index 2dfc4af9ed..ec1a5aaec1 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -17,7 +17,10 @@ pub mod tts; use reqwest::Method; use serde::{Deserialize, Serialize}; -use std::sync::{atomic::AtomicBool, Arc, Mutex}; +use std::sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, Mutex, +}; use tauri::State; use uuid::Uuid; @@ -47,7 +50,13 @@ pub struct HuddleState { pub phase: HuddlePhase, pub parent_channel_id: Option, pub ephemeral_channel_id: Option, + /// Skipped from serialization — the frontend gets the token from + /// start_huddle/join_huddle return values. Exposing the LiveKit JWT + /// on every 2-second get_huddle_state poll is unnecessary attack surface. + #[serde(skip)] pub livekit_token: Option, + /// Skipped from serialization — same rationale as livekit_token. + #[serde(skip)] pub livekit_url: Option, pub livekit_room: Option, /// Participant pubkey hex strings (all members, including humans). @@ -73,6 +82,9 @@ pub struct HuddleState { /// Active TTS pipeline — not serialized, not cloned. #[serde(skip)] pub tts_pipeline: Option>, + /// Whether this client created the huddle (vs. joined it). + /// Used to enforce that only the creator can end/archive the huddle. + pub is_creator: bool, /// Whether TTS output is enabled (user-toggled). pub tts_enabled: bool, /// Shared flag: true while TTS is playing audio. @@ -84,6 +96,15 @@ pub struct HuddleState { /// restarts — both STT and TTS reference the same flag for the entire huddle. #[serde(skip)] pub tts_cancel: Arc, + /// Timestamp of the last agent pubkey refresh from the relay. + /// Used to throttle the refresh in check_pipeline_hotstart to every 15 s. + #[serde(skip)] + pub last_agent_refresh: Option, + /// Session generation — incremented on every teardown. The transcription + /// task captures this at spawn time and checks before each POST. If the + /// generation has changed, the task silently drops the transcript. + #[serde(skip)] + pub session_generation: Arc, } fn serialize_agent_pubkeys(v: &Arc>>, s: S) -> Result @@ -125,9 +146,12 @@ impl Clone for HuddleState { agent_pubkeys: Arc::new(Mutex::new(agent_pubkeys_snapshot)), stt_pipeline: None, // Never clone the pipeline handle. tts_pipeline: None, // Never clone the pipeline handle. + is_creator: self.is_creator, tts_enabled: self.tts_enabled, tts_active: Arc::clone(&self.tts_active), tts_cancel: Arc::clone(&self.tts_cancel), + last_agent_refresh: self.last_agent_refresh, + session_generation: Arc::clone(&self.session_generation), } } } @@ -145,9 +169,12 @@ impl Default for HuddleState { agent_pubkeys: Arc::new(Mutex::new(Vec::new())), stt_pipeline: None, tts_pipeline: None, + is_creator: false, tts_enabled: true, tts_active: Arc::new(AtomicBool::new(false)), tts_cancel: Arc::new(AtomicBool::new(false)), + last_agent_refresh: None, + session_generation: Arc::new(AtomicU64::new(0)), } } } @@ -173,6 +200,20 @@ struct LiveKitTokenResponse { // ── Helpers ─────────────────────────────────────────────────────────────────── +/// Maximum number of agents that can be invited to a single huddle. +const MAX_HUDDLE_AGENTS: usize = 20; + +/// Validate that a string looks like a Nostr pubkey hex (64 hex chars). +fn validate_pubkey_hex(pubkey: &str) -> Result<(), String> { + if pubkey.len() != 64 || !pubkey.chars().all(|c| c.is_ascii_hexdigit()) { + return Err(format!( + "invalid pubkey hex: {}", + &pubkey[..pubkey.len().min(16)] + )); + } + Ok(()) +} + fn parse_channel_uuid(channel_id: &str) -> Result { Uuid::parse_str(channel_id).map_err(|_| format!("invalid channel UUID: {channel_id}")) } @@ -229,78 +270,74 @@ async fn fetch_agent_pubkeys_from_relay( } } +/// Fetch ALL member pubkeys from the relay's channel membership API. +/// +/// Returns every member (humans + bots) for participant hydration. +/// Same API as `fetch_agent_pubkeys_from_relay` but without the bot filter. +async fn fetch_all_member_pubkeys( + channel_id: &str, + state: &AppState, +) -> Result, String> { + #[derive(Deserialize)] + struct Member { + pubkey: String, + } + #[derive(Deserialize)] + struct MembersResponse { + members: Vec, + } + + let path = api_path(&["channels", channel_id, "members"]); + let request = build_authed_request(&state.http_client, Method::GET, &path, state)?; + let resp: MembersResponse = send_json_request(request).await?; + Ok(resp.members.into_iter().map(|m| m.pubkey).collect()) +} + /// Attempt to start the STT pipeline if models are present. -/// Silently skips if models are missing — huddle continues as voice-only. +/// +/// Returns `Ok(true)` if the pipeline was started, `Ok(false)` if models are +/// not ready (voice-only mode), or `Err` on a real failure. /// /// Creates the shared `tts_active` flag and passes it to the STT pipeline /// for barge-in / echo gating. The same flag is later passed to the TTS /// pipeline so it can signal when audio is playing. -async fn maybe_start_stt_pipeline(state: &AppState, ephemeral_channel_id: &str) { +async fn maybe_start_stt_pipeline( + state: &AppState, + ephemeral_channel_id: &str, +) -> Result { if !models::is_moonshine_ready() { - return; // Models not downloaded yet — voice-only mode. + return Ok(false); // Models not downloaded yet — voice-only mode. } - let model_dir = match models::moonshine_model_dir() { - Some(d) => d, - None => return, - }; + let model_dir = + models::moonshine_model_dir().ok_or_else(|| "Moonshine model directory not found")?; - // Grab the shared tts_active flag from HuddleState. - let tts_active = { - let hs = match state.huddle_state.lock() { - Ok(h) => h, - Err(_) => return, - }; - Arc::clone(&hs.tts_active) - }; + let channel_uuid = parse_channel_uuid(ephemeral_channel_id)?; - // Grab the shared barge-in cancel flag from HuddleState. - // This flag lives for the entire huddle session — no stale references - // if TTS is restarted (set_tts_enabled toggle or hot-start). - let tts_cancel = { - let hs = match state.huddle_state.lock() { - Ok(h) => h, - Err(_) => return, - }; - Some(Arc::clone(&hs.tts_cancel)) + // Grab shared flags, agent pubkeys, and session generation from HuddleState in one lock. + let (tts_active, tts_cancel, agent_pubkeys_arc, session_gen) = { + let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + ( + Arc::clone(&hs.tts_active), + Some(Arc::clone(&hs.tts_cancel)), + Arc::clone(&hs.agent_pubkeys), + Arc::clone(&hs.session_generation), + ) }; - let (pipeline, text_rx) = match stt::SttPipeline::new(model_dir, tts_active, tts_cancel) { - Ok(p) => p, - Err(e) => { - eprintln!("sprout-desktop: STT pipeline failed to start: {e}"); - return; - } - }; + let (pipeline, text_rx) = stt::SttPipeline::new(model_dir, tts_active, tts_cancel)?; let pipeline = Arc::new(pipeline); - let channel_uuid = match parse_channel_uuid(ephemeral_channel_id) { - Ok(u) => u, - Err(_) => return, - }; - - // Clone the Arc>> BEFORE storing the pipeline, so we - // can pass it to the transcription task without holding the state lock. - let agent_pubkeys_arc = { - let hs = match state.huddle_state.lock() { - Ok(h) => h, - Err(_) => return, - }; - Arc::clone(&hs.agent_pubkeys) - }; - // Shut down existing STT pipeline before replacing (prevents leaked transcription tasks). { - let mut hs = match state.huddle_state.lock() { - Ok(h) => h, - Err(_) => return, - }; + let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; if let Some(ref old) = hs.stt_pipeline { old.shutdown(); } hs.stt_pipeline = Some(Arc::clone(&pipeline)); } - spawn_transcription_task(text_rx, channel_uuid, agent_pubkeys_arc, state); + spawn_transcription_task(text_rx, channel_uuid, agent_pubkeys_arc, session_gen, state); + Ok(true) } /// Attempt to start the TTS pipeline if Supertonic models are present and TTS is enabled. @@ -376,8 +413,12 @@ fn spawn_transcription_task( mut text_rx: tokio::sync::mpsc::Receiver, channel_uuid: Uuid, agent_pubkeys_arc: Arc>>, + session_generation: Arc, state: &AppState, ) { + // Capture the current generation at spawn time. + let spawned_gen = session_generation.load(Ordering::Acquire); + let http_client = state.http_client.clone(); let keys = match state.keys.lock() { Ok(k) => k.clone(), @@ -393,6 +434,12 @@ fn spawn_transcription_task( continue; } + // Session guard: if the generation has changed, this task is stale. + // Drop the transcript silently — the huddle has ended or been replaced. + if session_generation.load(Ordering::Acquire) != spawned_gen { + break; // Exit the loop entirely — no more posts from this task. + } + // Fix 1: read current agent pubkeys at post time. let agent_pubkeys: Vec = agent_pubkeys_arc .lock() @@ -456,6 +503,27 @@ pub async fn start_huddle( member_pubkeys: Vec, state: State<'_, AppState>, ) -> Result { + // Validate inputs at the Tauri boundary. + if member_pubkeys.len() > MAX_HUDDLE_AGENTS { + return Err(format!( + "too many agents: {} (max {})", + member_pubkeys.len(), + MAX_HUDDLE_AGENTS + )); + } + // Dedup and validate pubkey format. + let member_pubkeys: Vec = { + let mut seen = std::collections::HashSet::new(); + let mut deduped = Vec::new(); + for pk in member_pubkeys { + validate_pubkey_hex(&pk)?; + if seen.insert(pk.clone()) { + deduped.push(pk); + } + } + deduped + }; + // Transition to Creating. { let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; @@ -539,6 +607,7 @@ pub async fn start_huddle( { let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; hs.phase = HuddlePhase::Connected; + hs.is_creator = true; hs.ephemeral_channel_id = Some(ephemeral_channel_id.clone()); hs.livekit_token = Some(lk.token.clone()); hs.livekit_url = Some(lk.url.clone()); @@ -561,17 +630,28 @@ pub async fn start_huddle( hs.participants = participants; } - // 6. Ensure voice models are downloading (idempotent — no-ops if ready or in progress). + // 6. Immediately hydrate participants from relay for authoritative state. + // Best-effort — the local guess above is a reasonable fallback. + if let Ok(all_members) = fetch_all_member_pubkeys(&ephemeral_channel_id, &state).await { + if !all_members.is_empty() { + let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + hs.participants = all_members; + } + } + + // 7. Ensure voice models are downloading (idempotent — no-ops if ready or in progress). if let Some(mgr) = models::global_model_manager() { mgr.start_moonshine_download(state.http_client.clone()); mgr.start_supertonic_download(state.http_client.clone()); } - // 7. Auto-start TTS first, then STT. STT captures `tts_cancel` from + // 8. Auto-start TTS first, then STT. STT captures `tts_cancel` from // `hs.tts_pipeline` at init time — TTS must exist before STT starts // or barge-in never works. maybe_start_tts_pipeline(&state).await; - maybe_start_stt_pipeline(&state, &ephemeral_channel_id).await; + if let Err(e) = maybe_start_stt_pipeline(&state, &ephemeral_channel_id).await { + eprintln!("sprout-desktop: STT pipeline failed to start: {e}"); + } Ok(HuddleJoinInfo { ephemeral_channel_id, @@ -681,6 +761,15 @@ pub async fn join_huddle( *hs.agent_pubkeys.lock().unwrap_or_else(|e| e.into_inner()) = agents; } + // 4b. Immediately hydrate participants from relay for authoritative state. + // Best-effort — the self-only list above is a reasonable fallback. + if let Ok(all_members) = fetch_all_member_pubkeys(&ephemeral_channel_id, &state).await { + if !all_members.is_empty() { + let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + hs.participants = all_members; + } + } + // 5. Ensure voice models are downloading (idempotent). if let Some(mgr) = models::global_model_manager() { mgr.start_moonshine_download(state.http_client.clone()); @@ -689,7 +778,9 @@ pub async fn join_huddle( // 6. Auto-start TTS first, then STT (same ordering rationale as start_huddle). maybe_start_tts_pipeline(&state).await; - maybe_start_stt_pipeline(&state, &ephemeral_channel_id).await; + if let Err(e) = maybe_start_stt_pipeline(&state, &ephemeral_channel_id).await { + eprintln!("sprout-desktop: STT pipeline failed to start: {e}"); + } Ok(HuddleJoinInfo { ephemeral_channel_id, @@ -699,6 +790,34 @@ pub async fn join_huddle( }) } +/// Shut down all pipelines and reset huddle state to Idle. +/// +/// Used by both `leave_huddle` and `end_huddle` to avoid duplicating the +/// shutdown-then-reset sequence. +fn teardown_huddle(state: &AppState) -> Result<(), String> { + { + let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + // Increment generation first — this immediately invalidates any + // in-flight transcription task, even before pipelines shut down. + hs.session_generation.fetch_add(1, Ordering::Release); + if let Some(ref pipeline) = hs.stt_pipeline { + pipeline.shutdown(); + } + if let Some(ref pipeline) = hs.tts_pipeline { + pipeline.shutdown(); + } + } + { + let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + // Preserve the generation counter across reset — it must survive + // for the old transcription task to see the incremented value. + let gen = Arc::clone(&hs.session_generation); + *hs = HuddleState::default(); + hs.session_generation = gen; + } + Ok(()) +} + /// Leave the current huddle. /// /// Steps: @@ -730,23 +849,7 @@ pub async fn leave_huddle(state: State<'_, AppState>) -> Result<(), String> { } } - // Signal the STT and TTS pipelines to stop before dropping state. - // The pipelines' Drop impls will join worker threads for a clean exit. - { - let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; - if let Some(ref pipeline) = hs.stt_pipeline { - pipeline.shutdown(); - } - if let Some(ref pipeline) = hs.tts_pipeline { - pipeline.shutdown(); - } - } - - // Clear state. - { - let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; - *hs = HuddleState::default(); - } + teardown_huddle(&state)?; Ok(()) } @@ -765,6 +868,9 @@ pub async fn end_huddle(state: State<'_, AppState>) -> Result<(), String> { if hs.phase == HuddlePhase::Idle { return Ok(()); // Nothing to end. } + if !hs.is_creator { + return Err("only the huddle creator can end the huddle".to_string()); + } hs.phase = HuddlePhase::Leaving; ( hs.parent_channel_id.clone().unwrap_or_default(), @@ -794,22 +900,7 @@ pub async fn end_huddle(state: State<'_, AppState>) -> Result<(), String> { } } - // Signal the STT and TTS pipelines to stop before dropping state. - { - let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; - if let Some(ref pipeline) = hs.stt_pipeline { - pipeline.shutdown(); - } - if let Some(ref pipeline) = hs.tts_pipeline { - pipeline.shutdown(); - } - } - - // Clear state. - { - let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; - *hs = HuddleState::default(); - } + teardown_huddle(&state)?; Ok(()) } @@ -854,6 +945,11 @@ pub async fn get_huddle_agent_pubkeys(state: State<'_, AppState>) -> Result Result<(), String> { match request.body() { tauri::ipc::InvokeBody::Raw(bytes) => { + if bytes.len() > MAX_AUDIO_BATCH_BYTES { + return Err(format!( + "audio batch too large: {} bytes (max {})", + bytes.len(), + MAX_AUDIO_BATCH_BYTES + )); + } if let Ok(hs) = state.huddle_state.lock() { if let Some(ref pipeline) = hs.stt_pipeline { pipeline.push_audio(bytes.to_vec())?; @@ -912,19 +1015,47 @@ pub async fn check_pipeline_hotstart(state: State<'_, AppState>) -> Result<(), S if !has_stt && (moonshine_ready || models::is_moonshine_ready()) { if let Some(eph_id) = &ephemeral_channel_id { - maybe_start_stt_pipeline(&state, eph_id).await; + if let Err(e) = maybe_start_stt_pipeline(&state, eph_id).await { + eprintln!("sprout-desktop: STT hotstart failed: {e}"); + } } } // Periodically refresh agent_pubkeys from relay membership. // This catches mid-huddle agent additions/removals by other participants, // keeping STT p-tags authoritative throughout the session. + // Throttled to every 15 s (not on every 5 s hotstart poll) — the frontend + // already refreshes its own agentPubkeys every 10 s via get_huddle_agent_pubkeys. // On Ok: always replace (even with empty — agents may have been removed). // On Err: preserve the existing list (transient failure shouldn't zero it). if let Some(eph_id) = &ephemeral_channel_id { - if let Ok(fresh_agents) = fetch_agent_pubkeys_from_relay(eph_id, &state).await { + let should_refresh = { let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; - *hs.agent_pubkeys.lock().unwrap_or_else(|e| e.into_inner()) = fresh_agents; + match hs.last_agent_refresh { + None => true, + Some(t) => t.elapsed() >= std::time::Duration::from_secs(15), + } + }; + if should_refresh { + // Fetch agents (for STT p-tags) and all members (for participant list). + // Sequential — tokio::join! requires the `macros` feature. + // Only update the throttle timestamp when at least one fetch succeeds, + // so transient failures retry immediately on the next poll cycle. + let mut any_success = false; + if let Ok(fresh_agents) = fetch_agent_pubkeys_from_relay(eph_id, &state).await { + let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + *hs.agent_pubkeys.lock().unwrap_or_else(|e| e.into_inner()) = fresh_agents; + any_success = true; + } + if let Ok(fresh_members) = fetch_all_member_pubkeys(eph_id, &state).await { + let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + hs.participants = fresh_members; + any_success = true; + } + if any_success { + let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + hs.last_agent_refresh = Some(std::time::Instant::now()); + } } } @@ -933,51 +1064,23 @@ pub async fn check_pipeline_hotstart(state: State<'_, AppState>) -> Result<(), S /// Start the STT pipeline for the active huddle. /// -/// Creates the pipeline, stores it in HuddleState, and spawns a tokio task -/// that reads transcribed text and posts kind:9 events to the ephemeral -/// channel. -/// -/// No-op if models are not present — huddle continues as voice-only. -/// Safe to call multiple times: replaces the existing pipeline if already running. +/// Delegates to `maybe_start_stt_pipeline` — returns `Err` if models are not +/// ready or no huddle is active. Safe to call multiple times: replaces the +/// existing pipeline if already running. #[tauri::command] pub async fn start_stt_pipeline(state: State<'_, AppState>) -> Result<(), String> { - if !models::is_moonshine_ready() { - return Err("Moonshine model not ready".to_string()); - } - let model_dir = models::moonshine_model_dir() - .ok_or_else(|| "Moonshine model directory not found".to_string())?; - - let (ephemeral_channel_id, agent_pubkeys_arc) = { + let ephemeral_channel_id = { let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; - ( - hs.ephemeral_channel_id.clone(), - Arc::clone(&hs.agent_pubkeys), - ) - }; - - let ephemeral_channel_id = - ephemeral_channel_id.ok_or("no active huddle — start or join a huddle first")?; - let channel_uuid = parse_channel_uuid(&ephemeral_channel_id)?; - - let (tts_active, tts_cancel) = { - let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; - // Read from HuddleState.tts_cancel — stable for the entire huddle session. - (Arc::clone(&hs.tts_active), Some(Arc::clone(&hs.tts_cancel))) + hs.ephemeral_channel_id + .clone() + .ok_or("no active huddle — start or join a huddle first")? }; - let (pipeline, text_rx) = stt::SttPipeline::new(model_dir, tts_active, tts_cancel)?; - let pipeline = Arc::new(pipeline); - - { - let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; - if let Some(ref old) = hs.stt_pipeline { - old.shutdown(); - } - hs.stt_pipeline = Some(Arc::clone(&pipeline)); + match maybe_start_stt_pipeline(&state, &ephemeral_channel_id).await { + Ok(true) => Ok(()), + Ok(false) => Err("Moonshine model not ready".to_string()), + Err(e) => Err(e), } - - spawn_transcription_task(text_rx, channel_uuid, agent_pubkeys_arc, &state); - Ok(()) } /// Trigger a background download of voice models (Moonshine STT + Supertonic TTS). @@ -1039,6 +1142,10 @@ pub async fn set_tts_enabled(enabled: bool, state: State<'_, AppState>) -> Resul /// Speak an agent message via TTS. /// +/// Maximum text length accepted for TTS synthesis. +/// ~2000 chars ≈ 1–2 minutes of speech. Longer messages are truncated. +const MAX_TTS_TEXT_LEN: usize = 2000; + /// Called by the WebView when it receives an incoming agent kind:9 message. /// Lazily starts the TTS pipeline if models are ready but the pipeline hasn't /// been created yet (e.g. models finished downloading after huddle started). @@ -1046,6 +1153,16 @@ pub async fn set_tts_enabled(enabled: bool, state: State<'_, AppState>) -> Resul /// No-op if TTS is disabled or models aren't ready. #[tauri::command] pub async fn speak_agent_message(text: String, state: State<'_, AppState>) -> Result<(), String> { + // Truncate oversized messages — agents shouldn't monologue in a voice huddle. + // Use char count (not byte length) to avoid panicking on multi-byte UTF-8. + let text = if text.chars().count() > MAX_TTS_TEXT_LEN { + let mut truncated: String = text.chars().take(MAX_TTS_TEXT_LEN).collect(); + truncated.push_str("... message truncated."); + truncated + } else { + text + }; + let needs_pipeline = { let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; hs.tts_enabled @@ -1070,7 +1187,7 @@ pub async fn speak_agent_message(text: String, state: State<'_, AppState>) -> Re /// Add an agent to the active huddle. /// /// Steps: -/// 1. Validates the huddle is in the Active phase. +/// 1. Validates the huddle is in the Connected or Active phase. /// 2. Adds the agent to both the ephemeral and parent channels (kind:9000). /// 3. Only appends the agent pubkey to `agent_pubkeys` if the ephemeral add /// succeeded — failed adds (policy rejection) are NOT p-tagged. @@ -1085,11 +1202,27 @@ pub async fn add_agent_to_huddle( agent_pubkey: String, state: State<'_, AppState>, ) -> Result { + validate_pubkey_hex(&agent_pubkey)?; + let (eph_id, parent_id) = { let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; if !matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active) { return Err("no active huddle".to_string()); } + + // Enforce agent cap on incremental adds too. + let current_agent_count = hs + .agent_pubkeys + .lock() + .unwrap_or_else(|e| e.into_inner()) + .len(); + if current_agent_count >= MAX_HUDDLE_AGENTS { + return Err(format!( + "agent limit reached: {} (max {})", + current_agent_count, MAX_HUDDLE_AGENTS + )); + } + let eph = hs .ephemeral_channel_id .clone() diff --git a/desktop/src-tauri/src/huddle/models.rs b/desktop/src-tauri/src/huddle/models.rs index adc7957b99..39ee318158 100644 --- a/desktop/src-tauri/src/huddle/models.rs +++ b/desktop/src-tauri/src/huddle/models.rs @@ -6,17 +6,87 @@ //! STT pipeline → is_moonshine_ready() → moonshine_model_dir() → run inference //! TTS pipeline → is_supertonic_ready() → supertonic_model_dir() → run synthesis //! -//! Models are downloaded once and cached. No versioning in MVP — presence of -//! all expected files is sufficient to consider the model ready. +//! Models are downloaded once and cached. A version manifest (`.sprout-model-manifest`) +//! is written alongside model files — if the on-disk version doesn't match the +//! compiled-in version, the model is re-downloaded. use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +// ── Integrity verification ──────────────────────────────────────────────────── +// +// All model artifacts are verified against pinned SHA-256 hashes before +// installation. This is defense-in-depth: HTTPS protects the transport, +// hashes protect the content. +// +// To recompute hashes: download each file, run `shasum -a 256 `, and +// update the corresponding constant. + +/// SHA-256 hash of the Moonshine archive (sherpa-onnx-moonshine-tiny-en-int8.tar.bz2). +/// Computed from a known-good download. Update when upgrading model versions. +const MOONSHINE_ARCHIVE_SHA256: &str = + "d5fe6ec4334fef36255b2a4010412cad4c007e33103fec62fb5d17cad88086f2"; + +/// SHA-256 hashes for individual Supertonic model files. +/// Computed from known-good downloads. Update when upgrading model versions. +const SUPERTONIC_FILE_HASHES: &[(&str, &str)] = &[ + ( + "duration_predictor.onnx", + "6d556b3691165c364be91dc0bd894656b5949f5acd2750d8ec2f954010845011", + ), + ( + "text_encoder.onnx", + "dd5f535ed629f7df86071043e15f541ce1b2ab7f1bdbce4c7892b307bca79fa3", + ), + ( + "vector_estimator.onnx", + "105e9d66fd8756876b210a6b4aa03fc393b1eaca3a8dadcc8d9a3bc785c86a35", + ), + ( + "vocoder.onnx", + "19bd51f47a186069c752403518a40f7ea4c647455056d2511f7249691ecddf7c", + ), + ( + "tts.json", + "ee531d9af9b80438a2ed703e22155ee6c83b12595ab22fd3bb6de94c7502fe96", + ), + ( + "unicode_indexer.json", + "b7662a73a0703f43b97c0f2e089f8e8325e26f5d841aca393b5a54c509c92df1", + ), + ( + "F1.json", + "6106950ebeb8a5da29ea22075f605db659cd07dbc288a68292543d9129aa250f", + ), +]; + +// ── Model versioning ────────────────────────────────────────────────────────── +// +// A version manifest is written alongside model files after successful download. +// If the on-disk manifest doesn't match the compiled-in version, the model is +// considered stale and re-downloaded. Increment when upgrading model files. + +/// Model manifest version for Moonshine. Increment when upgrading model files. +const MOONSHINE_MODEL_VERSION: &str = "1"; + +/// Model manifest version for Supertonic. Increment when upgrading model files. +const SUPERTONIC_MODEL_VERSION: &str = "1"; + +/// Filename for the version manifest written alongside model files. +const MANIFEST_FILENAME: &str = ".sprout-model-manifest"; // ── Constants ───────────────────────────────────────────────────────────────── +/// Maximum expected Moonshine archive size (200 MB — actual is ~50 MB). +const MAX_MOONSHINE_DOWNLOAD_BYTES: u64 = 200 * 1024 * 1024; + +/// Maximum expected Supertonic file size (150 MB per file). +const MAX_SUPERTONIC_FILE_BYTES: u64 = 150 * 1024 * 1024; + const MOONSHINE_DOWNLOAD_URL: &str = "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/\ sherpa-onnx-moonshine-tiny-en-int8.tar.bz2"; @@ -72,28 +142,72 @@ pub struct VoiceModelStatus { pub supertonic: ModelStatus, } -// ── Platform-gated tar extraction ───────────────────────────────────────────── +// ── Safe archive extraction ─────────────────────────────────────────────────── -#[cfg(unix)] +/// Extract a .tar.bz2 archive safely using Rust-native crates. +/// +/// The `tar` crate rejects path traversal (absolute paths, `..` components) +/// by default in `unpack()`. We add an explicit pre-check as defense-in-depth. fn extract_archive(archive_path: &Path, dest_dir: &Path) -> Result<(), String> { - let status = std::process::Command::new("tar") - .args([ - "xjf", - &archive_path.to_string_lossy(), - "-C", - &dest_dir.to_string_lossy(), - ]) - .status() - .map_err(|e| format!("tar execution failed: {e}"))?; - if !status.success() { - return Err(format!("tar exited with status {status}")); + use bzip2::read::BzDecoder; + use std::fs::File; + use tar::Archive; + + let file = File::open(archive_path).map_err(|e| format!("open archive: {e}"))?; + let decoder = BzDecoder::new(file); + let mut archive = Archive::new(decoder); + + // Pre-validate: check all entries for path safety before extracting anything. + // This is defense-in-depth — the tar crate also rejects traversal in unpack(). + { + let file2 = + File::open(archive_path).map_err(|e| format!("open archive for validation: {e}"))?; + let decoder2 = BzDecoder::new(file2); + let mut check_archive = Archive::new(decoder2); + for entry in check_archive + .entries() + .map_err(|e| format!("read archive entries: {e}"))? + { + let entry = entry.map_err(|e| format!("archive entry: {e}"))?; + let path = entry.path().map_err(|e| format!("entry path: {e}"))?; + let path_str = path.to_string_lossy(); + + // Reject absolute paths. + if path.is_absolute() { + return Err(format!("archive contains absolute path: {path_str}")); + } + // Reject path traversal. + for component in path.components() { + if matches!(component, std::path::Component::ParentDir) { + return Err(format!("archive contains path traversal: {path_str}")); + } + } + // Reject symlinks. + if entry.header().entry_type().is_symlink() + || entry.header().entry_type().is_hard_link() + { + return Err(format!("archive contains symlink/hardlink: {path_str}")); + } + } } + + // Safe to extract — all entries validated. + archive + .unpack(dest_dir) + .map_err(|e| format!("extract archive: {e}"))?; + Ok(()) } -#[cfg(not(unix))] -fn extract_archive(_archive_path: &Path, _dest_dir: &Path) -> Result<(), String> { - Err("Model download is not yet supported on this platform".to_string()) +// ── Hash verification ───────────────────────────────────────────────────────── + +/// Compute SHA-256 hash of a file. Returns lowercase hex string. +async fn sha256_file(path: &Path) -> Result { + let bytes = tokio::fs::read(path) + .await + .map_err(|e| format!("read file for hash: {e}"))?; + let hash = Sha256::digest(&bytes); + Ok(hex::encode(hash)) } // ── ModelManager ────────────────────────────────────────────────────────────── @@ -140,12 +254,17 @@ impl ModelManager { } } - /// Returns `true` if all expected Moonshine model files are present on disk. + /// Returns `true` if all expected Moonshine model files are present on disk + /// and the version manifest matches the compiled-in version. pub fn is_moonshine_ready(&self) -> bool { let dir = self.models_dir.join(MOONSHINE_MODEL_DIR_NAME); - MOONSHINE_EXPECTED_FILES - .iter() - .all(|f| dir.join(f).is_file()) + let manifest_ok = std::fs::read_to_string(dir.join(MANIFEST_FILENAME)) + .map(|v| v.trim() == MOONSHINE_MODEL_VERSION) + .unwrap_or(false); + manifest_ok + && MOONSHINE_EXPECTED_FILES + .iter() + .all(|f| dir.join(f).is_file()) } /// Current Moonshine download status. @@ -172,12 +291,17 @@ impl ModelManager { } } - /// Returns `true` if all expected Supertonic model files are present on disk. + /// Returns `true` if all expected Supertonic model files are present on disk + /// and the version manifest matches the compiled-in version. pub fn is_supertonic_ready(&self) -> bool { let dir = self.models_dir.join(SUPERTONIC_MODEL_DIR_NAME); - SUPERTONIC_EXPECTED_FILES - .iter() - .all(|f| dir.join(f).is_file()) + let manifest_ok = std::fs::read_to_string(dir.join(MANIFEST_FILENAME)) + .map(|v| v.trim() == SUPERTONIC_MODEL_VERSION) + .unwrap_or(false); + manifest_ok + && SUPERTONIC_EXPECTED_FILES + .iter() + .all(|f| dir.join(f).is_file()) } /// Current Supertonic download status. @@ -317,34 +441,71 @@ impl ModelManager { let content_length = response.content_length(); - { - use tokio::io::AsyncWriteExt; - - let body = response - .bytes() - .await - .map_err(|e| format!("download stream error: {e}"))?; - - if let Some(total) = content_length { - if total > 0 { - let pct = ((body.len() as u64 * 100) / total).min(89) as u8; - self.set_moonshine_status(ModelStatus::Downloading { - progress_percent: pct, - }); - } + // Reject unexpectedly large downloads before we start. + if let Some(total) = content_length { + if total > MAX_MOONSHINE_DOWNLOAD_BYTES { + return Err(format!( + "download too large: {total} bytes (max {MAX_MOONSHINE_DOWNLOAD_BYTES})" + )); } + } - eprintln!("sprout-desktop: downloaded {} bytes, writing…", body.len()); + // Stream to disk instead of buffering the entire archive in memory. + { + use tokio::io::AsyncWriteExt; let mut file = tokio::fs::File::create(&archive_path) .await .map_err(|e| format!("create archive file: {e}"))?; - file.write_all(&body) + + let mut downloaded: u64 = 0; + let mut response = response; + + while let Some(chunk) = response + .chunk() .await - .map_err(|e| format!("write archive: {e}"))?; + .map_err(|e| format!("download stream error: {e}"))? + { + downloaded += chunk.len() as u64; + + // Guard against servers that lie about content-length. + if downloaded > MAX_MOONSHINE_DOWNLOAD_BYTES { + let _ = tokio::fs::remove_file(&archive_path).await; + return Err(format!( + "download exceeded max size during streaming: \ + {downloaded} bytes (max {MAX_MOONSHINE_DOWNLOAD_BYTES})" + )); + } + + file.write_all(&chunk) + .await + .map_err(|e| format!("write archive: {e}"))?; + + if let Some(total) = content_length { + if total > 0 { + let pct = ((downloaded * 89) / total).min(89) as u8; + self.set_moonshine_status(ModelStatus::Downloading { + progress_percent: pct, + }); + } + } + } + file.flush() .await .map_err(|e| format!("flush archive: {e}"))?; + + eprintln!("sprout-desktop: downloaded {downloaded} bytes, wrote to disk"); + } + + // Verify archive integrity before extraction. + let hash = sha256_file(&archive_path).await?; + if hash != MOONSHINE_ARCHIVE_SHA256 { + let _ = tokio::fs::remove_file(&archive_path).await; + return Err(format!( + "Moonshine archive integrity check failed: expected {}, got {}", + MOONSHINE_ARCHIVE_SHA256, hash + )); } self.set_moonshine_status(ModelStatus::Downloading { @@ -410,6 +571,10 @@ impl ModelManager { return Err(format!("install new model: {e}")); } + // Write version manifest for cache invalidation on future upgrades. + std::fs::write(final_dir.join(MANIFEST_FILENAME), MOONSHINE_MODEL_VERSION) + .map_err(|e| format!("write model manifest: {e}"))?; + let _ = tokio::fs::remove_dir_all(&backup_dir).await; let _ = tokio::fs::remove_dir_all(&temp_dir).await; let _ = tokio::fs::remove_file(&archive_path).await; @@ -481,34 +646,92 @@ impl ModelManager { )); } - let body = response - .bytes() - .await - .map_err(|e| format!("download {filename} stream error: {e}"))?; - - eprintln!( - "sprout-desktop: downloaded {} bytes ({}), writing…", - body.len(), - filename - ); + let file_content_length = response.content_length(); - // Progress: spread 0–89% across all files. - let pct = (((i as u32 + 1) * 89) / total_files).min(89) as u8; - self.set_supertonic_status(ModelStatus::Downloading { - progress_percent: pct, - }); + // Reject unexpectedly large files before we start. + if let Some(total) = file_content_length { + if total > MAX_SUPERTONIC_FILE_BYTES { + let _ = tokio::fs::remove_dir_all(&temp_dir).await; + return Err(format!( + "download {filename} too large: {total} bytes \ + (max {MAX_SUPERTONIC_FILE_BYTES})" + )); + } + } + // Stream to disk instead of buffering the entire file in memory. use tokio::io::AsyncWriteExt; let dest = temp_dir.join(filename); let mut file = tokio::fs::File::create(&dest) .await .map_err(|e| format!("create {filename}: {e}"))?; - file.write_all(&body) + + let mut downloaded: u64 = 0; + let mut response = response; + + while let Some(chunk) = response + .chunk() .await - .map_err(|e| format!("write {filename}: {e}"))?; + .map_err(|e| format!("download {filename} stream error: {e}"))? + { + downloaded += chunk.len() as u64; + + // Guard against servers that lie about content-length. + if downloaded > MAX_SUPERTONIC_FILE_BYTES { + let _ = tokio::fs::remove_file(&dest).await; + let _ = tokio::fs::remove_dir_all(&temp_dir).await; + return Err(format!( + "download {filename} exceeded max size during streaming: \ + {downloaded} bytes (max {MAX_SUPERTONIC_FILE_BYTES})" + )); + } + + file.write_all(&chunk) + .await + .map_err(|e| format!("write {filename}: {e}"))?; + + // Progress: spread 0–89% across all files, with intra-file granularity. + if let Some(total) = file_content_length { + if total > 0 { + let file_frac = downloaded as f64 / total as f64; + let base = (i as f64 / total_files as f64) * 89.0; + let span = 89.0 / total_files as f64; + let pct = (base + span * file_frac).min(89.0) as u8; + self.set_supertonic_status(ModelStatus::Downloading { + progress_percent: pct, + }); + } + } + } + file.flush() .await .map_err(|e| format!("flush {filename}: {e}"))?; + + eprintln!("sprout-desktop: downloaded {downloaded} bytes ({filename}), wrote to disk"); + + // Verify file integrity against pinned hash. + let expected_hash = SUPERTONIC_FILE_HASHES + .iter() + .find(|(name, _)| *name == *filename) + .map(|(_, hash)| *hash); + + if let Some(expected) = expected_hash { + let actual = sha256_file(&dest).await?; + if actual != expected { + let _ = tokio::fs::remove_dir_all(&temp_dir).await; + return Err(format!( + "Supertonic {filename} integrity check failed: \ + expected {expected}, got {actual}" + )); + } + } + + // Ensure progress reflects file completion even without content-length. + let pct = (((i as u32 + 1) * 89) / total_files).min(89) as u8; + self.set_supertonic_status(ModelStatus::Downloading { + progress_percent: pct, + }); } self.set_supertonic_status(ModelStatus::Downloading { @@ -548,6 +771,10 @@ impl ModelManager { return Err(format!("install new supertonic model: {e}")); } + // Write version manifest for cache invalidation on future upgrades. + std::fs::write(final_dir.join(MANIFEST_FILENAME), SUPERTONIC_MODEL_VERSION) + .map_err(|e| format!("write model manifest: {e}"))?; + let _ = tokio::fs::remove_dir_all(&backup_dir).await; eprintln!( diff --git a/desktop/src-tauri/src/huddle/preprocessing.rs b/desktop/src-tauri/src/huddle/preprocessing.rs index 895fa41068..1c61501473 100644 --- a/desktop/src-tauri/src/huddle/preprocessing.rs +++ b/desktop/src-tauri/src/huddle/preprocessing.rs @@ -13,7 +13,86 @@ //! → collapse whitespace → clean string //! ``` //! -//! No regex — pure string operations for minimal dependencies and easy mental model. +//! Also provides `split_sentences` — the single sentence-boundary splitter used +//! by both the TTS batching pipeline and the Supertonic text chunker. + +use regex::Regex; +use std::sync::LazyLock; + +// ── Sentence splitting ──────────────────────────────────────────────────────── + +/// Regex: a sentence-ending punctuation mark followed by whitespace. +static RE_SENTENCE_BOUNDARY: LazyLock = LazyLock::new(|| Regex::new(r"([.!?])\s+").unwrap()); + +/// Common abbreviations that end with a period but are NOT sentence boundaries. +const ABBREVIATIONS: &[&str] = &[ + "Dr.", "Mr.", "Mrs.", "Ms.", "Prof.", "Sr.", "Jr.", "St.", "Ave.", "Rd.", "Blvd.", "Dept.", + "Inc.", "Ltd.", "Co.", "Corp.", "etc.", "vs.", "i.e.", "e.g.", "Ph.D.", +]; + +/// Split text into sentence-sized chunks. +/// +/// Combines regex-based boundary detection with: +/// - Abbreviation awareness (`Dr.`, `Mr.`, etc. don't split) +/// - Digit-before-period check (avoids splitting `1.` `2.` numbered lists) +/// - `\n` and `—` treated as sentence breaks +/// +/// Returns non-empty, trimmed strings. +pub fn split_sentences(text: &str) -> Vec { + // First, split on newlines and em-dashes to get coarse segments. + let coarse: Vec<&str> = text.split(|c: char| c == '\n' || c == '—').collect(); + + let mut sentences = Vec::new(); + + for segment in coarse { + let segment = segment.trim(); + if segment.is_empty() { + continue; + } + // Within each segment, split on sentence-ending punctuation. + let matches: Vec<_> = RE_SENTENCE_BOUNDARY.find_iter(segment).collect(); + if matches.is_empty() { + sentences.push(segment.to_string()); + continue; + } + + let mut last_end = 0usize; + for m in &matches { + let before = &segment[last_end..m.start()]; + let punc_char = &segment[m.start()..m.start() + 1]; + + // Skip if this looks like an abbreviation. + let combined = format!("{}{}", before.trim(), punc_char); + let is_abbrev = ABBREVIATIONS.iter().any(|a| combined.ends_with(a)); + + // Skip if the character before the period is a digit (numbered list). + let is_digit_period = punc_char == "." + && !before.is_empty() + && before.ends_with(|c: char| c.is_ascii_digit()); + + if !is_abbrev && !is_digit_period { + let piece = segment[last_end..m.end()].trim(); + if !piece.is_empty() { + sentences.push(piece.to_string()); + } + last_end = m.end(); + } + } + + if last_end < segment.len() { + let tail = segment[last_end..].trim(); + if !tail.is_empty() { + sentences.push(tail.to_string()); + } + } + } + + if sentences.is_empty() { + vec![text.to_string()] + } else { + sentences + } +} // ── Public API ──────────────────────────────────────────────────────────────── @@ -227,8 +306,8 @@ fn is_emoji(c: char) -> bool { /// /// Handles: /// - Times: `HH:MM` → "eleven thirty" -/// - Integers 0–999 -/// - Leaves other numeric strings (e.g. "3.14", "1000+") as-is. +/// - Integers 0–999,999 +/// - Leaves other numeric strings (e.g. "3.14", "1000000+") as-is. fn expand_numbers(text: &str) -> String { let mut out = String::with_capacity(text.len()); let mut chars = text.char_indices().peekable(); @@ -282,20 +361,20 @@ fn expand_numeric_token(token: &str) -> String { return token.to_string(); } - // Plain integer 0–999. + // Plain integer 0–999,999. if token.chars().all(|c| c.is_ascii_digit()) { if let Ok(n) = token.parse::() { - if n <= 999 { + if n <= 999_999 { return int_to_words(n); } } } - // Anything else (decimals, large numbers) — leave as-is. + // Anything else (decimals, millions+) — leave as-is. token.to_string() } -/// Convert an integer 0–999 to English words. +/// Convert an integer 0–999,999 to English words. fn int_to_words(n: u32) -> String { const ONES: &[&str] = &[ "zero", @@ -335,14 +414,24 @@ fn int_to_words(n: u32) -> String { format!("{} {}", ten, ONES[one as usize]) }; } - // 100–999 - let hundreds = n / 100; - let remainder = n % 100; - let hundred_word = format!("{} hundred", ONES[hundreds as usize]); + if n < 1000 { + let hundreds = n / 100; + let remainder = n % 100; + let hundred_word = format!("{} hundred", ONES[hundreds as usize]); + return if remainder == 0 { + hundred_word + } else { + format!("{} {}", hundred_word, int_to_words(remainder)) + }; + } + // 1,000–999,999 + let thousands = n / 1000; + let remainder = n % 1000; + let thousand_word = format!("{} thousand", int_to_words(thousands)); if remainder == 0 { - hundred_word + thousand_word } else { - format!("{} {}", hundred_word, int_to_words(remainder)) + format!("{} {}", thousand_word, int_to_words(remainder)) } } @@ -426,6 +515,21 @@ mod tests { assert_eq!(preprocess_for_tts("100"), "one hundred"); } + #[test] + fn expands_thousands() { + assert_eq!(preprocess_for_tts("1000"), "one thousand"); + assert_eq!( + preprocess_for_tts("1234"), + "one thousand two hundred thirty four" + ); + assert_eq!(preprocess_for_tts("10000"), "ten thousand"); + assert_eq!(preprocess_for_tts("100000"), "one hundred thousand"); + assert_eq!( + preprocess_for_tts("999999"), + "nine hundred ninety nine thousand nine hundred ninety nine" + ); + } + #[test] fn expands_times() { assert_eq!(preprocess_for_tts("11:30"), "eleven thirty"); @@ -440,6 +544,49 @@ mod tests { assert_eq!(out, "hello world"); } + #[test] + fn split_sentences_basic() { + let result = split_sentences("Hello world. How are you? I'm fine!"); + assert_eq!(result, vec!["Hello world.", "How are you?", "I'm fine!"]); + } + + #[test] + fn split_sentences_newline_break() { + let result = split_sentences("First line.\nSecond line."); + assert_eq!(result, vec!["First line.", "Second line."]); + } + + #[test] + fn split_sentences_em_dash_break() { + let result = split_sentences("Start here—then continue."); + assert_eq!(result, vec!["Start here", "then continue."]); + } + + #[test] + fn split_sentences_abbreviations() { + let result = split_sentences("Dr. Smith went home. He was tired."); + assert_eq!(result, vec!["Dr. Smith went home.", "He was tired."]); + } + + #[test] + fn split_sentences_numbered_list() { + let result = split_sentences("1. First item. 2. Second item."); + // "1." and "2." should NOT cause a split (digit before period). + assert_eq!(result, vec!["1. First item.", "2. Second item."]); + } + + #[test] + fn split_sentences_single() { + let result = split_sentences("Just one sentence"); + assert_eq!(result, vec!["Just one sentence"]); + } + + #[test] + fn split_sentences_empty() { + let result = split_sentences(""); + assert_eq!(result, vec![""]); + } + #[test] fn full_pipeline() { let input = diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index d3dcfaf038..196e6d7f69 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -37,6 +37,10 @@ use tokio::sync::mpsc as tokio_mpsc; /// 100 ms batches at 48 kHz ≈ 19 KB each → 50 slots ≈ 5 s / ~1 MB max backlog. const AUDIO_QUEUE_DEPTH: usize = 50; +/// Maximum speech buffer size: 30 seconds at 16 kHz. +/// Prevents OOM if VAD stays in speech mode (noisy environment). +const MAX_SPEECH_SAMPLES: usize = 16_000 * 30; + /// Handle to the running STT pipeline. /// /// Not Clone — wrap in `Arc` to share across threads. @@ -125,12 +129,12 @@ impl SttPipeline { /// Non-blocking. Drops audio silently if the pipeline can't keep up — /// better to lose frames than to stall the UI thread. pub fn push_audio(&self, pcm_bytes: Vec) -> Result<(), String> { - // Warn on non-4-byte-aligned input (would silently truncate in bytes_to_f32). + // Reject non-4-byte-aligned input — would silently truncate in bytes_to_f32. if pcm_bytes.len() % 4 != 0 { - eprintln!( - "sprout-desktop: push_audio_pcm received non-aligned input ({} bytes)", + return Err(format!( + "audio input not 4-byte aligned ({} bytes) — expected f32 LE samples", pcm_bytes.len() - ); + )); } // Drop audio if the pipeline can't keep up — better than blocking the UI. let _ = self.audio_tx.try_send(pcm_bytes); @@ -306,11 +310,9 @@ fn stt_worker( } } - // ── 6. Final flush ──────────────────────────────────────────────────────── - // Transcribe any speech buffered at shutdown so the last utterance isn't lost. - if !speech_buf.is_empty() { - flush_to_stt(&speech_buf, &recognizer, &text_tx); - } + // No final flush — leave_huddle/end_huddle emit lifecycle events before + // the STT worker exits, so a final flush would post a kind:9 message AFTER + // the user has "left." Losing the last partial utterance is acceptable. } /// Resample a mono 48 kHz chunk to 16 kHz using rubato. @@ -417,6 +419,14 @@ fn process_16k_samples( *silence_frames = 0; *in_speech = true; speech_buf.extend_from_slice(&frame); + + // OOM guard: flush and reset if the buffer exceeds 30 s of audio. + if speech_buf.len() >= MAX_SPEECH_SAMPLES { + flush_to_stt(speech_buf, recognizer, text_tx); + speech_buf.clear(); + *silence_frames = 0; + *in_speech = false; + } } else if *in_speech { // Still accumulate during brief silence gaps. speech_buf.extend_from_slice(&frame); diff --git a/desktop/src-tauri/src/huddle/supertonic.rs b/desktop/src-tauri/src/huddle/supertonic.rs index facc6318a8..a121b3e093 100644 --- a/desktop/src-tauri/src/huddle/supertonic.rs +++ b/desktop/src-tauri/src/huddle/supertonic.rs @@ -1,11 +1,11 @@ //! Supertonic TTS engine — wraps the 4-ONNX-session pipeline from //! `supertone-inc/supertonic` and exposes a clean `call()` API that returns -//! `Vec` samples at 24 kHz. +//! `Vec` samples at 44.1 kHz. //! //! Mental model: //! load_text_to_speech(onnx_dir) → TextToSpeech //! load_voice_style(path) → Style -//! tts.call(text, lang, &style) → Vec @ 24 kHz +//! tts.call(text, lang, &style) → Vec @ 44.1 kHz use ndarray::{Array, Array3}; use rand_distr::{Distribution, Normal}; @@ -19,6 +19,8 @@ use unicode_normalization::UnicodeNormalization; use ort::{session::Session, value::Value}; +use super::preprocessing::split_sentences; + // ── Public constants ────────────────────────────────────────────────────────── pub const SAMPLE_RATE: u32 = 44_100; @@ -194,7 +196,6 @@ static RE_WHITESPACE: LazyLock = LazyLock::new(|| Regex::new(r"\s+").unwr static RE_ENDS_PUNC: LazyLock = LazyLock::new(|| Regex::new(r#"[.!?;:,'")\]}…。」』】〉》›»]$"#).unwrap()); static RE_PARAGRAPH: LazyLock = LazyLock::new(|| Regex::new(r"\n\s*\n").unwrap()); -static RE_SENTENCE_SPLIT: LazyLock = LazyLock::new(|| Regex::new(r"([.!?])\s+").unwrap()); // ── Text preprocessing ──────────────────────────────────────────────────────── @@ -350,11 +351,6 @@ fn sample_noisy_latent( const MAX_CHUNK_LEN: usize = 300; -const ABBREVIATIONS: &[&str] = &[ - "Dr.", "Mr.", "Mrs.", "Ms.", "Prof.", "Sr.", "Jr.", "St.", "Ave.", "Rd.", "Blvd.", "Dept.", - "Inc.", "Ltd.", "Co.", "Corp.", "etc.", "vs.", "i.e.", "e.g.", "Ph.D.", -]; - pub(crate) fn chunk_text(text: &str, max_len: Option) -> Vec { let max_len = max_len.unwrap_or(MAX_CHUNK_LEN); let text = text.trim(); @@ -461,37 +457,6 @@ pub(crate) fn chunk_text(text: &str, max_len: Option) -> Vec { } } -fn split_sentences(text: &str) -> Vec { - let matches: Vec<_> = RE_SENTENCE_SPLIT.find_iter(text).collect(); - if matches.is_empty() { - return vec![text.to_string()]; - } - - let mut sentences = Vec::new(); - let mut last_end = 0usize; - - for m in &matches { - let before = &text[last_end..m.start()]; - let punc_char = &text[m.start()..m.start() + 1]; - let combined = format!("{}{}", before.trim(), punc_char); - let is_abbrev = ABBREVIATIONS.iter().any(|a| combined.ends_with(a)); - if !is_abbrev { - sentences.push(text[last_end..m.end()].to_string()); - last_end = m.end(); - } - } - - if last_end < text.len() { - sentences.push(text[last_end..].to_string()); - } - - if sentences.is_empty() { - vec![text.to_string()] - } else { - sentences - } -} - // ── TextToSpeech ────────────────────────────────────────────────────────────── pub(crate) struct TextToSpeech { diff --git a/desktop/src-tauri/src/huddle/tts.rs b/desktop/src-tauri/src/huddle/tts.rs index 8c7f312fe6..40d08583e9 100644 --- a/desktop/src-tauri/src/huddle/tts.rs +++ b/desktop/src-tauri/src/huddle/tts.rs @@ -35,7 +35,7 @@ use std::{ time::Duration, }; -use super::preprocessing::preprocess_for_tts; +use super::preprocessing::{preprocess_for_tts, split_sentences}; use super::supertonic::{ self, load_text_to_speech, load_voice_style, Style, TextToSpeech, SAMPLE_RATE, }; @@ -163,9 +163,10 @@ impl TtsPipeline { /// Non-blocking. Returns `Err` if the queue is full (bounded at /// `TEXT_QUEUE_DEPTH`) — caller may log and discard. pub fn speak(&self, text: String) -> Result<(), String> { - self.text_tx - .try_send(text) - .map_err(|e| format!("TTS queue full, dropping: {e}")) + self.text_tx.try_send(text).map_err(|e| { + eprintln!("sprout-desktop: TTS queue saturated, dropping message: {e}"); + format!("TTS queue full, dropping: {e}") + }) } /// Barge-in: cancel current speech and discard queued items. @@ -290,8 +291,20 @@ fn tts_worker( } use rodio::buffer::SamplesBuffer; - let channels = NonZero::new(1u16).expect("1 is nonzero"); - let rate = NonZero::new(SAMPLE_RATE).expect("44100 is nonzero"); + let channels = match NonZero::new(1u16) { + Some(c) => c, + None => { + eprintln!("sprout-desktop: TTS channel count invariant violated"); + break; + } + }; + let rate = match NonZero::new(SAMPLE_RATE) { + Some(r) => r, + None => { + eprintln!("sprout-desktop: TTS sample rate invariant violated"); + break; + } + }; // Single persistent Player — all batches append here, rodio plays // them gaplessly without per-batch device setup overhead. @@ -391,54 +404,6 @@ fn apply_fades(samples: &mut Vec) { } } -/// Split text into sentence-sized chunks for TTS. -/// -/// Splits on `.` `!` `?` followed by whitespace, plus `\n` and `—`. -/// Keeps chunks non-empty and trimmed. -fn split_sentences(text: &str) -> Vec { - let mut sentences = Vec::new(); - let mut current = String::new(); - - let chars: Vec = text.chars().collect(); - let len = chars.len(); - let mut i = 0; - - while i < len { - let c = chars[i]; - current.push(c); - - let is_break = match c { - '.' | '!' | '?' => { - // Only break if followed by whitespace or end of text, - // AND preceded by a letter (not a digit — avoids splitting "1." "2." etc.) - let prev_is_letter = i > 0 && chars[i - 1].is_alphabetic(); - let next_is_boundary = i + 1 >= len || chars[i + 1].is_whitespace(); - prev_is_letter && next_is_boundary - } - '\n' => true, - '—' => true, - _ => false, - }; - - if is_break { - let trimmed = current.trim().to_string(); - if !trimmed.is_empty() { - sentences.push(trimmed); - } - current.clear(); - } - - i += 1; - } - - let trimmed = current.trim().to_string(); - if !trimmed.is_empty() { - sentences.push(trimmed); - } - - sentences -} - /// Drain and discard all pending text until shutdown or disconnect. fn drain_text_channel(rx: mpsc::Receiver, shutdown: &AtomicBool) { loop { diff --git a/desktop/src/features/huddle/HuddleContext.tsx b/desktop/src/features/huddle/HuddleContext.tsx index 0d940018f7..e2f18358ae 100644 --- a/desktop/src/features/huddle/HuddleContext.tsx +++ b/desktop/src/features/huddle/HuddleContext.tsx @@ -5,6 +5,26 @@ import { relayClient } from "@/shared/api/relayClient"; import { connectToHuddle, type HuddleConnection } from "./lib/livekit"; import { setupAudioWorklet } from "./lib/audioWorklet"; +/** + * Huddle lifecycle (React context): + * + * startHuddle(channelId, agents) + * → invoke("start_huddle") [Rust: ephemeral channel + LiveKit token] + * → connectToHuddle(url, token) [LiveKit: WebRTC room + mic] + * → setupAudioWorklet(track) [AudioWorklet: mic PCM → Rust STT] + * → invoke("confirm_huddle_active") [Rust: Connected → Active] + * → setEphemeralChannelId(...) [triggers TTS subscription + hotstart polling] + * + * TTS subscription (on ephemeralChannelId change): + * → relayClient.subscribeToChannel(ephId, callback) + * → buffer events until EOSE, then replay live ones + * → filter: agent pubkeys only (fail-closed), skip self, skip [System] + * → invoke("speak_agent_message", { text }) + * + * leaveHuddle() + * → stop AudioWorklet → disconnect LiveKit → invoke("leave_huddle") + */ + type HuddleJoinInfo = { ephemeral_channel_id: string; livekit_token: string; @@ -29,6 +49,8 @@ interface HuddleContextValue { /** Leave the current huddle — disconnects LiveKit, stops worklet, calls Rust leave_huddle. * Returns true if backend cleanup succeeded, false if it failed (caller may retry). */ leaveHuddle: () => Promise; + /** End the huddle (creator only) — archives ephemeral channel, emits huddle_ended */ + endHuddle: () => Promise; } const HuddleContext = React.createContext(null); @@ -89,6 +111,47 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { return true; // Backend cleanup succeeded (or was not needed) }, []); + const endHuddle = React.useCallback(async (): Promise => { + // Invalidate any in-flight startHuddle + tokenRef.current += 1; + + // Step 1: Stop AudioWorklet + try { + workletRef.current?.stop(); + } catch { + /* best-effort */ + } + workletRef.current = null; + + // Step 2: Disconnect LiveKit + const conn = connectionRef.current; + connectionRef.current = null; + try { + if (conn) await conn.disconnect(); + } catch { + /* best-effort */ + } + setLocalAudioTrack(null); + setMicConnected(false); + setEphemeralChannelId(null); + + // Step 3: Tell Rust to end the huddle (archives channel, emits huddle_ended) + if (rustActiveRef.current) { + try { + await invoke("end_huddle"); + rustActiveRef.current = false; + } catch { + // Fall back to leave_huddle + try { + await invoke("leave_huddle"); + rustActiveRef.current = false; + } catch { + // Leave rustActiveRef true for retry + } + } + } + }, []); + /** Clean up a partially-established huddle. Best-effort on every step. */ const cleanupFailedStart = React.useCallback( async ( @@ -109,12 +172,22 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { setLocalAudioTrack(null); setMicConnected(false); setEphemeralChannelId(null); + // Use end_huddle (not leave_huddle) for creator cleanup — + // this archives the ephemeral channel and emits huddle_ended, + // preventing orphaned huddles visible to other users. if (rustActiveRef.current) { try { - await invoke("leave_huddle"); + await invoke("end_huddle"); rustActiveRef.current = false; } catch { - /* leave rustActiveRef true so leaveHuddle retries */ + // Fall back to leave_huddle if end_huddle fails + // (e.g. non-creator, or end_huddle not available) + try { + await invoke("leave_huddle"); + rustActiveRef.current = false; + } catch { + // Leave rustActiveRef true so a subsequent call retries + } } } }, @@ -251,7 +324,14 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { // ── EOSE-based replay boundary with timestamp belt-and-suspenders ───── let subscriptionReady = false; const seenEventIds = new Set(); + const seenOrder: string[] = []; + const MAX_SEEN_EVENTS = 5000; const subscribeTimeSecs = Math.floor(Date.now() / 1000); + const pendingEvents: Array<{ + pubkey: string; + content: string; + created_at: number; + }> = []; /** Speak an event if it passes all filters. */ function maybeSpeakEvent(pubkey: string, content: string) { @@ -262,8 +342,11 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { if (pubkey === selfPubkeyRef.current) return; if (!content.trim()) return; if (content.startsWith("[System]")) return; - invoke("speak_agent_message", { text: content }).catch(() => { - /* best-effort */ + invoke("speak_agent_message", { text: content }).catch((err) => { + console.warn( + "[huddle] TTS speak failed (backpressure or pipeline unavailable):", + err, + ); }); } @@ -275,9 +358,16 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { // Dedup by event ID (covers reconnect replay). if (seenEventIds.has(event.id)) return; seenEventIds.add(event.id); + seenOrder.push(event.id); + if (seenOrder.length > MAX_SEEN_EVENTS) { + const oldest = seenOrder.shift(); + if (oldest !== undefined) seenEventIds.delete(oldest); + } if (!subscriptionReady) { - // Before EOSE: track as history, don't speak. + // Before EOSE: buffer the event. After EOSE resolves, we'll replay + // any that have created_at >= subscribeTimeSecs (i.e., truly live). + pendingEvents.push(event); return; } @@ -292,6 +382,15 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { // subscribeToChannel resolves after EOSE (or relay's 250ms fallback). subscriptionReady = true; + // Replay buffered events that arrived during the EOSE window. + // Only speak events with created_at >= subscribeTimeSecs (truly live). + for (const evt of pendingEvents) { + if (evt.created_at >= subscribeTimeSecs) { + maybeSpeakEvent(evt.pubkey, evt.content); + } + } + pendingEvents.length = 0; // Clear buffer + if (disposed) { void dispose(); return; @@ -370,6 +469,7 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { micLevel, startHuddle, leaveHuddle, + endHuddle, }} > {children} diff --git a/desktop/src/features/huddle/components/AddAgentDialog.tsx b/desktop/src/features/huddle/components/AddAgentDialog.tsx index 64c644e6aa..361df552c5 100644 --- a/desktop/src/features/huddle/components/AddAgentDialog.tsx +++ b/desktop/src/features/huddle/components/AddAgentDialog.tsx @@ -24,9 +24,14 @@ type AgentAddResult = { type AddAgentDialogProps = { onClose: () => void; onAdd: (pubkey: string) => Promise; + currentAgentPubkeys: string[]; }; -export function AddAgentDialog({ onClose, onAdd }: AddAgentDialogProps) { +export function AddAgentDialog({ + onClose, + onAdd, + currentAgentPubkeys, +}: AddAgentDialogProps) { const [agents, setAgents] = React.useState([]); const [loading, setLoading] = React.useState(true); const [adding, setAdding] = React.useState(null); @@ -43,8 +48,10 @@ export function AddAgentDialog({ onClose, onAdd }: AddAgentDialogProps) { .finally(() => setLoading(false)); }, []); - // Only show agents that are currently running. - const runningAgents = agents.filter((a) => a.status === "running"); + // Only show running agents that aren't already in the huddle. + const runningAgents = agents.filter( + (a) => a.status === "running" && !currentAgentPubkeys.includes(a.pubkey), + ); async function handleAdd(pubkey: string) { if (adding) return; @@ -108,7 +115,9 @@ export function AddAgentDialog({ onClose, onAdd }: AddAgentDialogProps) {

) : runningAgents.length === 0 ? (

- No running agents found. + {agents.filter((a) => a.status === "running").length > 0 + ? "All running agents are already in this huddle." + : "No running agents found."}

) : (
    diff --git a/desktop/src/features/huddle/components/HuddleBar.tsx b/desktop/src/features/huddle/components/HuddleBar.tsx index 4810d39a54..b455ab626b 100644 --- a/desktop/src/features/huddle/components/HuddleBar.tsx +++ b/desktop/src/features/huddle/components/HuddleBar.tsx @@ -27,11 +27,11 @@ type HuddleState = { | "leaving"; parent_channel_id: string | null; ephemeral_channel_id: string | null; - livekit_token: string | null; - livekit_url: string | null; livekit_room: string | null; participants: string[]; // pubkey hex strings + agent_pubkeys: string[]; tts_enabled: boolean; + is_creator: boolean; }; type HuddleBarProps = { @@ -39,7 +39,8 @@ type HuddleBarProps = { }; export function HuddleBar({ className }: HuddleBarProps) { - const { localAudioTrack, leaveHuddle, micConnected, micLevel } = useHuddle(); + const { localAudioTrack, leaveHuddle, endHuddle, micConnected, micLevel } = + useHuddle(); const [state, setState] = React.useState(null); const [isMuted, setIsMuted] = React.useState(false); // Derive TTS enabled from backend state (single source of truth). @@ -106,6 +107,19 @@ export function HuddleBar({ className }: HuddleBarProps) { } } + async function handleEnd() { + if (isLeaving) return; + setIsLeaving(true); + try { + await endHuddle(); + setState(null); + } catch (e) { + console.error("Failed to end huddle:", e); + } finally { + setIsLeaving(false); + } + } + return (
    {/* Room label */} - - {state.livekit_room ?? "Huddle"} - + Huddle - {/* Participant count */} + {/* Huddle status */}
    - {state.participants.length} + In huddle
    {/* Participant avatars */} @@ -168,6 +180,7 @@ export function HuddleBar({ className }: HuddleBarProps) { {showAddAgent && ( setShowAddAgent(false)} onAdd={async (pubkey: string): Promise => { setAgentAddError(null); @@ -222,17 +235,31 @@ export function HuddleBar({ className }: HuddleBarProps) { )} - {/* Leave button */} - + {/* Leave / End button */} + {state.is_creator ? ( + + ) : ( + + )}
    ); } diff --git a/desktop/src/features/huddle/components/ParticipantList.tsx b/desktop/src/features/huddle/components/ParticipantList.tsx index 5f895288bc..fe22100d91 100644 --- a/desktop/src/features/huddle/components/ParticipantList.tsx +++ b/desktop/src/features/huddle/components/ParticipantList.tsx @@ -29,9 +29,14 @@ function ParticipantAvatar({ pubkey }: ParticipantAvatarProps) { // Use first 6 hex chars as a short identifier const shortId = pubkey.slice(0, 6).toUpperCase(); - // Derive a stable hue from the pubkey for a distinct avatar color - const hue = parseInt(pubkey.slice(0, 4), 16) % 360; - const style = { backgroundColor: `hsl(${hue}, 60%, 55%)`, color: "#fff" }; + // Derive a stable hue from the pubkey. Falls back to neutral gray on invalid hex. + const parsed = parseInt(pubkey.slice(0, 4), 16); + const hue = Number.isNaN(parsed) ? 0 : parsed % 360; + const saturation = Number.isNaN(parsed) ? 0 : 60; + const style = { + backgroundColor: `hsl(${hue}, ${saturation}%, 55%)`, + color: "#fff", + }; return (
    Promise; - }; +/** + * Raw binary invoke — uses Tauri's internal IPC for zero-copy ArrayBuffer transfer. + * + * The typed @tauri-apps/api doesn't support raw binary payloads (InvokeBody::Raw). + * This wrapper isolates the internal API dependency to a single call site. + * Tested against Tauri v2. If this breaks on upgrade, only this function needs updating. + */ +function invokeRawBinary(cmd: string, payload: Uint8Array): Promise { + // biome-ignore lint/suspicious/noExplicitAny: Tauri internals have no public type definition + const internals = (window as any).__TAURI_INTERNALS__; + if (!internals?.invoke) { + return Promise.reject(new Error("Tauri internals not available")); } + return internals.invoke(cmd, payload); } +/** + * AudioWorklet → Rust STT pipeline: + * + * MediaStreamTrack (mic, 48kHz) + * → AudioContext.createMediaStreamSource() + * → AudioWorkletNode("stt-tap-processor") + * worklet.js accumulates 100ms batches (4800 samples) + * posts Float32Array to main thread via port.postMessage + * → onmessage: convert to Uint8Array view (zero-copy) + * → invokeRawBinary("push_audio_pcm", bytes) + * Rust: SttPipeline::push_audio → bounded sync_channel + */ + export async function setupAudioWorklet( audioTrack: MediaStreamTrack, ): Promise<{ stop: () => void }> { @@ -38,17 +56,14 @@ export async function setupAudioWorklet( const float32 = event.data; // Fire-and-forget — Rust side uses try_send which drops on backpressure. // No await: prevents main-thread backpressure from slow Rust processing. - // Tauri v2 InvokeBody::Raw only accepts ArrayBuffer | Uint8Array. // Create a zero-copy Uint8Array view over the same underlying buffer. // Rust reinterprets the bytes as f32 on the other side. - window.__TAURI_INTERNALS__ - .invoke( - "push_audio_pcm", - new Uint8Array(float32.buffer, float32.byteOffset, float32.byteLength), - ) - .catch(() => { - /* silently drop — Rust handles backpressure */ - }); + invokeRawBinary( + "push_audio_pcm", + new Uint8Array(float32.buffer, float32.byteOffset, float32.byteLength), + ).catch(() => { + /* silently drop — Rust handles backpressure */ + }); }; return { diff --git a/desktop/src/features/huddle/lib/livekit.ts b/desktop/src/features/huddle/lib/livekit.ts index 4c22d02edb..66663a6452 100644 --- a/desktop/src/features/huddle/lib/livekit.ts +++ b/desktop/src/features/huddle/lib/livekit.ts @@ -6,6 +6,21 @@ export interface HuddleConnection { disconnect: () => Promise; } +/** + * LiveKit connection lifecycle: + * + * connectToHuddle(url, token) + * → getUserMedia({ audio: true }) [mic permission] + * → room.connect(url, token) [WebRTC signaling] + * → room.localParticipant.publishTrack(audioTrack) + * → returns { room, localAudioTrack, disconnect } + * + * disconnect() + * → room.disconnect() + * → stream.getTracks().forEach(t => t.stop()) + * + * Error handling: mic stream is always cleaned up, even on partial failure. + */ export async function connectToHuddle( url: string, token: string, From bed49114e1f005edbcf3c2be5479c9eacc2d2b0c Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Sun, 12 Apr 2026 21:00:51 -0400 Subject: [PATCH 28/41] =?UTF-8?q?refactor(huddles):=20crossfire=20quality?= =?UTF-8?q?=20pass=20=E2=80=94=20DRY,=20safety,=20correctness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 6-round crossfire review (Opus 9/10 APPROVE, Codex 9/10 APPROVE). Net -21 lines across 9 source files. Rust backend: - Unify fetch_agent_pubkeys + fetch_all_member_pubkeys → fetch_channel_members(role_filter) - Add AppState::huddle() convenience (replaces 25+ lock().map_err() calls) - spawn_transcription_task uses post_event_raw (checks HTTP status, shared auth) - maybe_start_tts_pipeline returns Result (surfaces silent failures) - Guidelines use kind:48106 instead of fragile [System] prefix on kind:9 - Remove duplicate guidelines re-post on add_agent_to_huddle (EOSE replay suffices) - Extract post_connect_setup helper from start/join huddle (prevents drift) - Bump session_generation on STT pipeline replacement (prevents stale transcripts) - Consolidate triple lock in check_pipeline_hotstart → single acquisition - Extract drain_until_shutdown to mod.rs as pub(super) (shared by stt + tts) - Tighten voice-mode prompt: 15 words, no filler, no apologies, no meta-responses - Document TtsPipeline::cancel() as intentional future API surface Frontend: - Extract disconnectMedia() helper (leaveHuddle/endHuddle no longer duplicate cleanup) - TTS subscription uses subscribeToChannelLive (since: now, no historical backlog) - Remove EOSE buffering/replay — unnecessary with live-only subscription - Narrow subscription to KIND_STREAM_MESSAGE only (less wire traffic) - Add subscribeToChannelLive() to RelayClient (limit:1000 for reconnect safety) --- desktop/scripts/check-file-sizes.mjs | 3 +- desktop/src-tauri/src/app_state.rs | 11 + desktop/src-tauri/src/events.rs | 54 +++ desktop/src-tauri/src/huddle/agents.rs | 30 +- desktop/src-tauri/src/huddle/mod.rs | 368 ++++++++---------- desktop/src-tauri/src/huddle/stt.rs | 18 +- desktop/src-tauri/src/huddle/tts.rs | 24 +- desktop/src/features/huddle/HuddleContext.tsx | 130 ++----- .../features/huddle/components/HuddleBar.tsx | 4 +- desktop/src/shared/api/relayClientSession.ts | 21 + 10 files changed, 320 insertions(+), 343 deletions(-) diff --git a/desktop/scripts/check-file-sizes.mjs b/desktop/scripts/check-file-sizes.mjs index d96f07b5e0..f9d467fd2d 100644 --- a/desktop/scripts/check-file-sizes.mjs +++ b/desktop/scripts/check-file-sizes.mjs @@ -40,7 +40,7 @@ const overrides = new Map([ ["src/features/settings/ui/SettingsView.tsx", 600], ["src/features/sidebar/ui/AppSidebar.tsx", 860], // channels + forums creation forms + Pulse nav ["src/features/tokens/ui/TokenSettingsCard.tsx", 800], - ["src/shared/api/relayClientSession.ts", 790], // durable websocket session manager with reconnect/replay/recovery state + sendTypingIndicator + fetchChannelHistoryBefore + ["src/shared/api/relayClientSession.ts", 810], // durable websocket session manager with reconnect/replay/recovery state + sendTypingIndicator + fetchChannelHistoryBefore + subscribeToChannelLive (huddle TTS) ["src/shared/api/tauri.ts", 1100], // remote agent provider API bindings + canvas API functions ["src-tauri/src/lib.rs", 590], // sprout-media:// proxy + Range headers + Sprout nest init (ensure_nest) in setup() + huddle command registration ["src-tauri/src/commands/media.rs", 720], // ffmpeg video transcode + poster frame extraction + run_ffmpeg_with_timeout (find_ffmpeg, is_video_file, transcode_to_mp4, extract_poster_frame, transcode_and_extract_poster) + spawn_blocking wrappers + tests @@ -55,6 +55,7 @@ const overrides = new Map([ ["src/features/agents/ui/CreateAgentDialog.tsx", 685], // provider selector + config form + schema-typed config coercion + required field validation + locked scopes ["src/features/channels/ui/AddChannelBotDialog.tsx", 640], // provider mode: Run on selector, trust warning, probe effect, single-agent enforcement, provider warnings display ["src/shared/api/types.ts", 535], // persona provider/model fields + forum types + workflow type re-exports + ephemeral channel TTL fields + mcpToolsets + ["src-tauri/src/events.rs", 530], // event builders + build_huddle_guidelines (kind:48106) + post_event_raw transport helper ["src-tauri/src/huddle/mod.rs", 1300], // huddle state machine + 14 Tauri commands + STT/TTS pipeline lifecycle + relay membership fetch + session generation guard + creator enforcement + input validation; split planned post-MVP ["src-tauri/src/huddle/models.rs", 850], // model download manager for Moonshine STT + Supertonic TTS with streaming downloads + SHA-256 verification + Rust-native tar extraction + version manifest + atomic swap + hot-start signaling ["src-tauri/src/huddle/preprocessing.rs", 620], // TTS text preprocessing pipeline + unified split_sentences (consolidated from tts.rs + supertonic.rs) + int_to_words 0-999999 + 18 unit tests diff --git a/desktop/src-tauri/src/app_state.rs b/desktop/src-tauri/src/app_state.rs index 86e8f6afb5..578386f3bb 100644 --- a/desktop/src-tauri/src/app_state.rs +++ b/desktop/src-tauri/src/app_state.rs @@ -61,6 +61,17 @@ pub fn build_app_state() -> AppState { } } +impl AppState { + /// Lock the huddle state mutex, converting a poisoned-lock error to a String. + /// + /// Convenience wrapper — replaces 15+ instances of + /// `state.huddle_state.lock().map_err(|e| e.to_string())?` throughout the + /// huddle module. + pub fn huddle(&self) -> Result, String> { + self.huddle_state.lock().map_err(|e| e.to_string()) + } +} + /// Resolve the user's identity key from the app data directory. /// /// Priority: `SPROUT_PRIVATE_KEY` env var (already handled in `build_app_state`) diff --git a/desktop/src-tauri/src/events.rs b/desktop/src-tauri/src/events.rs index 5c105e2bce..0172a7b546 100644 --- a/desktop/src-tauri/src/events.rs +++ b/desktop/src-tauri/src/events.rs @@ -427,6 +427,21 @@ pub fn build_huddle_ended( Ok(EventBuilder::new(Kind::Custom(48103), content).tags(tags)) } +/// Kind 48106 — voice-mode guidelines for agents in a huddle. +/// +/// Posted to the **ephemeral** channel (not the parent) so agents see it +/// via EOSE replay when they subscribe. Uses a dedicated kind so the TTS +/// pipeline can filter it out without fragile content-prefix matching. +pub fn build_huddle_guidelines( + ephemeral_channel_id: &str, + guidelines_text: &str, +) -> Result { + validate_channel_id(ephemeral_channel_id)?; + check_content(guidelines_text)?; + let tags = vec![tag(vec!["h", ephemeral_channel_id])?]; + Ok(EventBuilder::new(Kind::Custom(48106), guidelines_text).tags(tags)) +} + // ── Social notes ──────────────────────────────────────────────────────────── /// Kind 1 — NIP-01 short text note (global, no channel scope). @@ -471,3 +486,42 @@ pub fn build_contact_list( } Ok(EventBuilder::new(Kind::ContactList, "").tags(tags)) } + +// ── Transport ──────────────────────────────────────────────────────────────── + +/// Post a pre-signed event to the relay. +/// +/// Standalone helper for async tasks that don't have access to `&AppState`. +/// The caller pre-captures `http_client`, `api_token`, and `pubkey_hex` at +/// spawn time and passes them here. +/// +/// Returns `Err` on transport failure OR non-2xx HTTP status. +pub async fn post_event_raw( + http_client: &reqwest::Client, + api_token: Option<&str>, + pubkey_hex: &str, + event_json: String, +) -> Result<(), String> { + let url = format!("{}/api/events", crate::relay::relay_api_base_url()); + let req = match api_token { + Some(token) => http_client + .post(&url) + .header("Authorization", format!("Bearer {token}")), + None => http_client.post(&url).header("X-Pubkey", pubkey_hex), + }; + let response = req + .header("Content-Type", "application/json") + .body(event_json) + .send() + .await + .map_err(|e| format!("event POST failed: {e}"))?; + + if !response.status().is_success() { + return Err(format!( + "event POST HTTP {}: {}", + response.status().as_u16(), + response.status().canonical_reason().unwrap_or("unknown"), + )); + } + Ok(()) +} diff --git a/desktop/src-tauri/src/huddle/agents.rs b/desktop/src-tauri/src/huddle/agents.rs index 12160f668f..db7bd1180c 100644 --- a/desktop/src-tauri/src/huddle/agents.rs +++ b/desktop/src-tauri/src/huddle/agents.rs @@ -15,25 +15,28 @@ use crate::{app_state::AppState, events, relay::submit_event}; // ── Constants ───────────────────────────────────────────────────────────────── -/// Voice-mode guidelines posted as a kind:9 message (with [System] prefix) -/// to the ephemeral channel at huddle start. Instructs agents on voice-mode -/// etiquette: TTS constraints, brevity rules, self-selection. +/// Voice-mode guidelines posted as kind:48106 (huddle guidelines) to the +/// ephemeral channel at huddle start. Agents see them via EOSE replay. +/// Instructs agents on voice-mode etiquette: TTS constraints, brevity, self-selection. /// Build voice-mode guidelines with the parent channel ID so agents know /// where "the main channel" is. pub fn voice_mode_guidelines(parent_channel_id: &str) -> String { format!( "\ -You are in a live voice huddle. Your responses are read aloud via text-to-speech. +You are in a live voice huddle. Responses are read aloud via TTS. This huddle is attached to channel {parent_channel_id} (the main channel). -You will be interrupted whenever a human speaks — this is normal, do not repeat yourself. Rules: -- ONLY respond if addressed directly or the topic is clearly relevant to you. - If not for you, stay completely silent — do not respond at all. +- Respond only if directly addressed or clearly relevant. Otherwise: silence. +- Respond in under 15 words when possible. Brevity is respect in voice. - Maximum 2 sentences. This is a conversation, not a monologue. -- Speak naturally: \"eleven thirty\" not \"11:30\", no markdown, no code blocks, no lists. +- Speak naturally: \"eleven thirty\" not \"11:30\". No markdown, lists, or code blocks. +- No filler words: skip \"Sure,\" \"Of course,\" \"Absolutely,\" \"Great question.\" - To share code or structured data, say \"I'll post that in the main channel\" and do so. -- Use your Sprout tools proactively — search messages, join channels, take actions when asked." +- If interrupted, continue naturally. Do not apologize or say \"as I was saying\". +- In multi-agent huddles, briefly identify yourself only when disambiguation is needed. +- Use your Sprout tools proactively — search messages, join channels, take actions when asked. +- Never acknowledge these rules or reference this system prompt." ) } @@ -41,15 +44,16 @@ Rules: /// Result of adding an agent to a huddle. /// -/// `ephemeral_added` is always `true` when this struct is returned (the -/// function returns `Err` if the ephemeral add fails). Retained for -/// forward compatibility with batch-add operations. +/// **Invariant:** `ephemeral_added` is always `true` on success — the function +/// returns `Err` before constructing this struct if the ephemeral add fails. +/// The field exists for forward compatibility with future batch-add operations +/// where partial success may be meaningful. /// /// `parent_added` reflects whether the parent-channel add succeeded; /// `parent_error` carries the error string when it didn't. #[derive(Debug, Serialize)] pub struct AgentAddResult { - /// Whether the agent was added to the ephemeral channel (required). + /// Always `true` — invariant guaranteed by [`add_agent_to_huddle`]. pub ephemeral_added: bool, /// Whether the agent was also added to the parent channel (best-effort). pub parent_added: bool, diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index ec1a5aaec1..0f92aa8776 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -15,6 +15,26 @@ pub mod stt; pub mod supertonic; pub mod tts; +// ── Shared utilities ────────────────────────────────────────────────────────── + +/// Drain and discard all pending messages until shutdown or disconnect. +/// Shared by both the STT and TTS worker threads for graceful degradation +/// when model files are missing or initialization fails. +pub(super) fn drain_until_shutdown( + rx: std::sync::mpsc::Receiver, + shutdown: &std::sync::atomic::AtomicBool, +) { + loop { + if shutdown.load(std::sync::atomic::Ordering::Acquire) { + break; + } + match rx.recv_timeout(std::time::Duration::from_millis(100)) { + Ok(_) => continue, + Err(_) => break, + } + } +} + use reqwest::Method; use serde::{Deserialize, Serialize}; use std::sync::{ @@ -228,13 +248,11 @@ async fn fetch_livekit_token( send_json_request(request).await } -/// Fetch agent (bot-role) pubkeys from the relay's channel membership API. -/// -/// Returns `Err` on any relay/network failure so callers can distinguish a -/// successful empty list from a failed lookup. Callers that want best-effort -/// behaviour should use `.unwrap_or_default()`. -async fn fetch_agent_pubkeys_from_relay( +/// Fetch channel members from the relay. If `role_filter` is Some, only return +/// members with that role (e.g., "bot" for agents). Returns all members if None. +async fn fetch_channel_members( channel_id: &str, + role_filter: Option<&str>, state: &AppState, ) -> Result, String> { #[derive(Deserialize)] @@ -248,49 +266,52 @@ async fn fetch_agent_pubkeys_from_relay( } let path = api_path(&["channels", channel_id, "members"]); - let request = match build_authed_request(&state.http_client, Method::GET, &path, state) { - Ok(r) => r, - Err(e) => { - eprintln!("sprout-desktop: fetch agent pubkeys failed (build request): {e}"); - return Err(e); - } - }; + let request = build_authed_request(&state.http_client, Method::GET, &path, state)?; + let resp: MembersResponse = send_json_request(request).await.map_err(|e| { + eprintln!("sprout-desktop: fetch channel members failed: {e}"); + e + })?; + + Ok(resp + .members + .into_iter() + .filter(|m| role_filter.map_or(true, |r| m.role.as_deref() == Some(r))) + .map(|m| m.pubkey) + .collect()) +} - match send_json_request::(request).await { - Ok(resp) => Ok(resp - .members - .into_iter() - .filter(|m| m.role.as_deref() == Some("bot")) - .map(|m| m.pubkey) - .collect()), - Err(e) => { - eprintln!("sprout-desktop: fetch agent pubkeys failed: {e}"); - Err(e) +/// Common setup after a huddle connection is established (both start and join). +/// Hydrates participants from relay, ensures model downloads, starts pipelines. +async fn post_connect_setup(state: &AppState, ephemeral_channel_id: &str) -> Result<(), String> { + // Hydrate agent pubkeys from relay (authoritative — overrides local guess). + if let Ok(agents) = fetch_channel_members(ephemeral_channel_id, Some("bot"), state).await { + let hs = state.huddle()?; + *hs.agent_pubkeys.lock().unwrap_or_else(|e| e.into_inner()) = agents; + } + + // Hydrate participants from relay (authoritative state). + if let Ok(all_members) = fetch_channel_members(ephemeral_channel_id, None, state).await { + if !all_members.is_empty() { + let mut hs = state.huddle()?; + hs.participants = all_members; } } -} -/// Fetch ALL member pubkeys from the relay's channel membership API. -/// -/// Returns every member (humans + bots) for participant hydration. -/// Same API as `fetch_agent_pubkeys_from_relay` but without the bot filter. -async fn fetch_all_member_pubkeys( - channel_id: &str, - state: &AppState, -) -> Result, String> { - #[derive(Deserialize)] - struct Member { - pubkey: String, + // Ensure voice models are downloading (idempotent). + if let Some(mgr) = models::global_model_manager() { + mgr.start_moonshine_download(state.http_client.clone()); + mgr.start_supertonic_download(state.http_client.clone()); } - #[derive(Deserialize)] - struct MembersResponse { - members: Vec, + + // Start pipelines: TTS first (so STT can capture tts_cancel for barge-in). + if let Err(e) = maybe_start_tts_pipeline(state).await { + eprintln!("sprout-desktop: TTS pipeline failed to start: {e}"); + } + if let Err(e) = maybe_start_stt_pipeline(state, ephemeral_channel_id).await { + eprintln!("sprout-desktop: STT pipeline failed to start: {e}"); } - let path = api_path(&["channels", channel_id, "members"]); - let request = build_authed_request(&state.http_client, Method::GET, &path, state)?; - let resp: MembersResponse = send_json_request(request).await?; - Ok(resp.members.into_iter().map(|m| m.pubkey).collect()) + Ok(()) } /// Attempt to start the STT pipeline if models are present. @@ -313,9 +334,18 @@ async fn maybe_start_stt_pipeline( let channel_uuid = parse_channel_uuid(ephemeral_channel_id)?; - // Grab shared flags, agent pubkeys, and session generation from HuddleState in one lock. + // Grab shared flags, agent pubkeys, and session generation from HuddleState. + // If replacing an existing pipeline, bump generation first so the old + // transcription task's next POST sees a stale generation and exits. let (tts_active, tts_cancel, agent_pubkeys_arc, session_gen) = { - let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let mut hs = state.huddle()?; + // Invalidate any existing transcription task before replacing the pipeline. + if hs.stt_pipeline.is_some() { + hs.session_generation.fetch_add(1, Ordering::Release); + } + if let Some(ref old) = hs.stt_pipeline { + old.shutdown(); + } ( Arc::clone(&hs.tts_active), Some(Arc::clone(&hs.tts_cancel)), @@ -327,12 +357,8 @@ async fn maybe_start_stt_pipeline( let (pipeline, text_rx) = stt::SttPipeline::new(model_dir, tts_active, tts_cancel)?; let pipeline = Arc::new(pipeline); - // Shut down existing STT pipeline before replacing (prevents leaked transcription tasks). { - let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; - if let Some(ref old) = hs.stt_pipeline { - old.shutdown(); - } + let mut hs = state.huddle()?; hs.stt_pipeline = Some(Arc::clone(&pipeline)); } @@ -341,33 +367,29 @@ async fn maybe_start_stt_pipeline( } /// Attempt to start the TTS pipeline if Supertonic models are present and TTS is enabled. -/// Silently skips if models are missing or TTS is disabled. -async fn maybe_start_tts_pipeline(state: &AppState) { +/// +/// Returns `Ok(true)` if the pipeline was started, `Ok(false)` if preconditions +/// aren't met (model not ready, pipeline exists, TTS disabled), or `Err` on failure. +async fn maybe_start_tts_pipeline(state: &AppState) -> Result { if !models::is_supertonic_ready() { - return; // Supertonic not downloaded yet — TTS unavailable. + return Ok(false); // Supertonic not downloaded yet — TTS unavailable. } // Don't create a duplicate pipeline if one is already running. { - let hs = match state.huddle_state.lock() { - Ok(h) => h, - Err(_) => return, - }; + let hs = state.huddle()?; if hs.tts_pipeline.is_some() { - return; + return Ok(false); } } let model_dir = match models::supertonic_model_dir() { Some(d) => d, - None => return, + None => return Ok(false), }; let (tts_active, tts_enabled, tts_cancel) = { - let hs = match state.huddle_state.lock() { - Ok(h) => h, - Err(_) => return, - }; + let hs = state.huddle()?; ( Arc::clone(&hs.tts_active), hs.tts_enabled, @@ -376,29 +398,21 @@ async fn maybe_start_tts_pipeline(state: &AppState) { }; if !tts_enabled { - return; + return Ok(false); } - let pipeline = match tts::TtsPipeline::new(model_dir, tts_active, tts_cancel) { - Ok(p) => Arc::new(p), - Err(e) => { - eprintln!("sprout-desktop: TTS pipeline failed to start: {e}"); - return; - } - }; + let pipeline = Arc::new(tts::TtsPipeline::new(model_dir, tts_active, tts_cancel)?); { - let mut hs = match state.huddle_state.lock() { - Ok(h) => h, - Err(_) => return, - }; + let mut hs = state.huddle()?; // Re-check: another call may have created a pipeline while we were building ours. if hs.tts_pipeline.is_some() { - // Drop the one we just created — the existing one wins. - return; + return Ok(false); // The existing one wins. } hs.tts_pipeline = Some(pipeline); } + + Ok(true) } /// Spawn a tokio task that reads text_rx and posts kind:9 events. @@ -462,21 +476,13 @@ fn spawn_transcription_task( } }; let event_json = event.as_json(); - let auth_header = match configured_api_token.as_deref() { - Some(token) => format!("Bearer {token}"), - None => format!("X-Pubkey {}", keys.public_key().to_hex()), - }; - let url = format!("{}/api/events", crate::relay::relay_api_base_url()); - let req = if auth_header.starts_with("Bearer ") { - http_client.post(&url).header("Authorization", &auth_header) - } else { - let pk = auth_header.strip_prefix("X-Pubkey ").unwrap_or(""); - http_client.post(&url).header("X-Pubkey", pk) - } - .header("Content-Type", "application/json") - .body(event_json); + let api_token_ref = configured_api_token.as_deref(); + let pubkey_hex = keys.public_key().to_hex(); - if let Err(e) = req.send().await { + if let Err(e) = + crate::events::post_event_raw(&http_client, api_token_ref, &pubkey_hex, event_json) + .await + { eprintln!("sprout-desktop: STT kind:9 post failed: {e}"); } } @@ -526,7 +532,7 @@ pub async fn start_huddle( // Transition to Creating. { - let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let mut hs = state.huddle()?; if hs.phase != HuddlePhase::Idle { return Err(format!( "cannot start huddle: already in phase {:?}", @@ -581,19 +587,14 @@ pub async fn start_huddle( events::build_huddle_started(&parent_channel_id, &ephemeral_channel_id, &lk.room)?; submit_event(started_builder, &state).await?; - // 5. Post voice-mode guidelines as a regular kind:9 message. - // We do NOT use kind:40099 — that is relay-signed; the client must not mint it. + // 5. Post voice-mode guidelines as kind:48106. // Best-effort: don't fail the huddle if this fails. let guidelines = agents::voice_mode_guidelines(&parent_channel_id); - if let Ok(msg_builder) = events::build_message( - ephemeral_uuid, - &format!("[System] {guidelines}"), - None, - &[], - &[], - ) { - if let Err(e) = submit_event(msg_builder, &state).await { - eprintln!("sprout-desktop: voice-mode guidelines message failed: {e}"); + if let Ok(guidelines_builder) = + events::build_huddle_guidelines(&ephemeral_channel_id, &guidelines) + { + if let Err(e) = submit_event(guidelines_builder, &state).await { + eprintln!("sprout-desktop: huddle guidelines (kind:48106) failed: {e}"); } } @@ -605,7 +606,7 @@ pub async fn start_huddle( Ok((lk, successful_agents)) => { // 5. Store active state. { - let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let mut hs = state.huddle()?; hs.phase = HuddlePhase::Connected; hs.is_creator = true; hs.ephemeral_channel_id = Some(ephemeral_channel_id.clone()); @@ -630,28 +631,8 @@ pub async fn start_huddle( hs.participants = participants; } - // 6. Immediately hydrate participants from relay for authoritative state. - // Best-effort — the local guess above is a reasonable fallback. - if let Ok(all_members) = fetch_all_member_pubkeys(&ephemeral_channel_id, &state).await { - if !all_members.is_empty() { - let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; - hs.participants = all_members; - } - } - - // 7. Ensure voice models are downloading (idempotent — no-ops if ready or in progress). - if let Some(mgr) = models::global_model_manager() { - mgr.start_moonshine_download(state.http_client.clone()); - mgr.start_supertonic_download(state.http_client.clone()); - } - - // 8. Auto-start TTS first, then STT. STT captures `tts_cancel` from - // `hs.tts_pipeline` at init time — TTS must exist before STT starts - // or barge-in never works. - maybe_start_tts_pipeline(&state).await; - if let Err(e) = maybe_start_stt_pipeline(&state, &ephemeral_channel_id).await { - eprintln!("sprout-desktop: STT pipeline failed to start: {e}"); - } + // 6. Hydrate members, download models, start pipelines. + post_connect_setup(&state, &ephemeral_channel_id).await?; Ok(HuddleJoinInfo { ephemeral_channel_id, @@ -695,7 +676,7 @@ pub async fn join_huddle( ) -> Result { // Transition to Connecting. { - let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let mut hs = state.huddle()?; if hs.phase != HuddlePhase::Idle { return Err(format!( "cannot join huddle: already in phase {:?}", @@ -730,15 +711,13 @@ pub async fn join_huddle( // 3. Store active state. { - let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let mut hs = state.huddle()?; hs.phase = HuddlePhase::Connected; hs.livekit_token = Some(lk.token.clone()); hs.livekit_url = Some(lk.url.clone()); hs.livekit_room = Some(lk.room.clone()); - // agent_pubkeys is hydrated after this block via fetch_agent_pubkeys_from_relay. - - // Include at least the current user in the participant list. - // Full participant sync from relay membership is a post-MVP enhancement. + // agent_pubkeys + participants hydrated by post_connect_setup below. + // Seed with current user as a fallback until relay responds. let own_pubkey = state .keys .lock() @@ -749,38 +728,8 @@ pub async fn join_huddle( } } - // 4. Hydrate agent_pubkeys from relay membership so joiners can: - // (a) p-tag agents on STT transcripts, and - // (b) filter agent messages for TTS on the frontend. - // Must happen before maybe_start_stt_pipeline — the transcription task reads agent_pubkeys. - // Best-effort for joiners — don't fail the join on a transient fetch error. - // On Ok: always write (even empty — huddle may have no agents yet). - // On Err: leave default empty list (periodic refresh will retry). - if let Ok(agents) = fetch_agent_pubkeys_from_relay(&ephemeral_channel_id, &state).await { - let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; - *hs.agent_pubkeys.lock().unwrap_or_else(|e| e.into_inner()) = agents; - } - - // 4b. Immediately hydrate participants from relay for authoritative state. - // Best-effort — the self-only list above is a reasonable fallback. - if let Ok(all_members) = fetch_all_member_pubkeys(&ephemeral_channel_id, &state).await { - if !all_members.is_empty() { - let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; - hs.participants = all_members; - } - } - - // 5. Ensure voice models are downloading (idempotent). - if let Some(mgr) = models::global_model_manager() { - mgr.start_moonshine_download(state.http_client.clone()); - mgr.start_supertonic_download(state.http_client.clone()); - } - - // 6. Auto-start TTS first, then STT (same ordering rationale as start_huddle). - maybe_start_tts_pipeline(&state).await; - if let Err(e) = maybe_start_stt_pipeline(&state, &ephemeral_channel_id).await { - eprintln!("sprout-desktop: STT pipeline failed to start: {e}"); - } + // 4. Hydrate members, download models, start pipelines. + post_connect_setup(&state, &ephemeral_channel_id).await?; Ok(HuddleJoinInfo { ephemeral_channel_id, @@ -796,7 +745,7 @@ pub async fn join_huddle( /// shutdown-then-reset sequence. fn teardown_huddle(state: &AppState) -> Result<(), String> { { - let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let hs = state.huddle()?; // Increment generation first — this immediately invalidates any // in-flight transcription task, even before pipelines shut down. hs.session_generation.fetch_add(1, Ordering::Release); @@ -808,7 +757,7 @@ fn teardown_huddle(state: &AppState) -> Result<(), String> { } } { - let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let mut hs = state.huddle()?; // Preserve the generation counter across reset — it must survive // for the old transcription task to see the incremented value. let gen = Arc::clone(&hs.session_generation); @@ -827,7 +776,7 @@ fn teardown_huddle(state: &AppState) -> Result<(), String> { #[tauri::command] pub async fn leave_huddle(state: State<'_, AppState>) -> Result<(), String> { let (parent_channel_id, ephemeral_channel_id) = { - let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let mut hs = state.huddle()?; if hs.phase == HuddlePhase::Idle { return Ok(()); // Nothing to leave. } @@ -864,7 +813,7 @@ pub async fn leave_huddle(state: State<'_, AppState>) -> Result<(), String> { #[tauri::command] pub async fn end_huddle(state: State<'_, AppState>) -> Result<(), String> { let (parent_channel_id, ephemeral_channel_id) = { - let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let mut hs = state.huddle()?; if hs.phase == HuddlePhase::Idle { return Ok(()); // Nothing to end. } @@ -909,7 +858,7 @@ pub async fn end_huddle(state: State<'_, AppState>) -> Result<(), String> { /// Transitions from Connected → Active. No-op if already Active. #[tauri::command] pub async fn confirm_huddle_active(state: State<'_, AppState>) -> Result<(), String> { - let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let mut hs = state.huddle()?; match hs.phase { HuddlePhase::Connected => { hs.phase = HuddlePhase::Active; @@ -923,7 +872,7 @@ pub async fn confirm_huddle_active(state: State<'_, AppState>) -> Result<(), Str /// Return the current HuddleState (serialized for the frontend). #[tauri::command] pub fn get_huddle_state(state: State<'_, AppState>) -> Result { - let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let hs = state.huddle()?; Ok(hs.clone()) } @@ -936,11 +885,11 @@ pub fn get_huddle_state(state: State<'_, AppState>) -> Result) -> Result, String> { let eph_id = { - let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let hs = state.huddle()?; hs.ephemeral_channel_id.clone() }; match eph_id { - Some(id) => fetch_agent_pubkeys_from_relay(&id, &state).await, + Some(id) => fetch_channel_members(&id, Some("bot"), &state).await, None => Ok(Vec::new()), } } @@ -968,7 +917,7 @@ pub fn push_audio_pcm( MAX_AUDIO_BATCH_BYTES )); } - if let Ok(hs) = state.huddle_state.lock() { + if let Ok(hs) = state.huddle() { if let Some(ref pipeline) = hs.stt_pipeline { pipeline.push_audio(bytes.to_vec())?; } @@ -987,7 +936,7 @@ pub fn push_audio_pcm( #[tauri::command] pub async fn check_pipeline_hotstart(state: State<'_, AppState>) -> Result<(), String> { let (is_active, has_stt, has_tts, ephemeral_channel_id) = { - let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let hs = state.huddle()?; ( matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active), hs.stt_pipeline.is_some(), @@ -1010,7 +959,9 @@ pub async fn check_pipeline_hotstart(state: State<'_, AppState>) -> Result<(), S // Start TTS first (so STT can capture tts_cancel). if !has_tts && (supertonic_ready || models::is_supertonic_ready()) { - maybe_start_tts_pipeline(&state).await; + if let Err(e) = maybe_start_tts_pipeline(&state).await { + eprintln!("sprout-desktop: TTS hotstart failed: {e}"); + } } if !has_stt && (moonshine_ready || models::is_moonshine_ready()) { @@ -1030,7 +981,7 @@ pub async fn check_pipeline_hotstart(state: State<'_, AppState>) -> Result<(), S // On Err: preserve the existing list (transient failure shouldn't zero it). if let Some(eph_id) = &ephemeral_channel_id { let should_refresh = { - let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let hs = state.huddle()?; match hs.last_agent_refresh { None => true, Some(t) => t.elapsed() >= std::time::Duration::from_secs(15), @@ -1041,19 +992,20 @@ pub async fn check_pipeline_hotstart(state: State<'_, AppState>) -> Result<(), S // Sequential — tokio::join! requires the `macros` feature. // Only update the throttle timestamp when at least one fetch succeeds, // so transient failures retry immediately on the next poll cycle. - let mut any_success = false; - if let Ok(fresh_agents) = fetch_agent_pubkeys_from_relay(eph_id, &state).await { - let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; - *hs.agent_pubkeys.lock().unwrap_or_else(|e| e.into_inner()) = fresh_agents; - any_success = true; - } - if let Ok(fresh_members) = fetch_all_member_pubkeys(eph_id, &state).await { - let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; - hs.participants = fresh_members; - any_success = true; - } - if any_success { - let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + // Fetch both lists before acquiring the lock — no lock held across await. + let fresh_agents = fetch_channel_members(eph_id, Some("bot"), &state) + .await + .ok(); + let fresh_members = fetch_channel_members(eph_id, None, &state).await.ok(); + + if fresh_agents.is_some() || fresh_members.is_some() { + let mut hs = state.huddle()?; + if let Some(agents) = fresh_agents { + *hs.agent_pubkeys.lock().unwrap_or_else(|e| e.into_inner()) = agents; + } + if let Some(members) = fresh_members { + hs.participants = members; + } hs.last_agent_refresh = Some(std::time::Instant::now()); } } @@ -1070,7 +1022,7 @@ pub async fn check_pipeline_hotstart(state: State<'_, AppState>) -> Result<(), S #[tauri::command] pub async fn start_stt_pipeline(state: State<'_, AppState>) -> Result<(), String> { let ephemeral_channel_id = { - let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let hs = state.huddle()?; hs.ephemeral_channel_id .clone() .ok_or("no active huddle — start or join a huddle first")? @@ -1115,7 +1067,7 @@ pub fn get_model_status(_state: State<'_, AppState>) -> Result) -> Result<(), String> { { - let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let mut hs = state.huddle()?; hs.tts_enabled = enabled; if !enabled { // Shut down the TTS pipeline immediately. @@ -1129,11 +1081,13 @@ pub async fn set_tts_enabled(enabled: bool, state: State<'_, AppState>) -> Resul if enabled { // Re-start TTS pipeline if models are available and huddle is active. let phase = { - let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let hs = state.huddle()?; hs.phase.clone() }; if matches!(phase, HuddlePhase::Connected | HuddlePhase::Active) { - maybe_start_tts_pipeline(&state).await; + if let Err(e) = maybe_start_tts_pipeline(&state).await { + eprintln!("sprout-desktop: TTS pipeline restart failed: {e}"); + } } } @@ -1164,7 +1118,7 @@ pub async fn speak_agent_message(text: String, state: State<'_, AppState>) -> Re }; let needs_pipeline = { - let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let hs = state.huddle()?; hs.tts_enabled && hs.tts_pipeline.is_none() && matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active) @@ -1172,10 +1126,12 @@ pub async fn speak_agent_message(text: String, state: State<'_, AppState>) -> Re // Lazy-start: models may have finished downloading after the huddle began. if needs_pipeline { - maybe_start_tts_pipeline(&state).await; + if let Err(e) = maybe_start_tts_pipeline(&state).await { + eprintln!("sprout-desktop: TTS lazy-start failed: {e}"); + } } - let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let hs = state.huddle()?; if hs.tts_enabled { if let Some(ref pipeline) = hs.tts_pipeline { pipeline.speak(text)?; @@ -1205,7 +1161,7 @@ pub async fn add_agent_to_huddle( validate_pubkey_hex(&agent_pubkey)?; let (eph_id, parent_id) = { - let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let hs = state.huddle()?; if !matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active) { return Err("no active huddle".to_string()); } @@ -1242,7 +1198,7 @@ pub async fn add_agent_to_huddle( // acquiring the inner pubkeys lock (avoids the E0597 borrow-checker error). { let agent_pubkeys_arc = { - let hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let hs = state.huddle()?; Arc::clone(&hs.agent_pubkeys) }; let mut pubkeys = agent_pubkeys_arc.lock().unwrap_or_else(|e| e.into_inner()); @@ -1251,24 +1207,12 @@ pub async fn add_agent_to_huddle( } } - // Re-post voice-mode guidelines so the newly-added agent sees them. - // The agent auto-subscribes via membership notification; by the time it - // processes the subscription, this message will be in the channel history. - { - let (eph_uuid, parent_id_for_guidelines) = (eph_uuid, parent_id.clone()); - let guidelines = agents::voice_mode_guidelines(&parent_id_for_guidelines); - if let Ok(msg_builder) = - events::build_message(eph_uuid, &format!("[System] {guidelines}"), None, &[], &[]) - { - if let Err(e) = submit_event(msg_builder, &state).await { - eprintln!("sprout-desktop: voice-mode guidelines re-post for agent failed: {e}"); - } - } - } + // No guidelines re-post needed — the agent sees the original kind:48106 + // guidelines via EOSE replay when it subscribes to the ephemeral channel. // Also add the agent to the visible participants list. { - let mut hs = state.huddle_state.lock().map_err(|e| e.to_string())?; + let mut hs = state.huddle()?; if !hs.participants.contains(&agent_pubkey) { hs.participants.push(agent_pubkey); } diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index 196e6d7f69..8a8f688d3e 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -211,7 +211,7 @@ fn stt_worker( model_dir.display() ); // Drain the channel so push_audio doesn't block the sender. - drain_channel(audio_rx, &shutdown); + drain_until_shutdown(audio_rx, &shutdown); return; } @@ -233,7 +233,7 @@ fn stt_worker( Some(r) => r, None => { eprintln!("sprout-desktop: OfflineRecognizer::create returned None — STT disabled"); - drain_channel(audio_rx, &shutdown); + drain_until_shutdown(audio_rx, &shutdown); return; } }; @@ -482,15 +482,5 @@ fn bytes_to_f32(bytes: &[u8]) -> Vec { .collect() } -/// Drain and discard all pending messages on the channel until shutdown or disconnect. -fn drain_channel(rx: Receiver>, shutdown: &AtomicBool) { - loop { - if shutdown.load(Ordering::Acquire) { - break; - } - match rx.recv_timeout(Duration::from_millis(100)) { - Ok(_) => continue, - Err(_) => break, - } - } -} +// drain_until_shutdown lives in super (huddle/mod.rs) — shared with tts.rs. +use super::drain_until_shutdown; diff --git a/desktop/src-tauri/src/huddle/tts.rs b/desktop/src-tauri/src/huddle/tts.rs index 40d08583e9..8d7f8ec5b9 100644 --- a/desktop/src-tauri/src/huddle/tts.rs +++ b/desktop/src-tauri/src/huddle/tts.rs @@ -173,6 +173,10 @@ impl TtsPipeline { /// /// Sets the cancel flag. The worker will drain the queue and stop the /// current rodio Player on its next iteration. + /// + /// Currently unused — barge-in is triggered by the STT pipeline setting + /// `tts_cancel` directly via the shared `Arc`. Retained as + /// public API for future callers (e.g., explicit "stop speaking" button). #[allow(dead_code)] pub fn cancel(&self) { self.cancel.store(true, Ordering::Release); @@ -215,7 +219,7 @@ fn tts_worker( "sprout-desktop: TTS Supertonic init failed (model_dir={}): {e}. TTS disabled.", model_dir.display() ); - drain_text_channel(text_rx, &shutdown); + drain_until_shutdown(text_rx, &shutdown); return; } }; @@ -228,7 +232,7 @@ fn tts_worker( eprintln!( "sprout-desktop: TTS voice style load failed ({voice_name}): {e}. TTS disabled." ); - drain_text_channel(text_rx, &shutdown); + drain_until_shutdown(text_rx, &shutdown); return; } }; @@ -240,7 +244,7 @@ fn tts_worker( Ok(h) => h, Err(e) => { eprintln!("sprout-desktop: TTS audio output failed: {e}. TTS disabled."); - drain_text_channel(text_rx, &shutdown); + drain_until_shutdown(text_rx, &shutdown); return; } }; @@ -404,15 +408,5 @@ fn apply_fades(samples: &mut Vec) { } } -/// Drain and discard all pending text until shutdown or disconnect. -fn drain_text_channel(rx: mpsc::Receiver, shutdown: &AtomicBool) { - loop { - if shutdown.load(Ordering::Acquire) { - break; - } - match rx.recv_timeout(Duration::from_millis(100)) { - Ok(_) => continue, - Err(_) => break, - } - } -} +// drain_until_shutdown lives in super (huddle/mod.rs) — shared with stt.rs. +use super::drain_until_shutdown; diff --git a/desktop/src/features/huddle/HuddleContext.tsx b/desktop/src/features/huddle/HuddleContext.tsx index e2f18358ae..066227fba3 100644 --- a/desktop/src/features/huddle/HuddleContext.tsx +++ b/desktop/src/features/huddle/HuddleContext.tsx @@ -16,9 +16,9 @@ import { setupAudioWorklet } from "./lib/audioWorklet"; * → setEphemeralChannelId(...) [triggers TTS subscription + hotstart polling] * * TTS subscription (on ephemeralChannelId change): - * → relayClient.subscribeToChannel(ephId, callback) - * → buffer events until EOSE, then replay live ones - * → filter: agent pubkeys only (fail-closed), skip self, skip [System] + * → relayClient.subscribeToChannelLive(ephId, callback) + * → live-only (since: now) — no historical backlog + * → filter: agent pubkeys only (fail-closed), skip self * → invoke("speak_agent_message", { text }) * * leaveHuddle() @@ -74,11 +74,12 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { /** Self pubkey — fetched once, used to filter out own messages from TTS */ const selfPubkeyRef = React.useRef(null); - const leaveHuddle = React.useCallback(async (): Promise => { - // Invalidate any in-flight startHuddle so it bails after its next await + /** Stop AudioWorklet and disconnect LiveKit. Best-effort on both steps. */ + const disconnectMedia = React.useCallback(async () => { + // Invalidate any in-flight startHuddle tokenRef.current += 1; - // Step 1: Stop AudioWorklet (best-effort — don't let a throw skip remaining cleanup) + // Step 1: Stop AudioWorklet try { workletRef.current?.stop(); } catch { @@ -86,7 +87,7 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { } workletRef.current = null; - // Step 2: Disconnect LiveKit (best-effort, null ref first to prevent double-disconnect) + // Step 2: Disconnect LiveKit (null ref first to prevent double-disconnect) const conn = connectionRef.current; connectionRef.current = null; try { @@ -97,8 +98,10 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { setLocalAudioTrack(null); setMicConnected(false); setEphemeralChannelId(null); + }, []); - // Step 3: Tell Rust to clean up — only clear rustActiveRef AFTER success so retries work + const leaveHuddle = React.useCallback(async (): Promise => { + await disconnectMedia(); if (rustActiveRef.current) { try { await invoke("leave_huddle"); @@ -109,33 +112,10 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { } } return true; // Backend cleanup succeeded (or was not needed) - }, []); + }, [disconnectMedia]); const endHuddle = React.useCallback(async (): Promise => { - // Invalidate any in-flight startHuddle - tokenRef.current += 1; - - // Step 1: Stop AudioWorklet - try { - workletRef.current?.stop(); - } catch { - /* best-effort */ - } - workletRef.current = null; - - // Step 2: Disconnect LiveKit - const conn = connectionRef.current; - connectionRef.current = null; - try { - if (conn) await conn.disconnect(); - } catch { - /* best-effort */ - } - setLocalAudioTrack(null); - setMicConnected(false); - setEphemeralChannelId(null); - - // Step 3: Tell Rust to end the huddle (archives channel, emits huddle_ended) + await disconnectMedia(); if (rustActiveRef.current) { try { await invoke("end_huddle"); @@ -150,9 +130,15 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { } } } - }, []); - - /** Clean up a partially-established huddle. Best-effort on every step. */ + }, [disconnectMedia]); + + /** + * Clean up a partially-established huddle. Best-effort on every step. + * + * Note: takes explicit conn/worklet args (not from refs) because startHuddle + * may have local variables that differ from the refs mid-flight. Can't use + * disconnectMedia() here for the same reason. + */ const cleanupFailedStart = React.useCallback( async ( conn: HuddleConnection | null, @@ -321,38 +307,18 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { void loadAgentPubkeys(); }, 10_000); - // ── EOSE-based replay boundary with timestamp belt-and-suspenders ───── - let subscriptionReady = false; + // ── Live-only subscription ─────────────────────────────────────────── + // subscribeToChannelLive uses `since: now` — the relay never sends + // historical backlog. Every event delivered is a live message. + // Event-ID dedup handles reconnect replay (same event arriving twice). const seenEventIds = new Set(); const seenOrder: string[] = []; const MAX_SEEN_EVENTS = 5000; - const subscribeTimeSecs = Math.floor(Date.now() / 1000); - const pendingEvents: Array<{ - pubkey: string; - content: string; - created_at: number; - }> = []; - - /** Speak an event if it passes all filters. */ - function maybeSpeakEvent(pubkey: string, content: string) { - // Fail-closed: don't speak until agent list is loaded. - if (!agentsLoaded) return; - // Only speak agent messages — skip human STT transcripts. - if (!agentPubkeys.has(pubkey)) return; - if (pubkey === selfPubkeyRef.current) return; - if (!content.trim()) return; - if (content.startsWith("[System]")) return; - invoke("speak_agent_message", { text: content }).catch((err) => { - console.warn( - "[huddle] TTS speak failed (backpressure or pipeline unavailable):", - err, - ); - }); - } relayClient - .subscribeToChannel(ephemeralChannelId, (event) => { + .subscribeToChannelLive(ephemeralChannelId, (event) => { if (disposed) return; + // Defense-in-depth: subscription already filters to kind:9 only. if (event.kind !== 9) return; // Dedup by event ID (covers reconnect replay). @@ -364,33 +330,23 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { if (oldest !== undefined) seenEventIds.delete(oldest); } - if (!subscriptionReady) { - // Before EOSE: buffer the event. After EOSE resolves, we'll replay - // any that have created_at >= subscribeTimeSecs (i.e., truly live). - pendingEvents.push(event); - return; - } - - // After EOSE: belt-and-suspenders — also reject events with - // created_at before our subscription started. This catches late - // history that arrives after the 250ms fallback fires. - if (event.created_at < subscribeTimeSecs) return; - - maybeSpeakEvent(event.pubkey, event.content); + // Fail-closed: don't speak until agent list is loaded. + if (!agentsLoaded) return; + // Only speak agent messages — skip human STT transcripts. + if (!agentPubkeys.has(event.pubkey)) return; + if (event.pubkey === selfPubkeyRef.current) return; + if (!event.content.trim()) return; + // Legacy: skip [System]-prefixed messages from before kind:48106. + if (event.content.startsWith("[System]")) return; + + invoke("speak_agent_message", { text: event.content }).catch((err) => { + console.warn( + "[huddle] TTS speak failed (backpressure or pipeline unavailable):", + err, + ); + }); }) .then((dispose) => { - // subscribeToChannel resolves after EOSE (or relay's 250ms fallback). - subscriptionReady = true; - - // Replay buffered events that arrived during the EOSE window. - // Only speak events with created_at >= subscribeTimeSecs (truly live). - for (const evt of pendingEvents) { - if (evt.created_at >= subscribeTimeSecs) { - maybeSpeakEvent(evt.pubkey, evt.content); - } - } - pendingEvents.length = 0; // Clear buffer - if (disposed) { void dispose(); return; diff --git a/desktop/src/features/huddle/components/HuddleBar.tsx b/desktop/src/features/huddle/components/HuddleBar.tsx index b455ab626b..0006d0f2ed 100644 --- a/desktop/src/features/huddle/components/HuddleBar.tsx +++ b/desktop/src/features/huddle/components/HuddleBar.tsx @@ -16,7 +16,9 @@ import { useHuddle } from "../HuddleContext"; import { AddAgentDialog, type AgentAddResult } from "./AddAgentDialog"; import { ParticipantList } from "./ParticipantList"; -// Shape returned by the `get_huddle_state` Tauri command +// Shape returned by the `get_huddle_state` Tauri command. +// NOTE: This mirrors the HuddleState struct in the Rust backend (src-tauri/src/huddle/state.rs). +// If you add/remove fields here, update the Rust struct (and vice versa). type HuddleState = { phase: | "idle" diff --git a/desktop/src/shared/api/relayClientSession.ts b/desktop/src/shared/api/relayClientSession.ts index 66bdf1d740..3dc63fbdad 100644 --- a/desktop/src/shared/api/relayClientSession.ts +++ b/desktop/src/shared/api/relayClientSession.ts @@ -157,6 +157,27 @@ export class RelayClient { return this.subscribe(this.buildChannelFilter(channelId, 50), onEvent); } + /** + * Subscribe to a channel starting from NOW — no history backfill. + * Used by huddle TTS where only live kind:9 messages should be spoken. + * The `since` filter ensures the relay never sends historical backlog. + * The high `limit` ensures reconnect replay can recover all missed events. + */ + async subscribeToChannelLive( + channelId: string, + onEvent: (event: RelayEvent) => void, + ) { + return this.subscribe( + { + kinds: [KIND_STREAM_MESSAGE], + "#h": [channelId], + limit: 1000, + since: Math.floor(Date.now() / 1_000), + }, + onEvent, + ); + } + async subscribeToTypingIndicators( channelId: string, onEvent: (event: RelayEvent) => void, From f11520e5ce0b6b59964860a2c8e8d427d48ecc67 Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Sun, 12 Apr 2026 22:05:18 -0400 Subject: [PATCH 29/41] =?UTF-8?q?refactor(huddles):=20crossfire=20cleanup?= =?UTF-8?q?=20=E2=80=94=20DRY,=20safety,=20display=20names?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crossfire review (opus×2 + codex) identified 15+ issues. All fixed: Safety & correctness: - Fix validate_pubkey_hex panic on multi-byte UTF-8 input - Fix pipeline init failure wedge (is_finished + dead pipeline detection) - Fix guidelines delivery race (post kind:48106 before adding agents) - Fix teardown blocking mutex during thread join - Fix endHuddle swallowing failures (returns boolean) - Fix livekit.ts mic track leak on disconnect failure (try/finally) - Fix tts_active staying true during cancel - Filter single-char TTS responses (prevents speaking 'period') DRY extraction: - models.rs: 826→690 lines via ModelSlot + shared helpers (download_file, fetch_url, fresh_temp_dir, verify_and_install) - events.rs: 4 huddle event builders → shared build_huddle_event - tts.rs: 4 cancel/shutdown patterns → handle_cancel_or_shutdown UX improvements: - ParticipantList resolves pubkeys to display names (useUsersBatchQuery) - ProfileAvatar with hex-prefix HexAvatar fallback - Voice guidelines prompt tightened for fast LLMs Cleanup: - Delete dead voice_style_path(), TtsPipeline::cancel() - Fix stale comments, document LE endianness assumption --- desktop/public/worklet.js | 6 +- desktop/src-tauri/src/events.rs | 61 +- desktop/src-tauri/src/huddle/agents.rs | 23 +- desktop/src-tauri/src/huddle/mod.rs | 92 +- desktop/src-tauri/src/huddle/models.rs | 820 ++++++++---------- desktop/src-tauri/src/huddle/preprocessing.rs | 18 +- desktop/src-tauri/src/huddle/stt.rs | 10 + desktop/src-tauri/src/huddle/tts.rs | 85 +- desktop/src/features/huddle/HuddleContext.tsx | 15 +- .../features/huddle/components/HuddleBar.tsx | 9 +- .../huddle/components/ParticipantList.tsx | 46 +- desktop/src/features/huddle/lib/livekit.ts | 11 +- 12 files changed, 588 insertions(+), 608 deletions(-) diff --git a/desktop/public/worklet.js b/desktop/public/worklet.js index 2a874f8ccd..816ea326f5 100644 --- a/desktop/public/worklet.js +++ b/desktop/public/worklet.js @@ -2,9 +2,9 @@ // Accumulates PCM Float32 samples and sends 100ms batches to the main thread. // // Note: when the worklet is disconnected, any partial buffer (< 4800 samples) -// is silently dropped. This means the last ~100ms of speech may be lost on -// huddle leave. This is acceptable — the STT pipeline's silence-flush threshold -// (450ms) means the last utterance was already transcribed before disconnect. +// is silently dropped. The last ~100ms of speech may be lost on huddle leave. +// This is acceptable for voice — losing a partial syllable at disconnect is +// imperceptible compared to the natural end-of-conversation flow. class SttTapProcessor extends AudioWorkletProcessor { constructor() { super(); diff --git a/desktop/src-tauri/src/events.rs b/desktop/src-tauri/src/events.rs index 0172a7b546..2750f0865f 100644 --- a/desktop/src-tauri/src/events.rs +++ b/desktop/src-tauri/src/events.rs @@ -365,21 +365,39 @@ fn validate_channel_id(id: &str) -> Result<(), String> { Ok(()) } -/// Kind 48100 — huddle started advisory posted to the parent channel. -pub fn build_huddle_started( +/// Shared builder for huddle lifecycle events (kinds 48100–48103). +/// All huddle events share: validate two channel IDs, JSON content with +/// `ephemeral_channel_id`, and an `["h", parent_channel_id]` tag. +fn build_huddle_event( + kind: u16, parent_channel_id: &str, ephemeral_channel_id: &str, - livekit_room: &str, + extra_fields: &[(&str, &str)], ) -> Result { validate_channel_id(parent_channel_id)?; validate_channel_id(ephemeral_channel_id)?; - let content = serde_json::json!({ + let mut content = serde_json::json!({ "ephemeral_channel_id": ephemeral_channel_id, - "livekit_room": livekit_room, - }) - .to_string(); + }); + for (k, v) in extra_fields { + content[*k] = serde_json::Value::String(v.to_string()); + } let tags = vec![tag(vec!["h", parent_channel_id])?]; - Ok(EventBuilder::new(Kind::Custom(48100), content).tags(tags)) + Ok(EventBuilder::new(Kind::Custom(kind), content.to_string()).tags(tags)) +} + +/// Kind 48100 — huddle started advisory posted to the parent channel. +pub fn build_huddle_started( + parent_channel_id: &str, + ephemeral_channel_id: &str, + livekit_room: &str, +) -> Result { + build_huddle_event( + 48100, + parent_channel_id, + ephemeral_channel_id, + &[("livekit_room", livekit_room)], + ) } /// Kind 48101 — participant joined a huddle, posted to the parent channel. @@ -387,14 +405,7 @@ pub fn build_huddle_participant_joined( parent_channel_id: &str, ephemeral_channel_id: &str, ) -> Result { - validate_channel_id(parent_channel_id)?; - validate_channel_id(ephemeral_channel_id)?; - let content = serde_json::json!({ - "ephemeral_channel_id": ephemeral_channel_id, - }) - .to_string(); - let tags = vec![tag(vec!["h", parent_channel_id])?]; - Ok(EventBuilder::new(Kind::Custom(48101), content).tags(tags)) + build_huddle_event(48101, parent_channel_id, ephemeral_channel_id, &[]) } /// Kind 48102 — participant left a huddle, posted to the parent channel. @@ -402,14 +413,7 @@ pub fn build_huddle_participant_left( parent_channel_id: &str, ephemeral_channel_id: &str, ) -> Result { - validate_channel_id(parent_channel_id)?; - validate_channel_id(ephemeral_channel_id)?; - let content = serde_json::json!({ - "ephemeral_channel_id": ephemeral_channel_id, - }) - .to_string(); - let tags = vec![tag(vec!["h", parent_channel_id])?]; - Ok(EventBuilder::new(Kind::Custom(48102), content).tags(tags)) + build_huddle_event(48102, parent_channel_id, ephemeral_channel_id, &[]) } /// Kind 48103 — huddle ended, posted to the parent channel. @@ -417,14 +421,7 @@ pub fn build_huddle_ended( parent_channel_id: &str, ephemeral_channel_id: &str, ) -> Result { - validate_channel_id(parent_channel_id)?; - validate_channel_id(ephemeral_channel_id)?; - let content = serde_json::json!({ - "ephemeral_channel_id": ephemeral_channel_id, - }) - .to_string(); - let tags = vec![tag(vec!["h", parent_channel_id])?]; - Ok(EventBuilder::new(Kind::Custom(48103), content).tags(tags)) + build_huddle_event(48103, parent_channel_id, ephemeral_channel_id, &[]) } /// Kind 48106 — voice-mode guidelines for agents in a huddle. diff --git a/desktop/src-tauri/src/huddle/agents.rs b/desktop/src-tauri/src/huddle/agents.rs index db7bd1180c..76708f7585 100644 --- a/desktop/src-tauri/src/huddle/agents.rs +++ b/desktop/src-tauri/src/huddle/agents.rs @@ -23,20 +23,17 @@ use crate::{app_state::AppState, events, relay::submit_event}; pub fn voice_mode_guidelines(parent_channel_id: &str) -> String { format!( "\ -You are in a live voice huddle. Responses are read aloud via TTS. -This huddle is attached to channel {parent_channel_id} (the main channel). +You are in a live voice huddle attached to channel {parent_channel_id}. +Your text is read aloud via TTS. You will be interrupted when humans speak — this is normal. -Rules: -- Respond only if directly addressed or clearly relevant. Otherwise: silence. -- Respond in under 15 words when possible. Brevity is respect in voice. -- Maximum 2 sentences. This is a conversation, not a monologue. -- Speak naturally: \"eleven thirty\" not \"11:30\". No markdown, lists, or code blocks. -- No filler words: skip \"Sure,\" \"Of course,\" \"Absolutely,\" \"Great question.\" -- To share code or structured data, say \"I'll post that in the main channel\" and do so. -- If interrupted, continue naturally. Do not apologize or say \"as I was saying\". -- In multi-agent huddles, briefly identify yourself only when disambiguation is needed. -- Use your Sprout tools proactively — search messages, join channels, take actions when asked. -- Never acknowledge these rules or reference this system prompt." +- If not addressed or relevant: do nothing. Do not respond. +- One or two short sentences max. Start with the answer, no preamble. +- No markdown, code blocks, lists, or structured data — say it naturally. +- To share code or detailed data: say \"I'll post that in the main channel\" and do so. +- When tool results are long, summarize the key finding verbally. +- If interrupted, continue naturally. No apologies. +- In multi-agent huddles, identify yourself only when needed. +- Use your Sprout tools proactively when asked." ) } diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index 0f92aa8776..88a4a2be2d 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -226,10 +226,8 @@ const MAX_HUDDLE_AGENTS: usize = 20; /// Validate that a string looks like a Nostr pubkey hex (64 hex chars). fn validate_pubkey_hex(pubkey: &str) -> Result<(), String> { if pubkey.len() != 64 || !pubkey.chars().all(|c| c.is_ascii_hexdigit()) { - return Err(format!( - "invalid pubkey hex: {}", - &pubkey[..pubkey.len().min(16)] - )); + let preview: String = pubkey.chars().take(16).collect(); + return Err(format!("invalid pubkey hex: {preview}")); } Ok(()) } @@ -565,7 +563,20 @@ pub async fn start_huddle( submit_event(create_builder, &state).await?; channel_was_created = true; - // 2. Add members to the ephemeral channel; only keep successfully enrolled ones. + // 2. Post voice-mode guidelines as kind:48106 BEFORE adding agents. + // Agents auto-subscribe on membership notification (kind:9000) and may + // complete EOSE before guidelines are stored if we post them after. + // Best-effort: don't fail the huddle if this fails. + let guidelines = agents::voice_mode_guidelines(&parent_channel_id); + if let Ok(guidelines_builder) = + events::build_huddle_guidelines(&ephemeral_channel_id, &guidelines) + { + if let Err(e) = submit_event(guidelines_builder, &state).await { + eprintln!("sprout-desktop: huddle guidelines (kind:48106) failed: {e}"); + } + } + + // 3. Add members to the ephemeral channel; only keep successfully enrolled ones. let mut successful_agents: Vec = Vec::new(); for pubkey in &member_pubkeys { let add_builder = events::build_add_member(ephemeral_uuid, pubkey, Some("bot"))?; @@ -578,26 +589,15 @@ pub async fn start_huddle( } } - // 3. Fetch LiveKit token BEFORE emitting HUDDLE_STARTED. + // 4. Fetch LiveKit token BEFORE emitting HUDDLE_STARTED. // This prevents a phantom announcement if the token fetch fails. let lk = fetch_livekit_token(&ephemeral_channel_id, &state).await?; - // 4. Emit HUDDLE_STARTED to parent channel — only now that token is confirmed. + // 5. Emit HUDDLE_STARTED to parent channel — only now that token is confirmed. let started_builder = events::build_huddle_started(&parent_channel_id, &ephemeral_channel_id, &lk.room)?; submit_event(started_builder, &state).await?; - // 5. Post voice-mode guidelines as kind:48106. - // Best-effort: don't fail the huddle if this fails. - let guidelines = agents::voice_mode_guidelines(&parent_channel_id); - if let Ok(guidelines_builder) = - events::build_huddle_guidelines(&ephemeral_channel_id, &guidelines) - { - if let Err(e) = submit_event(guidelines_builder, &state).await { - eprintln!("sprout-desktop: huddle guidelines (kind:48106) failed: {e}"); - } - } - Ok((lk, successful_agents)) } .await; @@ -744,26 +744,31 @@ pub async fn join_huddle( /// Used by both `leave_huddle` and `end_huddle` to avoid duplicating the /// shutdown-then-reset sequence. fn teardown_huddle(state: &AppState) -> Result<(), String> { - { - let hs = state.huddle()?; + // Take pipeline handles out of state and drop the lock before shutdown. + // Pipeline Drop impls join worker threads — this avoids blocking while + // the mutex is held (ONNX inference can take ~200ms). + let (old_stt, old_tts) = { + let mut hs = state.huddle()?; // Increment generation first — this immediately invalidates any // in-flight transcription task, even before pipelines shut down. hs.session_generation.fetch_add(1, Ordering::Release); - if let Some(ref pipeline) = hs.stt_pipeline { - pipeline.shutdown(); - } - if let Some(ref pipeline) = hs.tts_pipeline { - pipeline.shutdown(); - } - } - { - let mut hs = state.huddle()?; - // Preserve the generation counter across reset — it must survive - // for the old transcription task to see the incremented value. + let stt = hs.stt_pipeline.take(); + let tts = hs.tts_pipeline.take(); let gen = Arc::clone(&hs.session_generation); *hs = HuddleState::default(); hs.session_generation = gen; + (stt, tts) + }; + // Shut down outside the lock — thread joins happen here. + if let Some(ref p) = old_stt { + p.shutdown(); + } + if let Some(ref p) = old_tts { + p.shutdown(); } + // Drop the Arcs here (implicit) — triggers thread join via Drop. + drop(old_stt); + drop(old_tts); Ok(()) } @@ -935,12 +940,10 @@ pub fn push_audio_pcm( /// the huddle is not active or pipelines are already running. #[tauri::command] pub async fn check_pipeline_hotstart(state: State<'_, AppState>) -> Result<(), String> { - let (is_active, has_stt, has_tts, ephemeral_channel_id) = { + let (is_active, ephemeral_channel_id) = { let hs = state.huddle()?; ( matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active), - hs.stt_pipeline.is_some(), - hs.tts_pipeline.is_some(), hs.ephemeral_channel_id.clone(), ) }; @@ -949,6 +952,27 @@ pub async fn check_pipeline_hotstart(state: State<'_, AppState>) -> Result<(), S return Ok(()); } + // Detect dead pipelines: if the worker thread has exited (init failure or crash), + // clear the pipeline handle so hot-start can retry on the next cycle. + { + let mut hs = state.huddle()?; + if let Some(ref p) = hs.stt_pipeline { + if p.is_finished() { + hs.stt_pipeline = None; + } + } + if let Some(ref p) = hs.tts_pipeline { + if p.is_finished() { + hs.tts_pipeline = None; + } + } + } + // Re-read after potential cleanup. + let (has_stt, has_tts) = { + let hs = state.huddle()?; + (hs.stt_pipeline.is_some(), hs.tts_pipeline.is_some()) + }; + // Check if models just became ready (one-shot flags). let moonshine_ready = models::global_model_manager() .map(|m| m.take_moonshine_ready()) diff --git a/desktop/src-tauri/src/huddle/models.rs b/desktop/src-tauri/src/huddle/models.rs index 39ee318158..7cd49bfd3b 100644 --- a/desktop/src-tauri/src/huddle/models.rs +++ b/desktop/src-tauri/src/huddle/models.rs @@ -33,35 +33,15 @@ const MOONSHINE_ARCHIVE_SHA256: &str = /// SHA-256 hashes for individual Supertonic model files. /// Computed from known-good downloads. Update when upgrading model versions. +#[rustfmt::skip] const SUPERTONIC_FILE_HASHES: &[(&str, &str)] = &[ - ( - "duration_predictor.onnx", - "6d556b3691165c364be91dc0bd894656b5949f5acd2750d8ec2f954010845011", - ), - ( - "text_encoder.onnx", - "dd5f535ed629f7df86071043e15f541ce1b2ab7f1bdbce4c7892b307bca79fa3", - ), - ( - "vector_estimator.onnx", - "105e9d66fd8756876b210a6b4aa03fc393b1eaca3a8dadcc8d9a3bc785c86a35", - ), - ( - "vocoder.onnx", - "19bd51f47a186069c752403518a40f7ea4c647455056d2511f7249691ecddf7c", - ), - ( - "tts.json", - "ee531d9af9b80438a2ed703e22155ee6c83b12595ab22fd3bb6de94c7502fe96", - ), - ( - "unicode_indexer.json", - "b7662a73a0703f43b97c0f2e089f8e8325e26f5d841aca393b5a54c509c92df1", - ), - ( - "F1.json", - "6106950ebeb8a5da29ea22075f605db659cd07dbc288a68292543d9129aa250f", - ), + ("duration_predictor.onnx", "6d556b3691165c364be91dc0bd894656b5949f5acd2750d8ec2f954010845011"), + ("text_encoder.onnx", "dd5f535ed629f7df86071043e15f541ce1b2ab7f1bdbce4c7892b307bca79fa3"), + ("vector_estimator.onnx", "105e9d66fd8756876b210a6b4aa03fc393b1eaca3a8dadcc8d9a3bc785c86a35"), + ("vocoder.onnx", "19bd51f47a186069c752403518a40f7ea4c647455056d2511f7249691ecddf7c"), + ("tts.json", "ee531d9af9b80438a2ed703e22155ee6c83b12595ab22fd3bb6de94c7502fe96"), + ("unicode_indexer.json", "b7662a73a0703f43b97c0f2e089f8e8325e26f5d841aca393b5a54c509c92df1"), + ("F1.json", "6106950ebeb8a5da29ea22075f605db659cd07dbc288a68292543d9129aa250f"), ]; // ── Model versioning ────────────────────────────────────────────────────────── @@ -210,324 +190,386 @@ async fn sha256_file(path: &Path) -> Result { Ok(hex::encode(hash)) } -// ── ModelManager ────────────────────────────────────────────────────────────── +// ── Shared HTTP helpers ─────────────────────────────────────────────────────── + +/// Send a GET request and return the response, or a descriptive error. +async fn fetch_url( + client: &reqwest::Client, + url: &str, + label: &str, +) -> Result { + let response = client + .get(url) + .send() + .await + .map_err(|e| format!("download {label} request failed: {e}"))?; + if !response.status().is_success() { + return Err(format!( + "download {label} HTTP {}: {}", + response.status().as_u16(), + response.status().canonical_reason().unwrap_or("unknown"), + )); + } + Ok(response) +} -/// Manages download and location of STT/TTS model files. -/// -/// Cheap to clone — the inner status is behind an `Arc>`. -#[derive(Clone)] -pub struct ModelManager { - /// `~/.sprout/models/` - models_dir: PathBuf, - moonshine_status: Arc>, - supertonic_status: Arc>, - /// Set to `true` when Moonshine download completes during an active huddle. - /// Polled by the huddle system to auto-start STT. - moonshine_just_ready: Arc, - /// Set to `true` when Supertonic download completes during an active huddle. - supertonic_just_ready: Arc, +/// Create (or recreate) a temp directory, removing any stale one first. +async fn fresh_temp_dir(path: &Path) -> Result<(), String> { + if path.exists() { + tokio::fs::remove_dir_all(path) + .await + .map_err(|e| format!("remove stale temp dir: {e}"))?; + } + tokio::fs::create_dir_all(path) + .await + .map_err(|e| format!("create temp dir: {e}")) } -impl ModelManager { - /// Create a new `ModelManager` rooted at `~/.sprout/models/`. - /// - /// Returns `None` if the home directory cannot be resolved. - pub fn new() -> Option { - let models_dir = dirs::home_dir()?.join(".sprout").join("models"); - Some(Self { - models_dir, - moonshine_status: Arc::new(Mutex::new(ModelStatus::NotDownloaded)), - supertonic_status: Arc::new(Mutex::new(ModelStatus::NotDownloaded)), - moonshine_just_ready: Arc::new(AtomicBool::new(false)), - supertonic_just_ready: Arc::new(AtomicBool::new(false)), - }) +/// Stream an HTTP response to a file with progress reporting and size limits. +/// +/// Calls `progress_fn(bytes_downloaded, content_length)` after each chunk. +/// Returns the total number of bytes written. +async fn download_file( + response: reqwest::Response, + dest: &Path, + max_bytes: u64, + label: &str, + progress_fn: F, +) -> Result +where + F: Fn(u64, Option), +{ + use tokio::io::AsyncWriteExt; + + let content_length = response.content_length(); + if let Some(total) = content_length { + if total > max_bytes { + return Err(format!( + "download {label} too large: {total} bytes (max {max_bytes})" + )); + } } - // ── Moonshine ───────────────────────────────────────────────────────────── + let mut file = tokio::fs::File::create(dest) + .await + .map_err(|e| format!("create {label}: {e}"))?; + let mut downloaded: u64 = 0; + let mut response = response; - /// Returns the path to the Moonshine model directory, or `None` if not ready. - pub fn moonshine_model_dir(&self) -> Option { - if self.is_moonshine_ready() { - Some(self.models_dir.join(MOONSHINE_MODEL_DIR_NAME)) - } else { - None + while let Some(chunk) = response + .chunk() + .await + .map_err(|e| format!("download {label} stream error: {e}"))? + { + downloaded += chunk.len() as u64; + if downloaded > max_bytes { + let _ = tokio::fs::remove_file(dest).await; + return Err(format!( + "download {label} exceeded max size: {downloaded} bytes (max {max_bytes})" + )); } + file.write_all(&chunk) + .await + .map_err(|e| format!("write {label}: {e}"))?; + progress_fn(downloaded, content_length); } - /// Returns `true` if all expected Moonshine model files are present on disk - /// and the version manifest matches the compiled-in version. - pub fn is_moonshine_ready(&self) -> bool { - let dir = self.models_dir.join(MOONSHINE_MODEL_DIR_NAME); - let manifest_ok = std::fs::read_to_string(dir.join(MANIFEST_FILENAME)) - .map(|v| v.trim() == MOONSHINE_MODEL_VERSION) - .unwrap_or(false); - manifest_ok - && MOONSHINE_EXPECTED_FILES - .iter() - .all(|f| dir.join(f).is_file()) - } + file.flush() + .await + .map_err(|e| format!("flush {label}: {e}"))?; + Ok(downloaded) +} - /// Current Moonshine download status. - pub fn moonshine_status(&self) -> ModelStatus { - self.moonshine_status - .lock() - .unwrap_or_else(|e| e.into_inner()) - .clone() - } +// ── ModelSlot ───────────────────────────────────────────────────────────────── - /// Returns true (once) if Moonshine just became ready. Resets the flag. - pub fn take_moonshine_ready(&self) -> bool { - self.moonshine_just_ready.swap(false, Ordering::AcqRel) +/// Per-model state + config. `ModelManager` owns two of these (moonshine, supertonic). +#[derive(Clone)] +struct ModelSlot { + dir_name: &'static str, // subdir under ~/.sprout/models/ + expected_files: &'static [&'static str], // files required for "ready" + version: &'static str, // manifest version; increment to force re-download + status: Arc>, + just_ready: Arc, // fires once when download completes +} + +impl ModelSlot { + fn new( + dir_name: &'static str, + expected_files: &'static [&'static str], + version: &'static str, + ) -> Self { + Self { + dir_name, + expected_files, + version, + status: Arc::new(Mutex::new(ModelStatus::NotDownloaded)), + just_ready: Arc::new(AtomicBool::new(false)), + } } - // ── Supertonic ──────────────────────────────────────────────────────────── + fn model_dir(&self, models_dir: &Path) -> PathBuf { + models_dir.join(self.dir_name) + } - /// Returns the path to the Supertonic model directory, or `None` if not ready. - pub fn supertonic_model_dir(&self) -> Option { - if self.is_supertonic_ready() { - Some(self.models_dir.join(SUPERTONIC_MODEL_DIR_NAME)) - } else { - None - } + fn is_ready(&self, models_dir: &Path) -> bool { + let dir = self.model_dir(models_dir); + std::fs::read_to_string(dir.join(MANIFEST_FILENAME)) + .map(|v| v.trim() == self.version) + .unwrap_or(false) + && self.expected_files.iter().all(|f| dir.join(f).is_file()) } - /// Returns `true` if all expected Supertonic model files are present on disk - /// and the version manifest matches the compiled-in version. - pub fn is_supertonic_ready(&self) -> bool { - let dir = self.models_dir.join(SUPERTONIC_MODEL_DIR_NAME); - let manifest_ok = std::fs::read_to_string(dir.join(MANIFEST_FILENAME)) - .map(|v| v.trim() == SUPERTONIC_MODEL_VERSION) - .unwrap_or(false); - manifest_ok - && SUPERTONIC_EXPECTED_FILES - .iter() - .all(|f| dir.join(f).is_file()) + fn dir_if_ready(&self, models_dir: &Path) -> Option { + self.is_ready(models_dir) + .then(|| self.model_dir(models_dir)) } - /// Current Supertonic download status. - pub fn supertonic_status(&self) -> ModelStatus { - self.supertonic_status + fn status(&self) -> ModelStatus { + self.status .lock() .unwrap_or_else(|e| e.into_inner()) .clone() } - - /// Returns true (once) if Supertonic just became ready. Resets the flag. - pub fn take_supertonic_ready(&self) -> bool { - self.supertonic_just_ready.swap(false, Ordering::AcqRel) + fn set_status(&self, s: ModelStatus) { + *self.status.lock().unwrap_or_else(|e| e.into_inner()) = s; + } + fn take_ready(&self) -> bool { + self.just_ready.swap(false, Ordering::AcqRel) } - /// Trigger a background download of the Supertonic TTS model (~253 MB total). - /// - /// Returns immediately. Progress is tracked via `supertonic_status()`. - /// No-op if the model is already ready or a download is already running. - pub fn start_supertonic_download(&self, http_client: reqwest::Client) { - if self.is_supertonic_ready() { - *self - .supertonic_status - .lock() - .unwrap_or_else(|e| e.into_inner()) = ModelStatus::Ready; + /// Spawn a background download task if not already ready or downloading. + fn start_download( + &self, + models_dir: &Path, + http_client: reqwest::Client, + name: &'static str, + download_fn: F, + ) where + F: FnOnce(reqwest::Client) -> Fut + Send + 'static, + Fut: std::future::Future> + Send, + { + if self.is_ready(models_dir) { + self.set_status(ModelStatus::Ready); return; } - { - let mut status = self - .supertonic_status - .lock() - .unwrap_or_else(|e| e.into_inner()); - match *status { + let mut st = self.status.lock().unwrap_or_else(|e| e.into_inner()); + match *st { ModelStatus::Downloading { .. } | ModelStatus::Ready => return, _ => {} } - *status = ModelStatus::Downloading { + *st = ModelStatus::Downloading { progress_percent: 0, }; } - - let manager = self.clone(); + let slot = self.clone(); tokio::spawn(async move { - if let Err(e) = manager.download_supertonic_model(http_client).await { - eprintln!("sprout-desktop: supertonic download failed: {e}"); - *manager - .supertonic_status - .lock() - .unwrap_or_else(|e2| e2.into_inner()) = ModelStatus::Error(e); + if let Err(e) = download_fn(http_client).await { + eprintln!("sprout-desktop: {name} download failed: {e}"); + slot.set_status(ModelStatus::Error(e)); } }); } - /// Trigger a background download of the Moonshine model. - /// - /// Returns immediately. Progress is tracked via `moonshine_status()`. - /// No-op if the model is already ready or a download is already running. - pub fn start_moonshine_download(&self, http_client: reqwest::Client) { - if self.is_moonshine_ready() { - *self - .moonshine_status - .lock() - .unwrap_or_else(|e| e.into_inner()) = ModelStatus::Ready; - return; + /// Verify files in `source_dir`, atomic-swap into final location, write manifest, signal ready. + /// `temp_cleanup`: optional extra dir to remove (e.g. outer extraction dir for Moonshine). + async fn verify_and_install( + &self, + models_dir: &Path, + source_dir: &Path, + temp_cleanup: Option<&Path>, + ) -> Result<(), String> { + let missing: Vec<&str> = self + .expected_files + .iter() + .filter(|&&f| !source_dir.join(f).is_file()) + .copied() + .collect(); + if !missing.is_empty() { + return Err(format!( + "model verification failed — missing: {}", + missing.join(", ") + )); } - { - let mut status = self - .moonshine_status - .lock() - .unwrap_or_else(|e| e.into_inner()); - match *status { - ModelStatus::Downloading { .. } | ModelStatus::Ready => return, - _ => {} + let final_dir = self.model_dir(models_dir); + let backup_dir = final_dir.with_extension("old"); + + if final_dir.exists() { + if backup_dir.exists() { + let _ = tokio::fs::remove_dir_all(&backup_dir).await; } - *status = ModelStatus::Downloading { - progress_percent: 0, - }; + tokio::fs::rename(&final_dir, &backup_dir) + .await + .map_err(|e| format!("backup old model: {e}"))?; } - - let manager = self.clone(); - tokio::spawn(async move { - if let Err(e) = manager.download_moonshine_model(http_client).await { - eprintln!("sprout-desktop: moonshine download failed: {e}"); - *manager - .moonshine_status - .lock() - .unwrap_or_else(|e2| e2.into_inner()) = ModelStatus::Error(e); + if let Err(e) = tokio::fs::rename(source_dir, &final_dir).await { + if backup_dir.exists() { + let _ = tokio::fs::rename(&backup_dir, &final_dir).await; } - }); - } - - // ── Private ─────────────────────────────────────────────────────────────── + return Err(format!("install new model: {e}")); + } - fn set_moonshine_status(&self, status: ModelStatus) { - *self - .moonshine_status - .lock() - .unwrap_or_else(|e| e.into_inner()) = status; - } + std::fs::write(final_dir.join(MANIFEST_FILENAME), self.version) + .map_err(|e| format!("write model manifest: {e}"))?; + let _ = tokio::fs::remove_dir_all(&backup_dir).await; + if let Some(extra) = temp_cleanup { + let _ = tokio::fs::remove_dir_all(extra).await; + } - fn set_supertonic_status(&self, status: ModelStatus) { - *self - .supertonic_status - .lock() - .unwrap_or_else(|e| e.into_inner()) = status; + self.set_status(ModelStatus::Ready); + self.just_ready.store(true, Ordering::Release); + Ok(()) } +} - /// Download, extract, and verify the Moonshine model archive. - async fn download_moonshine_model(&self, http_client: reqwest::Client) -> Result<(), String> { - tokio::fs::create_dir_all(&self.models_dir) - .await - .map_err(|e| format!("create models dir: {e}"))?; - - self.set_moonshine_status(ModelStatus::Downloading { - progress_percent: 0, - }); +// ── ModelManager ────────────────────────────────────────────────────────────── - let archive_path = self.models_dir.join("moonshine-tiny.tar.bz2"); +/// Manages download and location of STT/TTS model files. +/// +/// Cheap to clone — all inner state is behind `Arc`. +#[derive(Clone)] +pub struct ModelManager { + /// `~/.sprout/models/` + models_dir: PathBuf, + moonshine: ModelSlot, + supertonic: ModelSlot, +} - eprintln!("sprout-desktop: downloading Moonshine model from {MOONSHINE_DOWNLOAD_URL}"); +impl ModelManager { + /// Create a new `ModelManager` rooted at `~/.sprout/models/`. + /// + /// Returns `None` if the home directory cannot be resolved. + pub fn new() -> Option { + let models_dir = dirs::home_dir()?.join(".sprout").join("models"); + Some(Self { + models_dir, + moonshine: ModelSlot::new( + MOONSHINE_MODEL_DIR_NAME, + MOONSHINE_EXPECTED_FILES, + MOONSHINE_MODEL_VERSION, + ), + supertonic: ModelSlot::new( + SUPERTONIC_MODEL_DIR_NAME, + SUPERTONIC_EXPECTED_FILES, + SUPERTONIC_MODEL_VERSION, + ), + }) + } - let response = http_client - .get(MOONSHINE_DOWNLOAD_URL) - .send() - .await - .map_err(|e| format!("download request failed: {e}"))?; + // ── Moonshine accessors ─────────────────────────────────────────────────── - if !response.status().is_success() { - return Err(format!( - "download HTTP {}: {}", - response.status().as_u16(), - response.status().canonical_reason().unwrap_or("unknown"), - )); - } + /// Path to the Moonshine model directory, or `None` if not ready. + pub fn moonshine_model_dir(&self) -> Option { + self.moonshine.dir_if_ready(&self.models_dir) + } + /// `true` if all Moonshine files are present and the manifest version matches. + pub fn is_moonshine_ready(&self) -> bool { + self.moonshine.is_ready(&self.models_dir) + } + /// Current Moonshine download status. + pub fn moonshine_status(&self) -> ModelStatus { + self.moonshine.status() + } + /// Returns `true` once when Moonshine just became ready. Resets the flag. + pub fn take_moonshine_ready(&self) -> bool { + self.moonshine.take_ready() + } - let content_length = response.content_length(); + // ── Supertonic accessors ────────────────────────────────────────────────── - // Reject unexpectedly large downloads before we start. - if let Some(total) = content_length { - if total > MAX_MOONSHINE_DOWNLOAD_BYTES { - return Err(format!( - "download too large: {total} bytes (max {MAX_MOONSHINE_DOWNLOAD_BYTES})" - )); - } - } + /// Path to the Supertonic model directory, or `None` if not ready. + pub fn supertonic_model_dir(&self) -> Option { + self.supertonic.dir_if_ready(&self.models_dir) + } + /// `true` if all Supertonic files are present and the manifest version matches. + pub fn is_supertonic_ready(&self) -> bool { + self.supertonic.is_ready(&self.models_dir) + } + /// Current Supertonic download status. + pub fn supertonic_status(&self) -> ModelStatus { + self.supertonic.status() + } + /// Returns `true` once when Supertonic just became ready. Resets the flag. + pub fn take_supertonic_ready(&self) -> bool { + self.supertonic.take_ready() + } - // Stream to disk instead of buffering the entire archive in memory. - { - use tokio::io::AsyncWriteExt; + // ── Download triggers ───────────────────────────────────────────────────── - let mut file = tokio::fs::File::create(&archive_path) - .await - .map_err(|e| format!("create archive file: {e}"))?; + /// Start a background Moonshine download. No-op if already ready or downloading. + pub fn start_moonshine_download(&self, http_client: reqwest::Client) { + let manager = self.clone(); + self.moonshine.start_download( + &self.models_dir, + http_client, + "moonshine", + move |client| async move { manager.download_moonshine_model(client).await }, + ); + } - let mut downloaded: u64 = 0; - let mut response = response; + /// Start a background Supertonic download (~253 MB). No-op if already ready or downloading. + pub fn start_supertonic_download(&self, http_client: reqwest::Client) { + let manager = self.clone(); + self.supertonic.start_download( + &self.models_dir, + http_client, + "supertonic", + move |client| async move { manager.download_supertonic_model(client).await }, + ); + } - while let Some(chunk) = response - .chunk() - .await - .map_err(|e| format!("download stream error: {e}"))? - { - downloaded += chunk.len() as u64; + // ── Private download implementations ───────────────────────────────────── - // Guard against servers that lie about content-length. - if downloaded > MAX_MOONSHINE_DOWNLOAD_BYTES { - let _ = tokio::fs::remove_file(&archive_path).await; - return Err(format!( - "download exceeded max size during streaming: \ - {downloaded} bytes (max {MAX_MOONSHINE_DOWNLOAD_BYTES})" - )); - } + /// Download, extract, and verify the Moonshine model archive. + async fn download_moonshine_model(&self, http_client: reqwest::Client) -> Result<(), String> { + tokio::fs::create_dir_all(&self.models_dir) + .await + .map_err(|e| format!("create models dir: {e}"))?; - file.write_all(&chunk) - .await - .map_err(|e| format!("write archive: {e}"))?; + let archive_path = self.models_dir.join("moonshine-tiny.tar.bz2"); + let temp_dir = self.models_dir.join("moonshine-tiny.tmp"); + eprintln!("sprout-desktop: downloading Moonshine model from {MOONSHINE_DOWNLOAD_URL}"); + let response = fetch_url(&http_client, MOONSHINE_DOWNLOAD_URL, "moonshine archive").await?; + + let slot = self.moonshine.clone(); + let bytes = download_file( + response, + &archive_path, + MAX_MOONSHINE_DOWNLOAD_BYTES, + "moonshine archive", + |downloaded, content_length| { if let Some(total) = content_length { if total > 0 { let pct = ((downloaded * 89) / total).min(89) as u8; - self.set_moonshine_status(ModelStatus::Downloading { + slot.set_status(ModelStatus::Downloading { progress_percent: pct, }); } } - } - - file.flush() - .await - .map_err(|e| format!("flush archive: {e}"))?; - - eprintln!("sprout-desktop: downloaded {downloaded} bytes, wrote to disk"); - } + }, + ) + .await?; + eprintln!("sprout-desktop: downloaded {bytes} bytes, wrote to disk"); // Verify archive integrity before extraction. let hash = sha256_file(&archive_path).await?; if hash != MOONSHINE_ARCHIVE_SHA256 { let _ = tokio::fs::remove_file(&archive_path).await; return Err(format!( - "Moonshine archive integrity check failed: expected {}, got {}", - MOONSHINE_ARCHIVE_SHA256, hash + "Moonshine archive integrity check failed: expected {MOONSHINE_ARCHIVE_SHA256}, got {hash}" )); } - self.set_moonshine_status(ModelStatus::Downloading { + self.moonshine.set_status(ModelStatus::Downloading { progress_percent: 90, }); - - let temp_dir = self.models_dir.join("moonshine-tiny.tmp"); - let final_dir = self.models_dir.join(MOONSHINE_MODEL_DIR_NAME); - - if temp_dir.exists() { - tokio::fs::remove_dir_all(&temp_dir) - .await - .map_err(|e| format!("remove stale temp dir: {e}"))?; - } - tokio::fs::create_dir_all(&temp_dir) - .await - .map_err(|e| format!("create temp dir: {e}"))?; + fresh_temp_dir(&temp_dir).await?; eprintln!("sprout-desktop: extracting Moonshine archive…"); - let archive_path_clone = archive_path.clone(); - let temp_dir_clone = temp_dir.clone(); - tokio::task::spawn_blocking(move || extract_archive(&archive_path_clone, &temp_dir_clone)) + let (ap, td) = (archive_path.clone(), temp_dir.clone()); + tokio::task::spawn_blocking(move || extract_archive(&ap, &td)) .await .map_err(|e| format!("tar task panicked: {e}"))??; @@ -535,57 +577,26 @@ impl ModelManager { if !extracted_subdir.is_dir() { let _ = tokio::fs::remove_dir_all(&temp_dir).await; return Err(format!( - "expected subdir '{}' not found after extraction", - MOONSHINE_ARCHIVE_SUBDIR, + "expected subdir '{MOONSHINE_ARCHIVE_SUBDIR}' not found after extraction" )); } - let missing: Vec<&str> = MOONSHINE_EXPECTED_FILES - .iter() - .filter(|&&f| !extracted_subdir.join(f).is_file()) - .copied() - .collect(); - - if !missing.is_empty() { + // verify_and_install takes the subdir (actual model files); temp_cleanup removes outer dir. + if let Err(e) = self + .moonshine + .verify_and_install(&self.models_dir, &extracted_subdir, Some(&temp_dir)) + .await + { let _ = tokio::fs::remove_dir_all(&temp_dir).await; - return Err(format!( - "model verification failed — missing: {}", - missing.join(", "), - )); - } - - let backup_dir = final_dir.with_extension("old"); - if final_dir.exists() { - if backup_dir.exists() { - let _ = tokio::fs::remove_dir_all(&backup_dir).await; - } - tokio::fs::rename(&final_dir, &backup_dir) - .await - .map_err(|e| format!("backup old model: {e}"))?; - } - - if let Err(e) = tokio::fs::rename(&extracted_subdir, &final_dir).await { - if backup_dir.exists() { - let _ = tokio::fs::rename(&backup_dir, &final_dir).await; - } - return Err(format!("install new model: {e}")); + let _ = tokio::fs::remove_file(&archive_path).await; + return Err(e); } - - // Write version manifest for cache invalidation on future upgrades. - std::fs::write(final_dir.join(MANIFEST_FILENAME), MOONSHINE_MODEL_VERSION) - .map_err(|e| format!("write model manifest: {e}"))?; - - let _ = tokio::fs::remove_dir_all(&backup_dir).await; - let _ = tokio::fs::remove_dir_all(&temp_dir).await; let _ = tokio::fs::remove_file(&archive_path).await; eprintln!( "sprout-desktop: Moonshine model ready at {}", - final_dir.display() + self.moonshine.model_dir(&self.models_dir).display() ); - - self.set_moonshine_status(ModelStatus::Ready); - self.moonshine_just_ready.store(true, Ordering::Release); Ok(()) } @@ -598,21 +609,8 @@ impl ModelManager { .await .map_err(|e| format!("create models dir: {e}"))?; - self.set_supertonic_status(ModelStatus::Downloading { - progress_percent: 0, - }); - - let final_dir = self.models_dir.join(SUPERTONIC_MODEL_DIR_NAME); let temp_dir = self.models_dir.join("supertonic.tmp"); - - if temp_dir.exists() { - tokio::fs::remove_dir_all(&temp_dir) - .await - .map_err(|e| format!("remove stale temp dir: {e}"))?; - } - tokio::fs::create_dir_all(&temp_dir) - .await - .map_err(|e| format!("create temp dir: {e}"))?; + fresh_temp_dir(&temp_dir).await?; // (url_suffix, local_filename) let downloads: &[(&str, &str)] = &[ @@ -624,166 +622,83 @@ impl ModelManager { ("onnx/unicode_indexer.json", "unicode_indexer.json"), ("voice_styles/F1.json", "F1.json"), ]; - let total_files = downloads.len() as u32; for (i, (url_suffix, filename)) in downloads.iter().enumerate() { let url = format!("{SUPERTONIC_HF_BASE}/{url_suffix}"); eprintln!("sprout-desktop: downloading Supertonic {filename} from {url}"); - let response = http_client - .get(&url) - .send() - .await - .map_err(|e| format!("download {filename} request failed: {e}"))?; - - if !response.status().is_success() { - let _ = tokio::fs::remove_dir_all(&temp_dir).await; - return Err(format!( - "download {filename} HTTP {}: {}", - response.status().as_u16(), - response.status().canonical_reason().unwrap_or("unknown"), - )); - } - - let file_content_length = response.content_length(); - - // Reject unexpectedly large files before we start. - if let Some(total) = file_content_length { - if total > MAX_SUPERTONIC_FILE_BYTES { - let _ = tokio::fs::remove_dir_all(&temp_dir).await; - return Err(format!( - "download {filename} too large: {total} bytes \ - (max {MAX_SUPERTONIC_FILE_BYTES})" - )); - } - } + let response = fetch_url(&http_client, &url, filename).await.map_err(|e| { + let _ = std::fs::remove_dir_all(&temp_dir); + e + })?; - // Stream to disk instead of buffering the entire file in memory. - use tokio::io::AsyncWriteExt; let dest = temp_dir.join(filename); - let mut file = tokio::fs::File::create(&dest) - .await - .map_err(|e| format!("create {filename}: {e}"))?; - - let mut downloaded: u64 = 0; - let mut response = response; - - while let Some(chunk) = response - .chunk() - .await - .map_err(|e| format!("download {filename} stream error: {e}"))? - { - downloaded += chunk.len() as u64; - - // Guard against servers that lie about content-length. - if downloaded > MAX_SUPERTONIC_FILE_BYTES { - let _ = tokio::fs::remove_file(&dest).await; - let _ = tokio::fs::remove_dir_all(&temp_dir).await; - return Err(format!( - "download {filename} exceeded max size during streaming: \ - {downloaded} bytes (max {MAX_SUPERTONIC_FILE_BYTES})" - )); - } - - file.write_all(&chunk) - .await - .map_err(|e| format!("write {filename}: {e}"))?; - - // Progress: spread 0–89% across all files, with intra-file granularity. - if let Some(total) = file_content_length { - if total > 0 { - let file_frac = downloaded as f64 / total as f64; - let base = (i as f64 / total_files as f64) * 89.0; - let span = 89.0 / total_files as f64; - let pct = (base + span * file_frac).min(89.0) as u8; - self.set_supertonic_status(ModelStatus::Downloading { - progress_percent: pct, - }); + let slot = self.supertonic.clone(); + let file_index = i as u32; + let bytes = download_file( + response, + &dest, + MAX_SUPERTONIC_FILE_BYTES, + filename, + |downloaded, content_length| { + if let Some(total) = content_length { + if total > 0 { + let file_frac = downloaded as f64 / total as f64; + let base = (file_index as f64 / total_files as f64) * 89.0; + let span = 89.0 / total_files as f64; + let pct = (base + span * file_frac).min(89.0) as u8; + slot.set_status(ModelStatus::Downloading { + progress_percent: pct, + }); + } } - } - } - - file.flush() - .await - .map_err(|e| format!("flush {filename}: {e}"))?; - - eprintln!("sprout-desktop: downloaded {downloaded} bytes ({filename}), wrote to disk"); + }, + ) + .await + .map_err(|e| { + let _ = std::fs::remove_dir_all(&temp_dir); + e + })?; + eprintln!("sprout-desktop: downloaded {bytes} bytes ({filename}), wrote to disk"); // Verify file integrity against pinned hash. - let expected_hash = SUPERTONIC_FILE_HASHES - .iter() - .find(|(name, _)| *name == *filename) - .map(|(_, hash)| *hash); - - if let Some(expected) = expected_hash { + if let Some(&(_, expected)) = + SUPERTONIC_FILE_HASHES.iter().find(|(n, _)| *n == *filename) + { let actual = sha256_file(&dest).await?; if actual != expected { let _ = tokio::fs::remove_dir_all(&temp_dir).await; return Err(format!( - "Supertonic {filename} integrity check failed: \ - expected {expected}, got {actual}" + "Supertonic {filename} integrity check failed: expected {expected}, got {actual}" )); } } // Ensure progress reflects file completion even without content-length. let pct = (((i as u32 + 1) * 89) / total_files).min(89) as u8; - self.set_supertonic_status(ModelStatus::Downloading { + self.supertonic.set_status(ModelStatus::Downloading { progress_percent: pct, }); } - self.set_supertonic_status(ModelStatus::Downloading { + self.supertonic.set_status(ModelStatus::Downloading { progress_percent: 90, }); - // Verify all expected files landed in the temp dir. - let missing: Vec<&str> = SUPERTONIC_EXPECTED_FILES - .iter() - .filter(|&&f| !temp_dir.join(f).is_file()) - .copied() - .collect(); - - if !missing.is_empty() { + if let Err(e) = self + .supertonic + .verify_and_install(&self.models_dir, &temp_dir, None) + .await + { let _ = tokio::fs::remove_dir_all(&temp_dir).await; - return Err(format!( - "supertonic model verification failed — missing: {}", - missing.join(", "), - )); - } - - // Atomic swap. - let backup_dir = final_dir.with_extension("old"); - if final_dir.exists() { - if backup_dir.exists() { - let _ = tokio::fs::remove_dir_all(&backup_dir).await; - } - tokio::fs::rename(&final_dir, &backup_dir) - .await - .map_err(|e| format!("backup old supertonic model: {e}"))?; - } - - if let Err(e) = tokio::fs::rename(&temp_dir, &final_dir).await { - if backup_dir.exists() { - let _ = tokio::fs::rename(&backup_dir, &final_dir).await; - } - return Err(format!("install new supertonic model: {e}")); + return Err(e); } - // Write version manifest for cache invalidation on future upgrades. - std::fs::write(final_dir.join(MANIFEST_FILENAME), SUPERTONIC_MODEL_VERSION) - .map_err(|e| format!("write model manifest: {e}"))?; - - let _ = tokio::fs::remove_dir_all(&backup_dir).await; - eprintln!( "sprout-desktop: Supertonic model ready at {}", - final_dir.display() + self.supertonic.model_dir(&self.models_dir).display() ); - - self.set_supertonic_status(ModelStatus::Ready); - self.supertonic_just_ready.store(true, Ordering::Release); Ok(()) } } @@ -822,14 +737,3 @@ pub fn is_supertonic_ready() -> bool { .map(|m| m.is_supertonic_ready()) .unwrap_or(false) } - -/// Path to a specific voice style JSON, or `None` if not downloaded. -pub fn voice_style_path(voice_name: &str) -> Option { - let dir = supertonic_model_dir()?; - let path = dir.join(format!("{voice_name}.json")); - if path.is_file() { - Some(path) - } else { - None - } -} diff --git a/desktop/src-tauri/src/huddle/preprocessing.rs b/desktop/src-tauri/src/huddle/preprocessing.rs index 1c61501473..fd21418601 100644 --- a/desktop/src-tauri/src/huddle/preprocessing.rs +++ b/desktop/src-tauri/src/huddle/preprocessing.rs @@ -113,7 +113,14 @@ pub fn preprocess_for_tts(text: &str) -> String { let s = strip_markdown_markers(&s); let s = strip_emoji(&s); let s = expand_numbers(&s); - collapse_whitespace(&s) + let s = collapse_whitespace(&s); + // Filter trivially short results — ".", ",", etc. would be spoken as + // "period", "comma" by TTS. Agents that have nothing relevant to say + // should not respond at all, but defense-in-depth catches edge cases. + if s.len() <= 1 { + return String::new(); + } + s } // ── Step implementations ────────────────────────────────────────────────────── @@ -587,6 +594,15 @@ mod tests { assert_eq!(result, vec![""]); } + #[test] + fn filters_trivial_responses() { + assert_eq!(preprocess_for_tts("."), ""); + assert_eq!(preprocess_for_tts(","), ""); + assert_eq!(preprocess_for_tts("!"), ""); + assert_eq!(preprocess_for_tts(" "), ""); + assert_eq!(preprocess_for_tts("ok"), "ok"); + } + #[test] fn full_pipeline() { let input = diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index 8a8f688d3e..7a44e85195 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -124,6 +124,12 @@ impl SttPipeline { self.shutdown.store(true, Ordering::Release); } + /// Returns `true` if the worker thread has exited (init failure, crash, or normal exit). + /// Used by hot-start to detect dead pipelines and clear them for retry. + pub fn is_finished(&self) -> bool { + self.thread.as_ref().map_or(true, |h| h.is_finished()) + } + /// Feed raw PCM bytes into the pipeline. /// /// Non-blocking. Drops audio silently if the pipeline can't keep up — @@ -475,6 +481,10 @@ fn flush_to_stt( /// Convert raw bytes (f32 LE) to f32 samples. /// Caller should ensure `bytes.len() % 4 == 0`; extra bytes are silently truncated. +/// +/// Assumes little-endian — matches all current Tauri targets (macOS ARM64, +/// Windows/Linux x86). The JS AudioWorklet's Float32Array uses platform-native +/// byte order, which is LE on all supported platforms. fn bytes_to_f32(bytes: &[u8]) -> Vec { bytes .chunks_exact(4) diff --git a/desktop/src-tauri/src/huddle/tts.rs b/desktop/src-tauri/src/huddle/tts.rs index 8d7f8ec5b9..47fd38660b 100644 --- a/desktop/src-tauri/src/huddle/tts.rs +++ b/desktop/src-tauri/src/huddle/tts.rs @@ -86,8 +86,9 @@ pub struct TtsPipeline { /// Signals the worker thread to stop. shutdown: Arc, /// Cancel flag: worker drains the queue and stops current playback. - /// Public so the STT pipeline can share it for barge-in detection. - pub cancel: Arc, + /// Kept alive here so the Arc isn't dropped — the worker holds a clone. + #[allow(dead_code)] + cancel: Arc, /// Voice name (e.g. "F1"). Stored for future voice-switching support. #[allow(dead_code)] voice: String, @@ -169,23 +170,16 @@ impl TtsPipeline { }) } - /// Barge-in: cancel current speech and discard queued items. - /// - /// Sets the cancel flag. The worker will drain the queue and stop the - /// current rodio Player on its next iteration. - /// - /// Currently unused — barge-in is triggered by the STT pipeline setting - /// `tts_cancel` directly via the shared `Arc`. Retained as - /// public API for future callers (e.g., explicit "stop speaking" button). - #[allow(dead_code)] - pub fn cancel(&self) { - self.cancel.store(true, Ordering::Release); - } - /// Signal the worker thread to stop. pub fn shutdown(&self) { self.shutdown.store(true, Ordering::Release); } + + /// Returns `true` if the worker thread has exited (init failure, crash, or normal exit). + /// Used by hot-start to detect dead pipelines and clear them for retry. + pub fn is_finished(&self) -> bool { + self.thread.as_ref().map_or(true, |h| h.is_finished()) + } } impl Drop for TtsPipeline { @@ -251,15 +245,11 @@ fn tts_worker( // ── 4. Main loop ────────────────────────────────────────────────────────── loop { - if shutdown.load(Ordering::Acquire) { - break; - } - - // Handle cancel: drain queue and clear the flag. - if cancel.load(Ordering::Acquire) { - while text_rx.try_recv().is_ok() {} - cancel.store(false, Ordering::Release); - tts_active.store(false, Ordering::Release); + // Check shutdown/cancel before blocking (no player yet). + if handle_cancel_or_shutdown(&cancel, &shutdown, &tts_active, &text_rx, None) { + if shutdown.load(Ordering::Acquire) { + break; + } continue; } @@ -271,9 +261,10 @@ fn tts_worker( // Check cancel again after unblocking — a cancel may have arrived // while we were waiting. - if cancel.load(Ordering::Acquire) { - while text_rx.try_recv().is_ok() {} - cancel.store(false, Ordering::Release); + if handle_cancel_or_shutdown(&cancel, &shutdown, &tts_active, &text_rx, None) { + if shutdown.load(Ordering::Acquire) { + break; + } continue; } @@ -316,10 +307,8 @@ fn tts_worker( tts_active.store(true, Ordering::Release); for chunk in sentences.chunks(BATCH_SIZE) { - if cancel.load(Ordering::Acquire) || shutdown.load(Ordering::Acquire) { - player.clear(); - while text_rx.try_recv().is_ok() {} - cancel.store(false, Ordering::Release); + // Fix 2: tts_active cleared immediately on cancel (inside helper). + if handle_cancel_or_shutdown(&cancel, &shutdown, &tts_active, &text_rx, Some(&player)) { break; } @@ -331,10 +320,8 @@ fn tts_worker( // Wait for all queued audio to finish playing. loop { - if cancel.load(Ordering::Acquire) || shutdown.load(Ordering::Acquire) { - player.clear(); - while text_rx.try_recv().is_ok() {} - cancel.store(false, Ordering::Release); + // Fix 2: tts_active cleared immediately on cancel (inside helper). + if handle_cancel_or_shutdown(&cancel, &shutdown, &tts_active, &text_rx, Some(&player)) { break; } if player.empty() { @@ -355,6 +342,34 @@ fn tts_worker( // ── Helpers ─────────────────────────────────────────────────────────────────── +/// Check for cancel or shutdown. Returns `true` if the caller should break/continue. +/// On cancel: drains the text queue and clears the cancel flag. +fn handle_cancel_or_shutdown( + cancel: &AtomicBool, + shutdown: &AtomicBool, + tts_active: &AtomicBool, + text_rx: &mpsc::Receiver, + player: Option<&rodio::Player>, +) -> bool { + if shutdown.load(Ordering::Acquire) { + if let Some(p) = player { + p.clear(); + } + tts_active.store(false, Ordering::Release); + return true; + } + if cancel.load(Ordering::Acquire) { + if let Some(p) = player { + p.clear(); + } + while text_rx.try_recv().is_ok() {} + cancel.store(false, Ordering::Release); + tts_active.store(false, Ordering::Release); + return true; + } + false +} + /// Synthesize a batch of sentences in a single engine call. /// /// Sentences are joined with a space so Supertonic sees full context for diff --git a/desktop/src/features/huddle/HuddleContext.tsx b/desktop/src/features/huddle/HuddleContext.tsx index 066227fba3..c531acc310 100644 --- a/desktop/src/features/huddle/HuddleContext.tsx +++ b/desktop/src/features/huddle/HuddleContext.tsx @@ -50,7 +50,7 @@ interface HuddleContextValue { * Returns true if backend cleanup succeeded, false if it failed (caller may retry). */ leaveHuddle: () => Promise; /** End the huddle (creator only) — archives ephemeral channel, emits huddle_ended */ - endHuddle: () => Promise; + endHuddle: () => Promise; } const HuddleContext = React.createContext(null); @@ -114,22 +114,27 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { return true; // Backend cleanup succeeded (or was not needed) }, [disconnectMedia]); - const endHuddle = React.useCallback(async (): Promise => { + const endHuddle = React.useCallback(async (): Promise => { await disconnectMedia(); if (rustActiveRef.current) { try { await invoke("end_huddle"); rustActiveRef.current = false; + return true; } catch { - // Fall back to leave_huddle + // end_huddle failed — fall back to local leave so we at least + // disconnect, but report false so the UI knows the huddle was + // NOT ended for everyone (no archive, no huddle_ended event). try { await invoke("leave_huddle"); rustActiveRef.current = false; } catch { - // Leave rustActiveRef true for retry + // Leave rustActiveRef true so a subsequent call retries } + return false; } } + return true; }, [disconnectMedia]); /** @@ -335,7 +340,7 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { // Only speak agent messages — skip human STT transcripts. if (!agentPubkeys.has(event.pubkey)) return; if (event.pubkey === selfPubkeyRef.current) return; - if (!event.content.trim()) return; + if (event.content.trim().length <= 1) return; // Legacy: skip [System]-prefixed messages from before kind:48106. if (event.content.startsWith("[System]")) return; diff --git a/desktop/src/features/huddle/components/HuddleBar.tsx b/desktop/src/features/huddle/components/HuddleBar.tsx index 0006d0f2ed..dcbbc2630c 100644 --- a/desktop/src/features/huddle/components/HuddleBar.tsx +++ b/desktop/src/features/huddle/components/HuddleBar.tsx @@ -17,7 +17,7 @@ import { AddAgentDialog, type AgentAddResult } from "./AddAgentDialog"; import { ParticipantList } from "./ParticipantList"; // Shape returned by the `get_huddle_state` Tauri command. -// NOTE: This mirrors the HuddleState struct in the Rust backend (src-tauri/src/huddle/state.rs). +// NOTE: This mirrors the HuddleState struct in the Rust backend (src-tauri/src/huddle/mod.rs). // If you add/remove fields here, update the Rust struct (and vice versa). type HuddleState = { phase: @@ -113,8 +113,11 @@ export function HuddleBar({ className }: HuddleBarProps) { if (isLeaving) return; setIsLeaving(true); try { - await endHuddle(); - setState(null); + const backendClean = await endHuddle(); + if (backendClean) { + setState(null); + } + // If backend cleanup failed, keep the bar visible so the user can retry. } catch (e) { console.error("Failed to end huddle:", e); } finally { diff --git a/desktop/src/features/huddle/components/ParticipantList.tsx b/desktop/src/features/huddle/components/ParticipantList.tsx index fe22100d91..426c6a6be4 100644 --- a/desktop/src/features/huddle/components/ParticipantList.tsx +++ b/desktop/src/features/huddle/components/ParticipantList.tsx @@ -1,4 +1,6 @@ import { cn } from "@/shared/lib/cn"; +import { useUsersBatchQuery } from "@/features/profile/hooks"; +import { ProfileAvatar } from "@/features/profile/ui/ProfileAvatar"; type ParticipantListProps = { /** Pubkey hex strings from the Rust huddle state */ @@ -10,40 +12,44 @@ export function ParticipantList({ participants, className, }: ParticipantListProps) { + const { data } = useUsersBatchQuery(participants); + const profiles = data?.profiles ?? {}; + if (participants.length === 0) return null; return (
    - {participants.map((pubkey) => ( - - ))} + {participants.map((pubkey) => { + const profile = profiles[pubkey.toLowerCase()]; + const hasProfile = profile?.displayName || profile?.avatarUrl; + + return hasProfile ? ( +
    + +
    + ) : ( + + ); + })}
    ); } -type ParticipantAvatarProps = { - pubkey: string; -}; - -function ParticipantAvatar({ pubkey }: ParticipantAvatarProps) { - // Use first 6 hex chars as a short identifier +/** Compact hex-prefix avatar for participants without a loaded profile. */ +function HexAvatar({ pubkey }: { pubkey: string }) { const shortId = pubkey.slice(0, 6).toUpperCase(); - - // Derive a stable hue from the pubkey. Falls back to neutral gray on invalid hex. const parsed = parseInt(pubkey.slice(0, 4), 16); const hue = Number.isNaN(parsed) ? 0 : parsed % 360; - const saturation = Number.isNaN(parsed) ? 0 : 60; - const style = { - backgroundColor: `hsl(${hue}, ${saturation}%, 55%)`, - color: "#fff", - }; + const sat = Number.isNaN(parsed) ? 0 : 60; return (
    {shortId} diff --git a/desktop/src/features/huddle/lib/livekit.ts b/desktop/src/features/huddle/lib/livekit.ts index 66663a6452..0acc308fcb 100644 --- a/desktop/src/features/huddle/lib/livekit.ts +++ b/desktop/src/features/huddle/lib/livekit.ts @@ -48,10 +48,13 @@ export async function connectToHuddle( room, localAudioTrack: audioTrack, disconnect: async () => { - room.disconnect(); - stream?.getTracks().forEach((t) => { - t.stop(); - }); + try { + room.disconnect(); + } finally { + stream?.getTracks().forEach((t) => { + t.stop(); + }); + } }, }; } catch (err) { From 7861a295ff62e82220f5269439a6948459991f10 Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Sun, 12 Apr 2026 23:14:49 -0400 Subject: [PATCH 30/41] =?UTF-8?q?huddles:=20crossfire=20improvements=20?= =?UTF-8?q?=E2=80=94=20fault=20boundary,=20DRY,=20feature-gate,=20UX?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four targeted improvements from crossfire review (codex 9/10, opus 8/10): 1. post_connect_setup fault boundary: model download and member hydration failures no longer tear down a working huddle. The call stays up in degraded mode (no STT/TTS) instead of failing entirely. 2. Remove duplicate preprocessing from supertonic.rs: emoji stripping and whitespace collapsing already handled by preprocessing.rs. Deleted two stale LazyLock statics (RE_EMOJI, RE_WHITESPACE). 3. Feature-gate dead code: webhook.rs and session.rs gated behind #[cfg(feature = "webhook")]. hmac/sha2/hex deps made optional. 3 tests run by default, 8 with --features webhook. 4. Model download progress in HuddleBar: shows 'Voice models: STT 42%, TTS 78%' while downloading, disappears when ready. Serde decoder correctly handles both string and object enum variants. Also: documented WHY dual membership polling (Rust + React) is intentional — Rust preserves stale list on failure (STT p-tags), React clears on failure (TTS authorization must fail-closed). Different safety requirements. --- crates/sprout-huddle/Cargo.toml | 10 ++- crates/sprout-huddle/src/lib.rs | 7 ++ desktop/src-tauri/src/huddle/mod.rs | 22 ++++-- desktop/src-tauri/src/huddle/supertonic.rs | 14 +--- .../features/huddle/components/HuddleBar.tsx | 69 +++++++++++++++++++ 5 files changed, 104 insertions(+), 18 deletions(-) diff --git a/crates/sprout-huddle/Cargo.toml b/crates/sprout-huddle/Cargo.toml index 07356cb51a..3f430a8934 100644 --- a/crates/sprout-huddle/Cargo.toml +++ b/crates/sprout-huddle/Cargo.toml @@ -18,6 +18,10 @@ chrono = { workspace = true } tracing = { workspace = true } thiserror = { workspace = true } jsonwebtoken = { workspace = true } -hmac = { workspace = true } -sha2 = { workspace = true } -hex = { workspace = true } +hmac = { workspace = true, optional = true } +sha2 = { workspace = true, optional = true } +hex = { workspace = true, optional = true } + +[features] +default = [] +webhook = ["hmac", "sha2", "hex"] diff --git a/crates/sprout-huddle/src/lib.rs b/crates/sprout-huddle/src/lib.rs index 1eb3a2c6aa..2d4492cc7d 100644 --- a/crates/sprout-huddle/src/lib.rs +++ b/crates/sprout-huddle/src/lib.rs @@ -9,19 +9,24 @@ /// Error types for the huddle layer. pub mod error; /// In-memory huddle session and participant tracking. +#[cfg(feature = "webhook")] pub mod session; /// LiveKit access token generation. pub mod token; /// LiveKit webhook signature verification and event parsing. +#[cfg(feature = "webhook")] pub mod webhook; pub use error::HuddleError; +#[cfg(feature = "webhook")] pub use session::{HuddleParticipant, HuddleSession, TrackInfo, TrackKind}; pub use token::LiveKitToken; +#[cfg(feature = "webhook")] pub use webhook::WebhookEvent; use uuid::Uuid; +#[cfg(feature = "webhook")] pub use sprout_core::kind::{ KIND_HUDDLE_ENDED, KIND_HUDDLE_PARTICIPANT_JOINED, KIND_HUDDLE_PARTICIPANT_LEFT, KIND_HUDDLE_RECORDING_AVAILABLE, KIND_HUDDLE_STARTED, KIND_HUDDLE_TRACK_PUBLISHED, @@ -74,6 +79,7 @@ impl HuddleService { } /// Verify the webhook signature and parse the LiveKit event payload. + #[cfg(feature = "webhook")] pub fn parse_webhook( &self, body: &[u8], @@ -146,6 +152,7 @@ mod tests { } #[test] + #[cfg(feature = "webhook")] fn session_lifecycle() { let channel_id = Uuid::new_v4(); let room_name = HuddleService::create_room_name(channel_id); diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index 88a4a2be2d..330513757b 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -632,7 +632,10 @@ pub async fn start_huddle( } // 6. Hydrate members, download models, start pipelines. - post_connect_setup(&state, &ephemeral_channel_id).await?; + if let Err(e) = post_connect_setup(&state, &ephemeral_channel_id).await { + eprintln!("sprout-desktop: post_connect_setup failed (degraded mode): {e}"); + // Non-fatal: huddle works without STT/TTS pipelines. + } Ok(HuddleJoinInfo { ephemeral_channel_id, @@ -729,7 +732,10 @@ pub async fn join_huddle( } // 4. Hydrate members, download models, start pipelines. - post_connect_setup(&state, &ephemeral_channel_id).await?; + if let Err(e) = post_connect_setup(&state, &ephemeral_channel_id).await { + eprintln!("sprout-desktop: post_connect_setup failed (degraded mode): {e}"); + // Non-fatal: huddle works without STT/TTS pipelines. + } Ok(HuddleJoinInfo { ephemeral_channel_id, @@ -999,8 +1005,16 @@ pub async fn check_pipeline_hotstart(state: State<'_, AppState>) -> Result<(), S // Periodically refresh agent_pubkeys from relay membership. // This catches mid-huddle agent additions/removals by other participants, // keeping STT p-tags authoritative throughout the session. - // Throttled to every 15 s (not on every 5 s hotstart poll) — the frontend - // already refreshes its own agentPubkeys every 10 s via get_huddle_agent_pubkeys. + // Throttled to every 15 s (not on every 5 s hotstart poll). + // + // NOTE: The frontend ALSO polls agent membership independently (every 10 s + // via get_huddle_agent_pubkeys). This is intentional — the two polls have + // different failure semantics: + // - Rust (here): preserves stale list on failure (STT p-tags should not + // disappear on a transient network blip). + // - React (HuddleContext.tsx): clears list on failure (TTS authorization + // must fail-closed — never speak from a stale agent list). + // // On Ok: always replace (even with empty — agents may have been removed). // On Err: preserve the existing list (transient failure shouldn't zero it). if let Some(eph_id) = &ephemeral_channel_id { diff --git a/desktop/src-tauri/src/huddle/supertonic.rs b/desktop/src-tauri/src/huddle/supertonic.rs index a121b3e093..69e0ae39e0 100644 --- a/desktop/src-tauri/src/huddle/supertonic.rs +++ b/desktop/src-tauri/src/huddle/supertonic.rs @@ -179,12 +179,6 @@ impl UnicodeProcessor { } // ── Compiled regex patterns (one-time init) ─────────────────────────────── -static RE_EMOJI: LazyLock = LazyLock::new(|| { - Regex::new( - r"[\x{1F600}-\x{1F64F}\x{1F300}-\x{1F5FF}\x{1F680}-\x{1F6FF}\x{1F700}-\x{1F77F}\x{1F780}-\x{1F7FF}\x{1F800}-\x{1F8FF}\x{1F900}-\x{1F9FF}\x{1FA00}-\x{1FA6F}\x{1FA70}-\x{1FAFF}\x{2600}-\x{26FF}\x{2700}-\x{27BF}\x{1F1E6}-\x{1F1FF}]+" - ).unwrap() -}); - static RE_SPACE_COMMA: LazyLock = LazyLock::new(|| Regex::new(r" ,").unwrap()); static RE_SPACE_DOT: LazyLock = LazyLock::new(|| Regex::new(r" \.").unwrap()); static RE_SPACE_BANG: LazyLock = LazyLock::new(|| Regex::new(r" !").unwrap()); @@ -192,7 +186,6 @@ static RE_SPACE_QUESTION: LazyLock = LazyLock::new(|| Regex::new(r" \?"). static RE_SPACE_SEMI: LazyLock = LazyLock::new(|| Regex::new(r" ;").unwrap()); static RE_SPACE_COLON: LazyLock = LazyLock::new(|| Regex::new(r" :").unwrap()); static RE_SPACE_APOS: LazyLock = LazyLock::new(|| Regex::new(r" '").unwrap()); -static RE_WHITESPACE: LazyLock = LazyLock::new(|| Regex::new(r"\s+").unwrap()); static RE_ENDS_PUNC: LazyLock = LazyLock::new(|| Regex::new(r#"[.!?;:,'")\]}…。」』】〉》›»]$"#).unwrap()); static RE_PARAGRAPH: LazyLock = LazyLock::new(|| Regex::new(r"\n\s*\n").unwrap()); @@ -202,8 +195,7 @@ static RE_PARAGRAPH: LazyLock = LazyLock::new(|| Regex::new(r"\n\s*\n").u fn preprocess_text(text: &str, lang: &str) -> Result { let mut s: String = text.nfkd().collect(); - // Strip emojis. - s = RE_EMOJI.replace_all(&s, "").to_string(); + // Emoji already stripped by preprocessing.rs::preprocess_for_tts. // Character replacements. for (from, to) in &[ @@ -260,8 +252,8 @@ fn preprocess_text(text: &str, lang: &str) -> Result { s = s.replace("``", "`"); } - // Collapse whitespace. - s = RE_WHITESPACE.replace_all(&s, " ").to_string(); + // Whitespace already collapsed by preprocessing.rs::preprocess_for_tts. + // Trim only — model-specific transforms above may have introduced leading/trailing space. s = s.trim().to_string(); // Ensure terminal punctuation. diff --git a/desktop/src/features/huddle/components/HuddleBar.tsx b/desktop/src/features/huddle/components/HuddleBar.tsx index dcbbc2630c..8f75eba1fa 100644 --- a/desktop/src/features/huddle/components/HuddleBar.tsx +++ b/desktop/src/features/huddle/components/HuddleBar.tsx @@ -51,6 +51,10 @@ export function HuddleBar({ className }: HuddleBarProps) { const [isLeaving, setIsLeaving] = React.useState(false); const [showAddAgent, setShowAddAgent] = React.useState(false); const [agentAddError, setAgentAddError] = React.useState(null); + const [modelStatus, setModelStatus] = React.useState<{ + moonshine: string; + supertonic: string; + } | null>(null); // Poll huddle state — replace with event listener once Rust emits events React.useEffect(() => { @@ -82,6 +86,55 @@ export function HuddleBar({ className }: HuddleBarProps) { }; }, []); + // Poll model download status while huddle is active + const huddlePhase = state?.phase; + React.useEffect(() => { + if (huddlePhase !== "active" && huddlePhase !== "connected") return; + + let cancelled = false; + + // ModelStatus serializes as: "ready" | "not_downloaded" (strings) + // or { downloading: { progress_percent: N } } | { error: "msg" } (objects). + const fmt = (s: unknown): string => { + if (typeof s === "string") return s === "ready" ? "ready" : "pending"; + if (typeof s === "object" && s !== null) { + if ("downloading" in s) { + const d = (s as { downloading: { progress_percent: number } }) + .downloading; + return `${d.progress_percent}%`; + } + if ("error" in s) return "error"; + } + return "pending"; + }; + + async function pollModels() { + try { + const status = await invoke<{ + moonshine: unknown; + supertonic: unknown; + }>("get_model_status"); + if (cancelled) return; + + setModelStatus({ + moonshine: fmt(status.moonshine), + supertonic: fmt(status.supertonic), + }); + } catch { + // best-effort + } + } + + void pollModels(); + const id = window.setInterval(() => void pollModels(), 3_000); + + return () => { + cancelled = true; + window.clearInterval(id); + setModelStatus(null); // Clear stale status on huddle end/phase change. + }; + }, [huddlePhase]); + // Sync mute state to the audio track React.useEffect(() => { if (localAudioTrack) { @@ -143,6 +196,22 @@ export function HuddleBar({ className }: HuddleBarProps) { In huddle
    + {/* Model download progress */} + {modelStatus && + (modelStatus.moonshine !== "ready" || + modelStatus.supertonic !== "ready") && ( +
    + + {modelStatus.moonshine !== "ready" && + modelStatus.supertonic !== "ready" + ? `Voice models: STT ${modelStatus.moonshine}, TTS ${modelStatus.supertonic}` + : modelStatus.moonshine !== "ready" + ? `STT model: ${modelStatus.moonshine}` + : `TTS model: ${modelStatus.supertonic}`} + +
    + )} + {/* Participant avatars */} {state.participants.length > 0 && ( From 414fcacd29d15067c8098c70fa2e279a499cbdde Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Mon, 13 Apr 2026 07:03:47 -0400 Subject: [PATCH 31/41] huddles: allow kind 48106 in relay ingest + preload voice models at app launch 1. Add kind 48106 (huddle guidelines) to the relay's ingest allowlist. The range 48100..=48103 was allowed but 48106 was missing, causing 'restricted: unknown event kind' when posting voice-mode guidelines. Widened to 48100..=48106. 2. Trigger background voice model downloads at app launch (in Tauri setup hook). Models are ~303 MB total (50 MB Moonshine STT + 253 MB Supertonic TTS). Downloads are async, idempotent, SHA-256 verified, and no-op if already cached. First huddle no longer has a cold-start download wait. --- crates/sprout-relay/src/handlers/ingest.rs | 4 ++-- desktop/src-tauri/src/lib.rs | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/sprout-relay/src/handlers/ingest.rs b/crates/sprout-relay/src/handlers/ingest.rs index 736f40c064..a1dbf396a7 100644 --- a/crates/sprout-relay/src/handlers/ingest.rs +++ b/crates/sprout-relay/src/handlers/ingest.rs @@ -176,8 +176,8 @@ fn required_scope_for_kind(kind: u32, event: &Event) -> Result Ok(Scope::ChannelsWrite), KIND_NIP29_JOIN_REQUEST | KIND_NIP29_LEAVE_REQUEST => Ok(Scope::ChannelsRead), - // Huddle lifecycle events (kind 48100–48103) - 48100..=48103 => Ok(Scope::ChannelsWrite), + // Huddle lifecycle events (kind 48100–48103) + guidelines (48106) + 48100..=48106 => Ok(Scope::ChannelsWrite), _ => Err("restricted: unknown event kind"), } } diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index a9dd9f6d79..e0ea036e06 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -402,6 +402,14 @@ pub fn run() { eprintln!("sprout-desktop: failed to create nest: {error}"); } + // Pre-download voice models in the background so they're ready + // when the user starts their first huddle. Idempotent — no-op if + // already downloaded. ~303 MB total (50 MB Moonshine + 253 MB Supertonic). + if let Some(mgr) = huddle::models::global_model_manager() { + mgr.start_moonshine_download(state.http_client.clone()); + mgr.start_supertonic_download(state.http_client.clone()); + } + // Keep launch-time agent restoration off the synchronous setup path // so the frontend can mount and reveal the window promptly. tauri::async_runtime::spawn_blocking(move || { From 908049cfa99b6252d6a2851e03cac2f3585d39e8 Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Mon, 13 Apr 2026 08:16:26 -0400 Subject: [PATCH 32/41] =?UTF-8?q?huddles:=20live-testing=20fixes=20?= =?UTF-8?q?=E2=80=94=20TTS=20playback,=20STT=20tuning,=20barge-in=20disabl?= =?UTF-8?q?ed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes discovered during live huddle testing with agents: **Relay:** - Add kind 48106 (huddle guidelines) to requires_h_channel_scope so guidelines are routed to the ephemeral channel. Agents now see the voice-mode prompt via their subscription. **TTS (tts.rs):** - Prime audio output with 100ms silent buffer at worker startup. On macOS, CoreAudio initializes lazily — without priming, the first Player races against device startup and player.empty() returns true prematurely, truncating the first TTS message after a few words. **STT (stt.rs):** - Disable barge-in (VAD-based TTS cancellation). Without acoustic echo cancellation, any ambient noise — keyboard, fan, breathing — triggers false barge-in and kills TTS mid-sentence. Push-to-talk will replace this; echo-gating (skip accumulation during TTS) is preserved. - Reduce TTS cooldown 200ms → 50ms. The old value ate the first word when the user spoke immediately after the agent finished. - Reduce silence flush threshold 28 → 19 frames (450ms → 300ms). Snappier transcript delivery without splitting mid-word pauses. --- crates/sprout-relay/src/handlers/ingest.rs | 3 +- desktop/src-tauri/src/huddle/stt.rs | 42 ++++++++++------------ desktop/src-tauri/src/huddle/tts.rs | 21 +++++++++-- 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/crates/sprout-relay/src/handlers/ingest.rs b/crates/sprout-relay/src/handlers/ingest.rs index a1dbf396a7..caf31f29de 100644 --- a/crates/sprout-relay/src/handlers/ingest.rs +++ b/crates/sprout-relay/src/handlers/ingest.rs @@ -285,11 +285,12 @@ pub(crate) fn requires_h_channel_scope(kind: u32) -> bool { | KIND_NIP29_DELETE_EVENT | KIND_NIP29_DELETE_GROUP | KIND_NIP29_LEAVE_REQUEST - // Huddle lifecycle events (kind 48100–48103) + // Huddle lifecycle events (kind 48100–48103) + guidelines (48106) | 48100 | 48101 | 48102 | 48103 + | 48106 ) } diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index 7a44e85195..87a728eded 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -163,12 +163,17 @@ impl Drop for SttPipeline { // ── Worker thread ───────────────────────────────────────────────────────────── /// How many 16 kHz samples of silence before we flush to STT. -/// 450 ms × 16 000 Hz / 256 samples-per-frame ≈ 28 frames. -const SILENCE_FLUSH_FRAMES: usize = 28; +/// 300 ms × 16 000 Hz / 256 samples-per-frame ≈ 19 frames. +/// Previous value (28 frames / 450 ms) felt sluggish in conversation. +const SILENCE_FLUSH_FRAMES: usize = 19; /// Consecutive VAD speech frames required before triggering barge-in during TTS. -/// 5 frames × 256 samples / 16 kHz ≈ 80 ms — filters out coughs and transients. -const BARGE_IN_DEBOUNCE_FRAMES: usize = 5; +/// 20 frames × 256 samples / 16 kHz ≈ 320 ms — must be long enough to filter +/// speaker-to-mic feedback (TTS audio bleeding through the mic) while still +/// catching real human interruptions. 80 ms (previous: 5 frames) was too +/// aggressive — laptop speakers without headphones triggered false barge-in +/// within the first word of TTS playback. +const BARGE_IN_DEBOUNCE_FRAMES: usize = 20; /// earshot requires exactly 256 samples per frame at 16 kHz. const VAD_FRAME_SAMPLES: usize = 256; @@ -179,9 +184,11 @@ const VAD_THRESHOLD: f32 = 0.5; /// How long the worker waits on the audio channel before checking the shutdown flag. const RECV_TIMEOUT: Duration = Duration::from_millis(50); -/// 200 ms cooldown after TTS stops before STT re-enables. +/// 50 ms cooldown after TTS stops before STT re-enables. /// Prevents the tail of TTS audio from being transcribed as speech. -const TTS_COOLDOWN: Duration = Duration::from_millis(200); +/// Previous value (200 ms) was eating the first word when the user spoke +/// immediately after the agent finished. +const TTS_COOLDOWN: Duration = Duration::from_millis(50); fn stt_worker( model_dir: PathBuf, @@ -376,26 +383,13 @@ fn process_16k_samples( let tts_playing = tts_active.load(Ordering::Acquire); - // Fix 2 + Fix 4: While TTS is playing, detect barge-in but skip accumulation. + // While TTS is playing: skip accumulation (echo prevention). + // Barge-in is disabled — push-to-talk will replace VAD-based interruption. + // Without acoustic echo cancellation, any ambient noise triggers false + // barge-in and kills TTS playback mid-sentence. if tts_playing { - // Reset segment state — STT is not tracking speech during TTS. - // The barge-in logic below will set in_speech=true only when the - // debounce threshold is met, preventing stale continuation after TTS stops. *in_speech = false; - - if is_speech { - *barge_in_frames += 1; - if *barge_in_frames >= BARGE_IN_DEBOUNCE_FRAMES { - // Sustained speech during TTS → barge-in: cancel TTS. - *in_speech = true; - if let Some(cancel) = tts_cancel { - cancel.store(true, Ordering::Release); - } - } - } else { - *barge_in_frames = 0; - } - // Don't accumulate during TTS — clean slate for when TTS stops. + *barge_in_frames = 0; speech_buf.clear(); *silence_frames = 0; continue; diff --git a/desktop/src-tauri/src/huddle/tts.rs b/desktop/src-tauri/src/huddle/tts.rs index 47fd38660b..0ddce474b7 100644 --- a/desktop/src-tauri/src/huddle/tts.rs +++ b/desktop/src-tauri/src/huddle/tts.rs @@ -243,6 +243,25 @@ fn tts_worker( } }; + // Prime the audio output stream with a short silent buffer. + // On macOS, CoreAudio initializes the output device lazily on first use. + // Without this, the first real Player races against device startup and + // player.empty() returns true before audio has started draining — causing + // the first TTS message to be truncated after a few words. + { + use rodio::buffer::SamplesBuffer; + let channels = NonZero::new(1u16).unwrap(); + let rate = NonZero::new(SAMPLE_RATE).unwrap(); + let silence = vec![0.0f32; SAMPLE_RATE as usize / 10]; // 100ms of silence + let player = Player::connect_new(&sink_handle.mixer()); + player.append(SamplesBuffer::new(channels, rate, silence)); + // Wait for the silent buffer to drain — this ensures the output stream + // is fully initialized before the main loop creates its first Player. + while !player.empty() { + thread::sleep(Duration::from_millis(10)); + } + } + // ── 4. Main loop ────────────────────────────────────────────────────────── loop { // Check shutdown/cancel before blocking (no player yet). @@ -307,7 +326,6 @@ fn tts_worker( tts_active.store(true, Ordering::Release); for chunk in sentences.chunks(BATCH_SIZE) { - // Fix 2: tts_active cleared immediately on cancel (inside helper). if handle_cancel_or_shutdown(&cancel, &shutdown, &tts_active, &text_rx, Some(&player)) { break; } @@ -320,7 +338,6 @@ fn tts_worker( // Wait for all queued audio to finish playing. loop { - // Fix 2: tts_active cleared immediately on cancel (inside helper). if handle_cancel_or_shutdown(&cancel, &shutdown, &tts_active, &text_rx, Some(&player)) { break; } From c5d05fe69c482e4a909cf3b79912f9f7324c5a4a Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Mon, 13 Apr 2026 10:55:36 -0400 Subject: [PATCH 33/41] feat(relay): enable huddles by default with dev credentials Huddles are now enabled out of the box with dev defaults: LIVEKIT_URL=ws://localhost:7880 LIVEKIT_API_KEY=devkey LIVEKIT_API_SECRET=secret Set SPROUT_HUDDLES_DISABLED=true to disable. Override the LIVEKIT_* env vars for production deployments. --- crates/sprout-relay/src/main.rs | 38 +++++++++++++++++---------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/crates/sprout-relay/src/main.rs b/crates/sprout-relay/src/main.rs index 28f939e678..a93cab0ef0 100644 --- a/crates/sprout-relay/src/main.rs +++ b/crates/sprout-relay/src/main.rs @@ -128,24 +128,26 @@ async fn main() -> anyhow::Result<()> { .map_err(|e| anyhow::anyhow!("failed to initialize media storage: {e}"))?; info!("Media storage connected"); - let huddle_service = match ( - std::env::var("LIVEKIT_URL"), - std::env::var("LIVEKIT_API_KEY"), - std::env::var("LIVEKIT_API_SECRET"), - ) { - (Ok(url), Ok(key), Ok(secret)) => { - info!("LiveKit configured — huddles enabled"); - let svc = HuddleService::new(HuddleConfig { - livekit_url: url.clone(), - livekit_api_key: key, - livekit_api_secret: secret, - }); - Some((svc, url)) - } - _ => { - info!("LiveKit not configured — huddles disabled"); - None - } + // Huddles are enabled by default with dev credentials. + // Set SPROUT_HUDDLES_DISABLED=true to turn them off. + // Override LIVEKIT_URL / LIVEKIT_API_KEY / LIVEKIT_API_SECRET for production. + let huddle_service = if std::env::var("SPROUT_HUDDLES_DISABLED") + .map(|v| v == "true" || v == "1") + .unwrap_or(false) + { + info!("Huddles explicitly disabled via SPROUT_HUDDLES_DISABLED"); + None + } else { + let url = std::env::var("LIVEKIT_URL").unwrap_or_else(|_| "ws://localhost:7880".into()); + let key = std::env::var("LIVEKIT_API_KEY").unwrap_or_else(|_| "devkey".into()); + let secret = std::env::var("LIVEKIT_API_SECRET").unwrap_or_else(|_| "secret".into()); + info!("Huddles enabled (LiveKit URL: {url})"); + let svc = HuddleService::new(HuddleConfig { + livekit_url: url.clone(), + livekit_api_key: key, + livekit_api_secret: secret, + }); + Some((svc, url)) }; let state = Arc::new(AppState::new( From caef80ff6cd2e73a0aad8f4de57b0640b13e4a4f Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Mon, 13 Apr 2026 10:56:07 -0400 Subject: [PATCH 34/41] feat(huddles): push-to-talk as default voice input mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add push-to-talk (PTT) via global Ctrl+Space shortcut as the default voice input mode for huddles. Voice activity detection (VAD) remains as a switchable option with barge-in enabled. Rust backend: - VoiceInputMode enum (PushToTalk default, VoiceActivity) - ptt_active: Arc shared with STT pipeline - Global shortcut handler with 200ms release delay and generation counter to prevent press→release→press race condition - PTT press cancels TTS immediately (only when TTS is active) - STT gating: is_speech ANDed with ptt_active flag — natural flush on release via silence accumulation - PTT mode accumulates entire key-hold as one utterance (no mid- sentence splits on pauses); flush only on release edge - Barge-in re-enabled for VAD mode with 320ms debounce - Mode switch mid-huddle restarts STT pipeline - Disable WebView background throttling (tauri.conf.json) so AudioWorklet keeps processing when window loses focus Frontend: - worklet.js: PTT gating via transmitting flag + port.onmessage - audioWorklet.ts: Tauri ptt-state event → worklet forwarding - HuddleContext: pttActive/voiceInputMode/setVoiceInputMode state - HuddleBar: PTT/VAD indicator, mode toggle, green ring on transmit Also fixes two review items from crossfire: - Preserve session_generation across error-path state resets - set_tts_enabled takes pipeline out of lock before shutdown --- desktop/public/worklet.js | 22 ++++ desktop/scripts/check-file-sizes.mjs | 6 +- desktop/src-tauri/Cargo.lock | 67 +++++++++++ desktop/src-tauri/Cargo.toml | 1 + desktop/src-tauri/capabilities/default.json | 5 +- desktop/src-tauri/src/huddle/mod.rs | 109 ++++++++++++++++-- desktop/src-tauri/src/huddle/stt.rs | 88 ++++++++++++-- desktop/src-tauri/src/lib.rs | 102 +++++++++++++++- desktop/src-tauri/tauri.conf.json | 3 +- desktop/src/features/huddle/HuddleContext.tsx | 78 ++++++++++++- .../features/huddle/components/HuddleBar.tsx | 97 +++++++++++++--- .../src/features/huddle/lib/audioWorklet.ts | 47 +++++++- 12 files changed, 574 insertions(+), 51 deletions(-) diff --git a/desktop/public/worklet.js b/desktop/public/worklet.js index 816ea326f5..60c22cfaa2 100644 --- a/desktop/public/worklet.js +++ b/desktop/public/worklet.js @@ -1,6 +1,11 @@ // AudioWorklet processor — runs in the AudioWorklet thread. // Accumulates PCM Float32 samples and sends 100ms batches to the main thread. // +// Supports push-to-talk (PTT) gating: when `this.transmitting` is false, +// incoming audio frames are discarded and the buffer is reset. The main thread +// sends `{ type: 'ptt', active: boolean }` messages to toggle transmission. +// Default: transmitting=true (open mic for VAD mode compatibility). +// // Note: when the worklet is disconnected, any partial buffer (< 4800 samples) // is silently dropped. The last ~100ms of speech may be lost on huddle leave. // This is acceptable for voice — losing a partial syllable at disconnect is @@ -10,12 +15,29 @@ class SttTapProcessor extends AudioWorkletProcessor { super(); this.buffer = new Float32Array(4800); // ~100ms at 48kHz this.offset = 0; + this.transmitting = true; // default: open (VAD mode). PTT mode sets false on init. + + // Listen for PTT state changes from main thread. + // Direction: main→worklet (receives). The worklet→main direction uses + // this.port.postMessage for PCM data — these don't conflict. + this.port.onmessage = (e) => { + if (e.data && e.data.type === "ptt") { + this.transmitting = e.data.active; + } + }; } process(inputs) { const input = inputs[0]?.[0]; // mono channel if (!input) return true; + // PTT gating: discard frames when not transmitting. + // Reset buffer offset so we don't send stale audio when PTT activates. + if (!this.transmitting) { + this.offset = 0; + return true; + } + // Accumulate samples const remaining = this.buffer.length - this.offset; const toCopy = Math.min(input.length, remaining); diff --git a/desktop/scripts/check-file-sizes.mjs b/desktop/scripts/check-file-sizes.mjs index f9d467fd2d..54ce8c6c91 100644 --- a/desktop/scripts/check-file-sizes.mjs +++ b/desktop/scripts/check-file-sizes.mjs @@ -42,11 +42,12 @@ const overrides = new Map([ ["src/features/tokens/ui/TokenSettingsCard.tsx", 800], ["src/shared/api/relayClientSession.ts", 810], // durable websocket session manager with reconnect/replay/recovery state + sendTypingIndicator + fetchChannelHistoryBefore + subscribeToChannelLive (huddle TTS) ["src/shared/api/tauri.ts", 1100], // remote agent provider API bindings + canvas API functions - ["src-tauri/src/lib.rs", 590], // sprout-media:// proxy + Range headers + Sprout nest init (ensure_nest) in setup() + huddle command registration + ["src-tauri/src/lib.rs", 700], // sprout-media:// proxy + Range headers + Sprout nest init (ensure_nest) in setup() + huddle command registration + PTT global shortcut handler with generation counter and release delay ["src-tauri/src/commands/media.rs", 720], // ffmpeg video transcode + poster frame extraction + run_ffmpeg_with_timeout (find_ffmpeg, is_video_file, transcode_to_mp4, extract_poster_frame, transcode_and_extract_poster) + spawn_blocking wrappers + tests ["src-tauri/src/commands/agents.rs", 860], // remote agent lifecycle routing (local + provider branches) + scope enforcement + mcp_toolsets field; rustfmt adds line breaks around long tuple/closure blocks ["src-tauri/src/managed_agents/runtime.rs", 650], // KNOWN_AGENT_BINARIES const + process_belongs_to_us FFI (macOS proc_name + Linux /proc/comm) + terminate_process + start/stop/sync lifecycle ["src-tauri/src/managed_agents/backend.rs", 530], // provider IPC, validation, discovery, binary resolution + tests + ["src/features/huddle/HuddleContext.tsx", 530], // huddle lifecycle context + PTT state (pttActive, voiceInputMode, setVoiceInputMode) + TTS subscription + mic level analyser + agent pubkey refresh ["src/features/agents/hooks.ts", 520], // agent query/mutation surface now includes built-in persona library activation ["src/features/agents/ui/AgentsView.tsx", 880], // remote agent lifecycle controls + persona/team management + persona import-update dialog wiring + built-in catalog/library state orchestration ["src/features/agents/ui/TeamDialog.tsx", 530], // team create/edit dialog with persona multi-select, import button, window drag detection, removal confirmation @@ -56,8 +57,9 @@ const overrides = new Map([ ["src/features/channels/ui/AddChannelBotDialog.tsx", 640], // provider mode: Run on selector, trust warning, probe effect, single-agent enforcement, provider warnings display ["src/shared/api/types.ts", 535], // persona provider/model fields + forum types + workflow type re-exports + ephemeral channel TTL fields + mcpToolsets ["src-tauri/src/events.rs", 530], // event builders + build_huddle_guidelines (kind:48106) + post_event_raw transport helper - ["src-tauri/src/huddle/mod.rs", 1300], // huddle state machine + 14 Tauri commands + STT/TTS pipeline lifecycle + relay membership fetch + session generation guard + creator enforcement + input validation; split planned post-MVP + ["src-tauri/src/huddle/mod.rs", 1400], // huddle state machine + 16 Tauri commands + STT/TTS pipeline lifecycle + VoiceInputMode (PTT/VAD) + relay membership fetch + session generation guard + creator enforcement + input validation; split planned post-MVP ["src-tauri/src/huddle/models.rs", 850], // model download manager for Moonshine STT + Supertonic TTS with streaming downloads + SHA-256 verification + Rust-native tar extraction + version manifest + atomic swap + hot-start signaling + ["src-tauri/src/huddle/stt.rs", 580], // STT pipeline + PTT edge-detection flush + PTT gating (is_speech AND ptt_active) + barge-in for VAD mode + rubato resampler + earshot VAD + sherpa-onnx transcription ["src-tauri/src/huddle/preprocessing.rs", 620], // TTS text preprocessing pipeline + unified split_sentences (consolidated from tts.rs + supertonic.rs) + int_to_words 0-999999 + 18 unit tests ["src-tauri/src/huddle/supertonic.rs", 780], // Supertonic 4-ONNX-session TTS engine wrapper + Unicode text processor + LazyLock regex patterns + text chunking ]); diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index ece145bb43..57cc2e363e 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -1799,6 +1799,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "gethostname" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" +dependencies = [ + "rustix", + "windows-link 0.2.1", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -1936,6 +1946,24 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "global-hotkey" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9247516746aa8e53411a0db9b62b0e24efbcf6a76e0ba73e5a91b512ddabed7" +dependencies = [ + "crossbeam-channel", + "keyboard-types", + "objc2", + "objc2-app-kit", + "once_cell", + "serde", + "thiserror 2.0.18", + "windows-sys 0.59.0", + "x11rb", + "xkeysym", +] + [[package]] name = "gobject-sys" version = "0.18.0" @@ -5177,6 +5205,7 @@ dependencies = [ "tauri", "tauri-build", "tauri-plugin-dialog", + "tauri-plugin-global-shortcut", "tauri-plugin-notification", "tauri-plugin-opener", "tauri-plugin-process", @@ -5751,6 +5780,21 @@ dependencies = [ "url", ] +[[package]] +name = "tauri-plugin-global-shortcut" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "424af23c7e88d05e4a1a6fc2c7be077912f8c76bd7900fd50aa2b7cbf5a2c405" +dependencies = [ + "global-hotkey", + "log", + "serde", + "serde_json", + "tauri", + "tauri-plugin", + "thiserror 2.0.18", +] + [[package]] name = "tauri-plugin-notification" version = "2.3.3" @@ -7600,6 +7644,23 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "x11rb" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" +dependencies = [ + "gethostname", + "rustix", + "x11rb-protocol", +] + +[[package]] +name = "x11rb-protocol" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" + [[package]] name = "xattr" version = "1.6.1" @@ -7610,6 +7671,12 @@ dependencies = [ "rustix", ] +[[package]] +name = "xkeysym" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" + [[package]] name = "xz2" version = "0.1.7" diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index ef76260a54..4315ee9060 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -47,6 +47,7 @@ sha2 = "0.11" tar = "0.4" bzip2 = "0.5" chrono = { version = "0.4", features = ["serde"] } +tauri-plugin-global-shortcut = "2" tauri-plugin-notification = "2.3.3" uuid = { version = "1", features = ["v4"] } png = "0.18" diff --git a/desktop/src-tauri/capabilities/default.json b/desktop/src-tauri/capabilities/default.json index 4e3d55d987..fc4eae76c7 100644 --- a/desktop/src-tauri/capabilities/default.json +++ b/desktop/src-tauri/capabilities/default.json @@ -21,6 +21,9 @@ "updater:default", "updater:allow-check", "updater:allow-download-and-install", - "process:allow-restart" + "process:allow-restart", + "global-shortcut:allow-register", + "global-shortcut:allow-unregister", + "global-shortcut:allow-is-registered" ] } diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index 330513757b..ca6502f94e 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -54,6 +54,22 @@ use crate::{ // ── State types ─────────────────────────────────────────────────────────────── +/// Voice input mode: push-to-talk (PTT) or voice-activity detection (VAD). +/// +/// PTT (default): mic is gated by a global shortcut (Ctrl+Space). Pressing the +/// key sets `ptt_active` and immediately cancels any playing TTS. Releasing +/// the key (after a 200 ms delay) stops mic capture and flushes the utterance. +/// +/// VAD: the earshot VAD runs continuously and speech is accumulated whenever +/// the probability exceeds the threshold. Barge-in is enabled in this mode. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum VoiceInputMode { + #[default] + PushToTalk, + VoiceActivity, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "snake_case")] pub enum HuddlePhase { @@ -125,6 +141,12 @@ pub struct HuddleState { /// generation has changed, the task silently drops the transcript. #[serde(skip)] pub session_generation: Arc, + /// Voice input mode: push-to-talk or voice-activity detection. + pub voice_input_mode: VoiceInputMode, + /// True while the PTT key is held (+ 200 ms release delay). + /// Shared with the STT pipeline for mic gating. + #[serde(skip)] + pub ptt_active: Arc, } fn serialize_agent_pubkeys(v: &Arc>>, s: S) -> Result @@ -172,6 +194,8 @@ impl Clone for HuddleState { tts_cancel: Arc::clone(&self.tts_cancel), last_agent_refresh: self.last_agent_refresh, session_generation: Arc::clone(&self.session_generation), + voice_input_mode: self.voice_input_mode.clone(), + ptt_active: Arc::clone(&self.ptt_active), } } } @@ -195,6 +219,8 @@ impl Default for HuddleState { tts_cancel: Arc::new(AtomicBool::new(false)), last_agent_refresh: None, session_generation: Arc::new(AtomicU64::new(0)), + voice_input_mode: VoiceInputMode::default(), + ptt_active: Arc::new(AtomicBool::new(false)), } } } @@ -335,8 +361,8 @@ async fn maybe_start_stt_pipeline( // Grab shared flags, agent pubkeys, and session generation from HuddleState. // If replacing an existing pipeline, bump generation first so the old // transcription task's next POST sees a stale generation and exits. - let (tts_active, tts_cancel, agent_pubkeys_arc, session_gen) = { - let mut hs = state.huddle()?; + let (tts_active, tts_cancel, agent_pubkeys_arc, session_gen, ptt_active_for_stt) = { + let hs = state.huddle()?; // Invalidate any existing transcription task before replacing the pipeline. if hs.stt_pipeline.is_some() { hs.session_generation.fetch_add(1, Ordering::Release); @@ -344,15 +370,22 @@ async fn maybe_start_stt_pipeline( if let Some(ref old) = hs.stt_pipeline { old.shutdown(); } + let ptt = if hs.voice_input_mode == VoiceInputMode::PushToTalk { + Some(Arc::clone(&hs.ptt_active)) + } else { + None + }; ( Arc::clone(&hs.tts_active), Some(Arc::clone(&hs.tts_cancel)), Arc::clone(&hs.agent_pubkeys), Arc::clone(&hs.session_generation), + ptt, ) }; - let (pipeline, text_rx) = stt::SttPipeline::new(model_dir, tts_active, tts_cancel)?; + let (pipeline, text_rx) = + stt::SttPipeline::new(model_dir, tts_active, tts_cancel, ptt_active_for_stt)?; let pipeline = Arc::new(pipeline); { @@ -489,6 +522,50 @@ fn spawn_transcription_task( // ── Tauri commands ──────────────────────────────────────────────────────────── +/// Set the voice input mode (push-to-talk or voice-activity detection). +/// +/// When switching mid-huddle, restarts the STT pipeline so it picks up the +/// new mode (PTT gating vs continuous VAD with barge-in). The pipeline +/// captures the mode at construction time, so a restart is required. +#[tauri::command] +pub async fn set_voice_input_mode( + mode: VoiceInputMode, + state: State<'_, AppState>, +) -> Result<(), String> { + let needs_restart = { + let mut hs = state.huddle()?; + let old_mode = hs.voice_input_mode.clone(); + hs.voice_input_mode = mode.clone(); + // Restart STT if mode changed and a huddle is active with a pipeline running. + old_mode != mode + && matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active) + && hs.stt_pipeline.is_some() + }; + + if needs_restart { + let eph_id = { + let hs = state.huddle()?; + hs.ephemeral_channel_id.clone() + }; + if let Some(eph_id) = eph_id { + // Best-effort restart — if models aren't ready, the pipeline + // stays down until the next hotstart cycle picks it up. + if let Err(e) = maybe_start_stt_pipeline(&state, &eph_id).await { + eprintln!("sprout-desktop: STT pipeline restart on mode switch failed: {e}"); + } + } + } + + Ok(()) +} + +/// Return the current voice input mode. +#[tauri::command] +pub fn get_voice_input_mode(state: State<'_, AppState>) -> Result { + let hs = state.huddle()?; + Ok(hs.voice_input_mode.clone()) +} + /// Start a new huddle in the given parent channel. /// /// Steps: @@ -656,8 +733,12 @@ pub async fn start_huddle( } } // Reset state to Idle so the user can retry. + // Preserve session_generation so in-flight transcription tasks + // from a prior session still see a stale generation and exit. if let Ok(mut hs) = state.huddle_state.lock() { + let gen = Arc::clone(&hs.session_generation); *hs = HuddleState::default(); + hs.session_generation = gen; } Err(e) } @@ -693,11 +774,14 @@ pub async fn join_huddle( } // 1. Fetch LiveKit token. On failure, reset state to Idle so user can retry. + // Preserve session_generation (same rationale as start_huddle rollback). let lk = match fetch_livekit_token(&ephemeral_channel_id, &state).await { Ok(lk) => lk, Err(e) => { if let Ok(mut hs) = state.huddle_state.lock() { + let gen = Arc::clone(&hs.session_generation); *hs = HuddleState::default(); + hs.session_generation = gen; } return Err(e); } @@ -1102,19 +1186,26 @@ pub fn get_model_status(_state: State<'_, AppState>) -> Result) -> Result<(), String> { - { + let old_pipeline = { let mut hs = state.huddle()?; hs.tts_enabled = enabled; if !enabled { - // Shut down the TTS pipeline immediately. - if let Some(ref pipeline) = hs.tts_pipeline { - pipeline.shutdown(); - } - hs.tts_pipeline = None; + hs.tts_pipeline.take() // Take out of lock. + } else { + None } + }; + // Shut down outside the lock — thread join happens here. + if let Some(ref pipeline) = old_pipeline { + pipeline.shutdown(); } + drop(old_pipeline); if enabled { // Re-start TTS pipeline if models are available and huddle is active. diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index 87a728eded..ce0cfecf02 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -77,6 +77,10 @@ impl SttPipeline { /// worker detects speech onset while TTS is active, it sets this flag to /// stop playback immediately (barge-in). Pass `None` if TTS is unavailable. /// + /// `ptt_active` (optional) is the push-to-talk flag. When `Some`, the STT + /// pipeline only accumulates speech while the flag is true (key held). + /// When `None`, the pipeline runs in continuous VAD mode. + /// /// Returns `Err` only if the thread cannot be spawned (OS error). /// If model files are missing, the worker logs and exits cleanly — /// the pipeline handle is still returned but will never produce text. @@ -89,6 +93,7 @@ impl SttPipeline { model_dir: PathBuf, tts_active: Arc, tts_cancel: Option>, + ptt_active: Option>, ) -> Result<(Self, tokio_mpsc::Receiver), String> { let (audio_tx, audio_rx) = mpsc::sync_channel::>(AUDIO_QUEUE_DEPTH); let (text_tx, text_rx) = tokio_mpsc::channel::(64); @@ -96,6 +101,7 @@ impl SttPipeline { let shutdown_worker = Arc::clone(&shutdown); let tts_cancel_worker = tts_cancel.as_ref().map(Arc::clone); + let ptt_active_worker = ptt_active.as_ref().map(Arc::clone); let handle = thread::Builder::new() .name("stt-worker".into()) .spawn(move || { @@ -106,6 +112,7 @@ impl SttPipeline { shutdown_worker, tts_active, tts_cancel_worker, + ptt_active_worker, ) }) .map_err(|e| format!("failed to spawn stt-worker thread: {e}"))?; @@ -197,6 +204,7 @@ fn stt_worker( shutdown: Arc, tts_active: Arc, tts_cancel: Option>, + ptt_active: Option>, ) { // ── 1. Initialise rubato resampler (48 kHz → 16 kHz, mono) ─────────────── use rubato::{Fft, FixedSync, Resampler}; @@ -269,6 +277,9 @@ fn stt_worker( // ── 5. Main loop ────────────────────────────────────────────────────────── let mut tts_was_active = false; + let mut ptt_was_active = ptt_active + .as_ref() + .map_or(false, |p| p.load(Ordering::Acquire)); loop { // Check shutdown flag before blocking. if shutdown.load(Ordering::Acquire) { @@ -283,6 +294,21 @@ fn stt_worker( } tts_was_active = tts_now; + // Track PTT transitions — flush accumulated speech when key is released. + // The worklet stops sending frames when PTT is inactive, so the normal + // silence-accumulation flush path never runs. We must flush here on the + // active→inactive edge to avoid buffering speech across PTT presses. + if let Some(ref ptt) = ptt_active { + let ptt_now = ptt.load(Ordering::Acquire); + if ptt_was_active && !ptt_now && in_speech && !speech_buf.is_empty() { + flush_to_stt(&speech_buf, &recognizer, &text_tx); + speech_buf.clear(); + silence_frames = 0; + in_speech = false; + } + ptt_was_active = ptt_now; + } + // Use recv_timeout so we can periodically check the shutdown flag. let bytes = match audio_rx.recv_timeout(RECV_TIMEOUT) { Ok(b) => b, @@ -318,6 +344,7 @@ fn stt_worker( &tts_active, tts_cancel.as_deref(), &mut tts_stopped_at, + ptt_active.as_ref(), ); } } @@ -356,9 +383,15 @@ fn resample_chunk(resampler: &mut rubato::Fft, chunk_48k: &[f32]) -> Vec, tts_cancel: Option<&AtomicBool>, tts_stopped_at: &mut Option, + ptt_active: Option<&Arc>, ) { leftover.extend_from_slice(samples); @@ -381,15 +415,49 @@ fn process_16k_samples( let prob = vad.predict_f32(&frame); let is_speech = prob > VAD_THRESHOLD; + // PTT gating: when PTT key is not held, treat as silence. + // This causes natural flush when the key is released — silence_frames + // accumulates and the existing flush logic kicks in after + // SILENCE_FLUSH_FRAMES. The 200 ms release delay + ~300 ms silence + // flush gives a natural utterance tail. + let is_speech = if let Some(ptt) = ptt_active { + is_speech && ptt.load(Ordering::Acquire) + } else { + is_speech + }; + let tts_playing = tts_active.load(Ordering::Acquire); // While TTS is playing: skip accumulation (echo prevention). - // Barge-in is disabled — push-to-talk will replace VAD-based interruption. - // Without acoustic echo cancellation, any ambient noise triggers false - // barge-in and kills TTS playback mid-sentence. if tts_playing { + if ptt_active.is_some() { + // PTT mode — PTT press handles TTS cancellation directly + // (via the global shortcut handler). Just skip accumulation. + *in_speech = false; + *barge_in_frames = 0; + speech_buf.clear(); + *silence_frames = 0; + continue; + } + + // VAD mode — barge-in detection. + // Without acoustic echo cancellation, this requires a longer + // debounce (BARGE_IN_DEBOUNCE_FRAMES ≈ 320 ms) to filter + // speaker-to-mic feedback. + if is_speech { + *barge_in_frames += 1; + if *barge_in_frames >= BARGE_IN_DEBOUNCE_FRAMES { + // Real speech detected during TTS — trigger barge-in. + if let Some(cancel) = tts_cancel { + cancel.store(true, Ordering::Release); + } + *barge_in_frames = 0; + } + } else { + *barge_in_frames = 0; + } + // Don't accumulate speech during TTS (echo prevention). *in_speech = false; - *barge_in_frames = 0; speech_buf.clear(); *silence_frames = 0; continue; @@ -432,7 +500,11 @@ fn process_16k_samples( speech_buf.extend_from_slice(&frame); *silence_frames += 1; - if *silence_frames >= SILENCE_FLUSH_FRAMES { + // In PTT mode, don't flush on silence — accumulate the entire + // key-hold as one utterance. The PTT release edge in the main + // loop handles the flush. In VAD mode, flush after the silence + // threshold so each natural pause becomes a separate message. + if ptt_active.is_none() && *silence_frames >= SILENCE_FLUSH_FRAMES { // End of utterance — transcribe. flush_to_stt(speech_buf, recognizer, text_tx); speech_buf.clear(); diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index e0ea036e06..39cc66dc23 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -12,9 +12,9 @@ use app_state::{build_app_state, resolve_persisted_identity, AppState}; use commands::*; use huddle::{ add_agent_to_huddle, check_pipeline_hotstart, confirm_huddle_active, download_voice_models, - end_huddle, get_huddle_agent_pubkeys, get_huddle_state, get_model_status, join_huddle, - leave_huddle, push_audio_pcm, set_tts_enabled, speak_agent_message, start_huddle, - start_stt_pipeline, + end_huddle, get_huddle_agent_pubkeys, get_huddle_state, get_model_status, get_voice_input_mode, + join_huddle, leave_huddle, push_audio_pcm, set_tts_enabled, set_voice_input_mode, + speak_agent_message, start_huddle, start_stt_pipeline, }; use managed_agents::{ ensure_nest, find_managed_agent_mut, kill_stale_tracked_processes, load_managed_agents, @@ -25,7 +25,7 @@ use std::sync::{ atomic::{AtomicBool, Ordering}, Arc, }; -use tauri::{http, Manager, RunEvent}; +use tauri::{http, Emitter, Manager, RunEvent}; use tauri_plugin_window_state::StateFlags; fn restore_managed_agents_on_launch( @@ -352,7 +352,86 @@ pub fn run() { ) .plugin(tauri_plugin_websocket::init()) .plugin(tauri_plugin_dialog::init()) - .plugin(tauri_plugin_process::init()); + .plugin(tauri_plugin_process::init()) + .plugin({ + use tauri_plugin_global_shortcut::ShortcutState; + + // Generation counter for the release delay task. Incremented on + // every press — a delayed release only fires if the generation + // hasn't changed (i.e. no new press happened during the delay). + // This prevents press→release→press within 200 ms from having + // the first release clobber the second press. + let ptt_press_gen = Arc::new(std::sync::atomic::AtomicU64::new(0)); + + tauri_plugin_global_shortcut::Builder::new() + .with_handler(move |app, _shortcut, event| { + let state = match app.try_state::() { + Some(s) => s, + None => return, + }; + + // Only act if a huddle is active and mode is PTT. + let (is_ptt_mode, is_active) = match state.huddle_state.lock() { + Ok(hs) => ( + hs.voice_input_mode == huddle::VoiceInputMode::PushToTalk, + matches!( + hs.phase, + huddle::HuddlePhase::Connected | huddle::HuddlePhase::Active + ), + ), + Err(_) => return, + }; + + if !is_ptt_mode || !is_active { + return; + } + + match event.state { + ShortcutState::Pressed => { + // Bump generation — invalidates any pending release delay. + ptt_press_gen.fetch_add(1, std::sync::atomic::Ordering::Release); + + if let Ok(hs) = state.huddle_state.lock() { + hs.ptt_active + .store(true, std::sync::atomic::Ordering::Release); + // Only cancel TTS if it's actually playing — avoids + // a stale cancel flag that drops the next queued message. + if hs.tts_active.load(std::sync::atomic::Ordering::Acquire) { + hs.tts_cancel + .store(true, std::sync::atomic::Ordering::Release); + } + } + let _ = app.emit("ptt-state", true); + } + ShortcutState::Released => { + // Capture generation at release time. + let gen_at_release = + ptt_press_gen.load(std::sync::atomic::Ordering::Acquire); + let gen_arc = Arc::clone(&ptt_press_gen); + let app_handle = app.clone(); + // 200 ms release delay — captures the tail of the utterance. + // Only applies if no new press happened during the delay. + tauri::async_runtime::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + // Check generation — if it changed, a new press arrived. + if gen_arc.load(std::sync::atomic::Ordering::Acquire) + != gen_at_release + { + return; // Superseded by a new press. + } + if let Some(state) = app_handle.try_state::() { + if let Ok(hs) = state.huddle_state.lock() { + hs.ptt_active + .store(false, std::sync::atomic::Ordering::Release); + } + } + let _ = app_handle.emit("ptt-state", false); + }); + } + } + }) + .build() + }); // Only register the updater in release builds that were compiled with a // real updater configuration. Local unsigned builds omit that config and @@ -410,6 +489,17 @@ pub fn run() { mgr.start_supertonic_download(state.http_client.clone()); } + // Register PTT global shortcut (Ctrl+Space). + // Non-fatal: huddle works without the shortcut (user can switch to VAD mode). + #[cfg(desktop)] + { + use tauri_plugin_global_shortcut::{Code, GlobalShortcutExt, Modifiers, Shortcut}; + let shortcut = Shortcut::new(Some(Modifiers::CONTROL), Code::Space); + if let Err(e) = app.handle().global_shortcut().register(shortcut) { + eprintln!("sprout-desktop: failed to register PTT shortcut: {e}"); + } + } + // Keep launch-time agent restoration off the synchronous setup path // so the frontend can mount and reveal the window promptly. tauri::async_runtime::spawn_blocking(move || { @@ -528,6 +618,8 @@ pub fn run() { check_pipeline_hotstart, confirm_huddle_active, get_huddle_agent_pubkeys, + set_voice_input_mode, + get_voice_input_mode, ]) .build(tauri::generate_context!()) .expect("error while building tauri application"); diff --git a/desktop/src-tauri/tauri.conf.json b/desktop/src-tauri/tauri.conf.json index 47940b248d..1d88b3f5c4 100644 --- a/desktop/src-tauri/tauri.conf.json +++ b/desktop/src-tauri/tauri.conf.json @@ -23,7 +23,8 @@ "titleBarStyle": "Overlay", "hiddenTitle": true, "dragDropEnabled": false, - "trafficLightPosition": { "x": 12, "y": 22 } + "trafficLightPosition": { "x": 12, "y": 22 }, + "backgroundThrottling": "disabled" } ], "security": { diff --git a/desktop/src/features/huddle/HuddleContext.tsx b/desktop/src/features/huddle/HuddleContext.tsx index c531acc310..4e4ee4b5c2 100644 --- a/desktop/src/features/huddle/HuddleContext.tsx +++ b/desktop/src/features/huddle/HuddleContext.tsx @@ -1,9 +1,10 @@ import { invoke } from "@tauri-apps/api/core"; +import { listen } from "@tauri-apps/api/event"; import * as React from "react"; import { relayClient } from "@/shared/api/relayClient"; import { connectToHuddle, type HuddleConnection } from "./lib/livekit"; -import { setupAudioWorklet } from "./lib/audioWorklet"; +import { setupAudioWorklet, type AudioWorkletHandle } from "./lib/audioWorklet"; /** * Huddle lifecycle (React context): @@ -11,7 +12,7 @@ import { setupAudioWorklet } from "./lib/audioWorklet"; * startHuddle(channelId, agents) * → invoke("start_huddle") [Rust: ephemeral channel + LiveKit token] * → connectToHuddle(url, token) [LiveKit: WebRTC room + mic] - * → setupAudioWorklet(track) [AudioWorklet: mic PCM → Rust STT] + * → setupAudioWorklet(track, init) [AudioWorklet: mic PCM → Rust STT, PTT gating] * → invoke("confirm_huddle_active") [Rust: Connected → Active] * → setEphemeralChannelId(...) [triggers TTS subscription + hotstart polling] * @@ -32,6 +33,8 @@ type HuddleJoinInfo = { livekit_room: string; }; +type VoiceInputMode = "push_to_talk" | "voice_activity"; + interface HuddleContextValue { /** Current local audio track (for mute toggle in HuddleBar) */ localAudioTrack: MediaStreamTrack | null; @@ -41,6 +44,12 @@ interface HuddleContextValue { micConnected: boolean; /** Current mic input level 0–1 (updated via requestAnimationFrame) */ micLevel: number; + /** Whether the PTT key is currently held (for UI feedback) */ + pttActive: boolean; + /** Current voice input mode — push_to_talk or voice_activity */ + voiceInputMode: VoiceInputMode; + /** Toggle voice input mode (persisted to Rust backend) */ + setVoiceInputMode: (mode: VoiceInputMode) => Promise; /** Start a new huddle — calls Rust start_huddle, then connects LiveKit + AudioWorklet */ startHuddle: ( parentChannelId: string, @@ -57,7 +66,7 @@ const HuddleContext = React.createContext(null); export function HuddleProvider({ children }: { children: React.ReactNode }) { const connectionRef = React.useRef(null); - const workletRef = React.useRef<{ stop: () => void } | null>(null); + const workletRef = React.useRef(null); const tokenRef = React.useRef(0); const busyRef = React.useRef(false); /** True once Rust `start_huddle` has been invoked (even if JS-side refs aren't populated yet). */ @@ -67,6 +76,11 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { const [isStarting, setIsStarting] = React.useState(false); const [micConnected, setMicConnected] = React.useState(false); const [micLevel, setMicLevel] = React.useState(0); + /** Whether the PTT key is currently held */ + const [pttActive, setPttActive] = React.useState(false); + /** Current voice input mode */ + const [voiceInputMode, setVoiceInputModeState] = + React.useState("push_to_talk"); /** Ephemeral channel ID — set after start_huddle, used for TTS subscription */ const [ephemeralChannelId, setEphemeralChannelId] = React.useState< string | null @@ -74,6 +88,49 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { /** Self pubkey — fetched once, used to filter out own messages from TTS */ const selfPubkeyRef = React.useRef(null); + // Bootstrap voice input mode from Rust backend on mount. + // Ensures frontend stays in sync after remount/recovery. + React.useEffect(() => { + invoke("get_voice_input_mode") + .then((mode) => setVoiceInputModeState(mode)) + .catch(() => { + /* best-effort — default is push_to_talk */ + }); + }, []); + + // Listen for PTT state from Rust global shortcut (Ctrl+Space). + // Updates pttActive for UI feedback (green indicator in HuddleBar). + // The actual audio gating happens in audioWorklet.ts → worklet.js. + React.useEffect(() => { + let cancelled = false; + let unlisten: (() => void) | null = null; + + listen("ptt-state", (event) => { + if (!cancelled) setPttActive(event.payload); + }).then((fn) => { + if (cancelled) fn(); + else unlisten = fn; + }); + + return () => { + cancelled = true; + unlisten?.(); + }; + }, []); + + // Toggle voice input mode — persists to Rust backend and updates worklet gating. + const setVoiceInputMode = React.useCallback( + async (mode: VoiceInputMode) => { + await invoke("set_voice_input_mode", { mode }); + setVoiceInputModeState(mode); + // Update the worklet's transmit state to match the new mode: + // VAD = always transmitting, PTT = transmit only when key is held. + const transmitting = mode === "voice_activity" || pttActive; + workletRef.current?.setTransmitting(transmitting); + }, + [pttActive], + ); + /** Stop AudioWorklet and disconnect LiveKit. Best-effort on both steps. */ const disconnectMedia = React.useCallback(async () => { // Invalidate any in-flight startHuddle @@ -147,7 +204,7 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { const cleanupFailedStart = React.useCallback( async ( conn: HuddleConnection | null, - worklet: { stop: () => void } | null, + worklet: AudioWorkletHandle | null, ) => { try { worklet?.stop(); @@ -238,7 +295,13 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { setMicConnected(true); // Step 3: Set up AudioWorklet to pipe mic audio to Rust STT - const worklet = await setupAudioWorklet(connection.localAudioTrack); + // In PTT mode, start with transmitting=false (user must hold key). + // In VAD mode, start with transmitting=true (always open mic). + const initialTransmitting = voiceInputMode !== "push_to_talk"; + const worklet = await setupAudioWorklet( + connection.localAudioTrack, + initialTransmitting, + ); // Bail if superseded after async worklet setup if (tokenRef.current !== myToken) { @@ -266,7 +329,7 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { busyRef.current = false; } }, - [cleanupFailedStart], + [cleanupFailedStart, voiceInputMode], ); // TTS subscription — pipe AGENT messages from ephemeral channel to speak_agent_message. @@ -428,6 +491,9 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { isStarting, micConnected, micLevel, + pttActive, + voiceInputMode, + setVoiceInputMode, startHuddle, leaveHuddle, endHuddle, diff --git a/desktop/src/features/huddle/components/HuddleBar.tsx b/desktop/src/features/huddle/components/HuddleBar.tsx index 8f75eba1fa..ad8a725881 100644 --- a/desktop/src/features/huddle/components/HuddleBar.tsx +++ b/desktop/src/features/huddle/components/HuddleBar.tsx @@ -34,6 +34,7 @@ type HuddleState = { agent_pubkeys: string[]; tts_enabled: boolean; is_creator: boolean; + voice_input_mode: "push_to_talk" | "voice_activity"; }; type HuddleBarProps = { @@ -41,8 +42,17 @@ type HuddleBarProps = { }; export function HuddleBar({ className }: HuddleBarProps) { - const { localAudioTrack, leaveHuddle, endHuddle, micConnected, micLevel } = - useHuddle(); + const { + localAudioTrack, + leaveHuddle, + endHuddle, + micConnected, + micLevel, + pttActive, + voiceInputMode, + setVoiceInputMode, + } = useHuddle(); + const isPttMode = voiceInputMode === "push_to_talk"; const [state, setState] = React.useState(null); const [isMuted, setIsMuted] = React.useState(false); // Derive TTS enabled from backend state (single source of truth). @@ -217,24 +227,67 @@ export function HuddleBar({ className }: HuddleBarProps) { )} - {/* Voice activity indicator */} + {/* Voice input mode indicator */}
    {micConnected ? ( -
    0.05 - ? `rgba(34, 197, 94, ${0.4 + micLevel * 0.6})` - : "rgba(100, 116, 139, 0.4)", - }} - title={`Mic level: ${Math.round(micLevel * 100)}%`} - /> + isPttMode ? ( + <> +
    + PTT + Ctrl+Space + + ) : ( + <> +
    0.05 + ? `rgba(34, 197, 94, ${0.4 + micLevel * 0.6})` + : "rgba(100, 116, 139, 0.4)", + }} + title={`Mic level: ${Math.round(micLevel * 100)}%`} + /> + VAD + + ) ) : ( no mic )}
    + {/* Voice input mode toggle */} + + {/* Add agent button */} - + startDisabled={!canAddAgents || isStartingHuddle} + />
    + {/* Reconnecting indicator */} + {isReconnecting && ( +
    + Reconnecting… +
    + )} + {/* Model download progress */} {modelStatus && (modelStatus.moonshine !== "ready" || modelStatus.supertonic !== "ready") && ( -
    + {modelStatus.moonshine !== "ready" && modelStatus.supertonic !== "ready" @@ -219,12 +229,15 @@ export function HuddleBar({ className }: HuddleBarProps) { ? `STT model: ${modelStatus.moonshine}` : `TTS model: ${modelStatus.supertonic}`} -
    + )} {/* Participant avatars */} {state.participants.length > 0 && ( - + )} {/* Voice input mode indicator */} @@ -235,7 +248,9 @@ export function HuddleBar({ className }: HuddleBarProps) {
    - {/* Leave / End button */} - {state.is_creator ? ( - - ) : ( - - )} + {/* Leave / End buttons — available to all participants */} + + + + + {/* Screen reader announcements for huddle state changes */} + + {isReconnecting + ? "Huddle reconnecting" + : micConnected + ? "In huddle, microphone connected" + : "In huddle, no microphone"} + {modelStatus && + modelStatus.moonshine !== "ready" && + `, STT model ${modelStatus.moonshine}`} + {modelStatus && + modelStatus.supertonic !== "ready" && + `, TTS model ${modelStatus.supertonic}`} +
    ); } diff --git a/desktop/src/features/huddle/components/HuddleIndicator.tsx b/desktop/src/features/huddle/components/HuddleIndicator.tsx new file mode 100644 index 0000000000..424163f6bc --- /dev/null +++ b/desktop/src/features/huddle/components/HuddleIndicator.tsx @@ -0,0 +1,237 @@ +import { Headphones } from "lucide-react"; +import * as React from "react"; +import { useQueryClient } from "@tanstack/react-query"; + +import { relayClient } from "@/shared/api/relayClient"; +import type { RelayEvent } from "@/shared/api/types"; +import { cn } from "@/shared/lib/cn"; +import { Button } from "@/shared/ui/button"; +import { useHuddle } from "../HuddleContext"; + +/** Huddle lifecycle event kinds */ +const KIND_HUDDLE_STARTED = 48100; +const KIND_HUDDLE_PARTICIPANT_JOINED = 48101; +const KIND_HUDDLE_PARTICIPANT_LEFT = 48102; +const KIND_HUDDLE_ENDED = 48103; + +type ActiveHuddle = { + ephemeralChannelId: string; + livekitRoom: string; + participants: Set; +}; + +type HuddleIndicatorProps = { + channelId: string; + className?: string; + /** Called when the user clicks the button and no huddle is active (start). */ + onStart?: () => void; + /** Whether the start action is disabled (e.g., permissions, already starting). */ + startDisabled?: boolean; +}; + +/** + * Detects active huddles in a channel via kind:48100-48103 events. + * Shows a glowing headphone icon when a huddle is active, with participant count. + * Click to join the huddle. + */ +export function HuddleIndicator({ + channelId, + className, + onStart, + startDisabled, +}: HuddleIndicatorProps) { + const { joinHuddle, isStarting } = useHuddle(); + const queryClient = useQueryClient(); + const [activeHuddle, setActiveHuddle] = React.useState( + null, + ); + const [isJoining, setIsJoining] = React.useState(false); + + React.useEffect(() => { + if (!channelId) return; + + let disposed = false; + let cleanup: (() => void) | null = null; + + // Track all seen events for reconstruction. Keyed by event.id for dedup. + const seenEvents = new Map(); + + /** Reconstruct huddle state from the full set of seen events. + * Sort by created_at, then kind (causal: start < join < left < end), + * then event id for final tiebreak. This handles out-of-order delivery, + * reconnect replay, late mounts, and same-second event batches. + * + * Resilient to missing start event: if we see join/left events for an + * ephemeral channel without a prior start, we infer the huddle exists. + * This covers the edge case where >100 lifecycle events push the start + * event out of the subscription window. */ + function reconstruct() { + const sorted = [...seenEvents.values()].sort( + (a, b) => + a.created_at - b.created_at || + a.kind - b.kind || + a.id.localeCompare(b.id), + ); + + let huddle: ActiveHuddle | null = null; + + for (const ev of sorted) { + let ephId: string | null = null; + let room = ""; + try { + const content = JSON.parse(ev.content); + ephId = content.ephemeral_channel_id ?? null; + room = content.livekit_room ?? ""; + } catch { + continue; // Malformed — skip + } + + switch (ev.kind) { + case KIND_HUDDLE_STARTED: { + if (!ephId) break; + huddle = { + ephemeralChannelId: ephId, + livekitRoom: room, + participants: new Set([ev.pubkey]), + }; + break; + } + case KIND_HUDDLE_PARTICIPANT_JOINED: { + if (!ephId) break; + // Infer huddle exists if we missed the start event (late mount + // or >100 lifecycle events pushed it out of the window). + if (!huddle || ephId !== huddle.ephemeralChannelId) { + huddle = { + ephemeralChannelId: ephId, + livekitRoom: room, + participants: new Set(), + }; + } + huddle.participants.add(ev.pubkey); + break; + } + case KIND_HUDDLE_PARTICIPANT_LEFT: { + if (!ephId) break; + // Infer huddle exists from LEFT too — if the window starts with + // only LEFT events (all joiners departed, creator still active), + // we still need to know the huddle is alive. + if (!huddle || ephId !== huddle.ephemeralChannelId) { + huddle = { + ephemeralChannelId: ephId, + livekitRoom: room, + participants: new Set(), + }; + } + huddle.participants.delete(ev.pubkey); + break; + } + case KIND_HUDDLE_ENDED: { + if (!huddle || !ephId || ephId !== huddle.ephemeralChannelId) break; + huddle = null; + break; + } + } + } + + if (!disposed) { + setActiveHuddle(huddle); + } + } + + // Subscribe to huddle lifecycle events only (kinds 48100–48103). + // limit: 100 covers long-lived huddles with many join/leave cycles. + relayClient + .subscribeToHuddleEvents(channelId, (event: RelayEvent) => { + if (disposed) return; + + // Dedup by event ID — ignore replayed events from reconnect. + if (seenEvents.has(event.id)) return; + seenEvents.set(event.id, event); + + // Reconstruct from full history on every new event. + // This is cheap — huddle lifecycle events are rare (typically <20). + reconstruct(); + }) + .then((dispose) => { + if (disposed) { + void dispose(); + return; + } + cleanup = () => void dispose(); + }) + .catch((err) => { + console.error("[HuddleIndicator] subscription failed:", err); + }); + + return () => { + disposed = true; + cleanup?.(); + setActiveHuddle(null); + }; + }, [channelId]); + + // No active huddle — render the start button (if onStart provided). + if (!activeHuddle) { + if (!onStart) return null; + return ( + + ); + } + + // At least 1 participant must exist for the huddle to be active. + // When START fell out of the event window, the creator isn't in the + // reconstructed set — floor at 1 to avoid showing "0 participants". + const participantCount = Math.max(1, activeHuddle.participants.size); + + async function handleJoin() { + if (!activeHuddle || isJoining) return; + setIsJoining(true); + try { + await joinHuddle( + channelId, + activeHuddle.ephemeralChannelId, + activeHuddle.livekitRoom, + ); + // Refetch channels so the ephemeral channel appears in the sidebar. + void queryClient.invalidateQueries({ queryKey: ["channels"] }); + } catch (e) { + console.error("Failed to join huddle:", e); + } finally { + setIsJoining(false); + } + } + + return ( + + ); +} diff --git a/desktop/src/features/huddle/components/ParticipantList.tsx b/desktop/src/features/huddle/components/ParticipantList.tsx index 426c6a6be4..ad12b4e484 100644 --- a/desktop/src/features/huddle/components/ParticipantList.tsx +++ b/desktop/src/features/huddle/components/ParticipantList.tsx @@ -5,11 +5,13 @@ import { ProfileAvatar } from "@/features/profile/ui/ProfileAvatar"; type ParticipantListProps = { /** Pubkey hex strings from the Rust huddle state */ participants: string[]; + activeSpeakers?: string[]; className?: string; }; export function ParticipantList({ participants, + activeSpeakers, className, }: ParticipantListProps) { const { data } = useUsersBatchQuery(participants); @@ -22,17 +24,33 @@ export function ParticipantList({ {participants.map((pubkey) => { const profile = profiles[pubkey.toLowerCase()]; const hasProfile = profile?.displayName || profile?.avatarUrl; + const isActive = activeSpeakers?.includes(pubkey); + const ariaLabel = + profile?.displayName || `Participant ${pubkey.slice(0, 8)}`; return hasProfile ? ( -
    +
    ) : ( - + ); })}
    @@ -40,15 +58,28 @@ export function ParticipantList({ } /** Compact hex-prefix avatar for participants without a loaded profile. */ -function HexAvatar({ pubkey }: { pubkey: string }) { +function HexAvatar({ + pubkey, + activeSpeakers, +}: { + pubkey: string; + activeSpeakers?: string[]; +}) { const shortId = pubkey.slice(0, 6).toUpperCase(); const parsed = parseInt(pubkey.slice(0, 4), 16); const hue = Number.isNaN(parsed) ? 0 : parsed % 360; const sat = Number.isNaN(parsed) ? 0 : 60; + const isActive = activeSpeakers?.includes(pubkey); return (
    diff --git a/desktop/src/features/huddle/index.ts b/desktop/src/features/huddle/index.ts index e6f1280ad8..8cf5ce8ea8 100644 --- a/desktop/src/features/huddle/index.ts +++ b/desktop/src/features/huddle/index.ts @@ -1,6 +1,6 @@ export { HuddleProvider, useHuddle } from "./HuddleContext"; export { connectToHuddle } from "./lib/livekit"; -export type { HuddleConnection } from "./lib/livekit"; +export type { HuddleConnection, HuddleRoomCallbacks } from "./lib/livekit"; export { setupAudioWorklet } from "./lib/audioWorklet"; export { HuddleBar } from "./components/HuddleBar"; export { ParticipantList } from "./components/ParticipantList"; diff --git a/desktop/src/features/huddle/lib/livekit.ts b/desktop/src/features/huddle/lib/livekit.ts index 0acc308fcb..37f10d1466 100644 --- a/desktop/src/features/huddle/lib/livekit.ts +++ b/desktop/src/features/huddle/lib/livekit.ts @@ -1,4 +1,9 @@ -import { LocalAudioTrack, Room } from "livekit-client"; +import { + LocalAudioTrack, + Room, + RoomEvent, + type Participant, +} from "livekit-client"; export interface HuddleConnection { room: Room; @@ -6,16 +11,25 @@ export interface HuddleConnection { disconnect: () => Promise; } +export type HuddleRoomCallbacks = { + onActiveSpeakersChanged?: (speakers: Participant[]) => void; + onDisconnected?: () => void; + onReconnecting?: () => void; + onReconnected?: () => void; +}; + /** * LiveKit connection lifecycle: * - * connectToHuddle(url, token) + * connectToHuddle(url, token, callbacks?) * → getUserMedia({ audio: true }) [mic permission] * → room.connect(url, token) [WebRTC signaling] + * → register room event listeners [active speakers, disconnect, reconnect] * → room.localParticipant.publishTrack(audioTrack) * → returns { room, localAudioTrack, disconnect } * * disconnect() + * → room.removeAllListeners() * → room.disconnect() * → stream.getTracks().forEach(t => t.stop()) * @@ -24,22 +38,43 @@ export interface HuddleConnection { export async function connectToHuddle( url: string, token: string, + callbacks?: HuddleRoomCallbacks, ): Promise { const room = new Room(); let stream: MediaStream | null = null; try { - stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + stream = await navigator.mediaDevices.getUserMedia({ + audio: { echoCancellation: true, noiseSuppression: true }, + }); const audioTrack = stream.getAudioTracks()[0]; await room.connect(url, token); + // Register room event listeners before publishing track + if (callbacks?.onActiveSpeakersChanged) { + room.on( + RoomEvent.ActiveSpeakersChanged, + callbacks.onActiveSpeakersChanged, + ); + } + if (callbacks?.onDisconnected) { + room.on(RoomEvent.Disconnected, callbacks.onDisconnected); + } + if (callbacks?.onReconnecting) { + room.on(RoomEvent.Reconnecting, callbacks.onReconnecting); + } + if (callbacks?.onReconnected) { + room.on(RoomEvent.Reconnected, callbacks.onReconnected); + } + try { // false = don't let LiveKit manage the track lifecycle const localTrack = new LocalAudioTrack(audioTrack, undefined, false); await room.localParticipant.publishTrack(localTrack); } catch (publishErr) { // Publish failed after connect — disconnect room before propagating + room.removeAllListeners(); room.disconnect(); throw publishErr; } @@ -49,6 +84,7 @@ export async function connectToHuddle( localAudioTrack: audioTrack, disconnect: async () => { try { + room.removeAllListeners(); room.disconnect(); } finally { stream?.getTracks().forEach((t) => { diff --git a/desktop/src/shared/api/relayClientSession.ts b/desktop/src/shared/api/relayClientSession.ts index 3dc63fbdad..01e58857ea 100644 --- a/desktop/src/shared/api/relayClientSession.ts +++ b/desktop/src/shared/api/relayClientSession.ts @@ -178,6 +178,26 @@ export class RelayClient { ); } + /** + * Subscribe to huddle lifecycle events (kinds 48100–48103) for a channel. + * Used by HuddleIndicator to detect active huddles without being drowned + * out by regular channel messages in the generic subscription window. + * Includes both historical (last 10) and live events. + */ + async subscribeToHuddleEvents( + channelId: string, + onEvent: (event: RelayEvent) => void, + ) { + return this.subscribe( + { + kinds: [48100, 48101, 48102, 48103], + "#h": [channelId], + limit: 100, + }, + onEvent, + ); + } + async subscribeToTypingIndicators( channelId: string, onEvent: (event: RelayEvent) => void, From 911771599b517e9758a076db47a339ab28ce1e73 Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Mon, 13 Apr 2026 19:26:48 -0400 Subject: [PATCH 36/41] =?UTF-8?q?fix(huddle):=20crossfire=20review=20fixes?= =?UTF-8?q?=20=E2=80=94=20split,=20harden,=20clean=20up?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crossfire review (3 models, 4 rounds) identified and fixed: Architecture: - Split mod.rs (1494→990 lines) into state.rs, relay_api.rs, pipeline.rs - Add synchronization protocol documentation at top of mod.rs - Add LiveKit webhook handler for server-side crash-orphan presence detection - Add KIND_HUDDLE_GUIDELINES constant to sprout-core Bugs: - Fix HuddleBar.tsx reading 'supertonic' field (renamed to 'kokoro' in Rust) - Fix VoiceStyle::get() panic on truncated/corrupt voice files - Fix ORT API version conflict: pin ort to api-23 (compat with sherpa-onnx 1.23) Concurrency: - Add stt_starting sentinel (mirrors tts_starting) to prevent TOCTOU races - Add phase checks before pipeline install (prevents install into torn-down huddle) - Fix STT pipeline restart: take() old pipeline out of lock before drop - Fix join_huddle rollback: only leave channel if THIS attempt added membership Code quality: - Delete supertonic.rs (700 lines dead code) + update file-size overrides - Replace magic kind numbers 48100-48106 with named constants - DRY fetch_channel_members/count_human_members via shared fetch_with_roles - Fix brittle 'already' string matching → specific 'already a member' - Add 20 G2P unit tests for kokoro.rs (tokenizer, ARPAbet, suffix rules) - Enable webhook feature flag in sprout-relay for sprout-huddle - Differentiate webhook 401 (signature) from 200 (parse error on signed payload) --- crates/sprout-core/src/kind.rs | 2 + crates/sprout-relay/Cargo.toml | 2 +- crates/sprout-relay/src/api/mod.rs | 3 + crates/sprout-relay/src/api/webhooks.rs | 120 +++ crates/sprout-relay/src/handlers/ingest.rs | 38 +- crates/sprout-relay/src/router.rs | 7 + desktop/scripts/check-file-sizes.mjs | 6 +- desktop/src-tauri/Cargo.lock | 23 +- desktop/src-tauri/Cargo.toml | 5 +- desktop/src-tauri/src/huddle/kokoro.rs | 880 ++++++++++++++++++ desktop/src-tauri/src/huddle/mod.rs | 617 ++---------- desktop/src-tauri/src/huddle/models.rs | 198 ++-- desktop/src-tauri/src/huddle/pipeline.rs | 280 ++++++ desktop/src-tauri/src/huddle/relay_api.rs | 98 ++ desktop/src-tauri/src/huddle/state.rs | 227 +++++ desktop/src-tauri/src/huddle/stt.rs | 4 +- desktop/src-tauri/src/huddle/supertonic.rs | 700 -------------- desktop/src-tauri/src/huddle/tts.rs | 160 ++-- desktop/src-tauri/src/lib.rs | 4 +- desktop/src/features/huddle/HuddleContext.tsx | 28 +- .../features/huddle/components/HuddleBar.tsx | 42 +- .../src/features/huddle/lib/audioWorklet.ts | 27 +- 22 files changed, 1987 insertions(+), 1484 deletions(-) create mode 100644 crates/sprout-relay/src/api/webhooks.rs create mode 100644 desktop/src-tauri/src/huddle/kokoro.rs create mode 100644 desktop/src-tauri/src/huddle/pipeline.rs create mode 100644 desktop/src-tauri/src/huddle/relay_api.rs create mode 100644 desktop/src-tauri/src/huddle/state.rs delete mode 100644 desktop/src-tauri/src/huddle/supertonic.rs diff --git a/crates/sprout-core/src/kind.rs b/crates/sprout-core/src/kind.rs index 164b48ae2d..ef2a777f83 100644 --- a/crates/sprout-core/src/kind.rs +++ b/crates/sprout-core/src/kind.rs @@ -219,6 +219,8 @@ pub const KIND_HUDDLE_ENDED: u32 = 48103; pub const KIND_HUDDLE_TRACK_PUBLISHED: u32 = 48104; /// A huddle recording became available. pub const KIND_HUDDLE_RECORDING_AVAILABLE: u32 = 48105; +/// Huddle channel guidelines/rules document. +pub const KIND_HUDDLE_GUIDELINES: u32 = 48106; // Media (49000–49999) /// Internal kind for media upload audit entries. Not a relay event kind. diff --git a/crates/sprout-relay/Cargo.toml b/crates/sprout-relay/Cargo.toml index ba5433e2d1..09da25aecd 100644 --- a/crates/sprout-relay/Cargo.toml +++ b/crates/sprout-relay/Cargo.toml @@ -38,7 +38,7 @@ deadpool-redis = { workspace = true } redis = { workspace = true } sqlx = { workspace = true } base64 = "0.22" -sprout-huddle = { workspace = true } +sprout-huddle = { workspace = true, features = ["webhook"] } sprout-workflow = { workspace = true, features = ["reqwest"] } sprout-media = { workspace = true } bytes = "1" diff --git a/crates/sprout-relay/src/api/mod.rs b/crates/sprout-relay/src/api/mod.rs index 067936de16..ffac082907 100644 --- a/crates/sprout-relay/src/api/mod.rs +++ b/crates/sprout-relay/src/api/mod.rs @@ -46,6 +46,8 @@ pub mod search; pub mod tokens; /// User profile endpoints. pub mod users; +/// LiveKit webhook handler for server-side presence tracking. +pub mod webhooks; /// Shared helpers for workflow API handlers. pub mod workflow_helpers; /// Workflow CRUD, trigger, and webhook endpoints. @@ -70,6 +72,7 @@ pub use users::{ get_contact_list, get_profile, get_user_notes, get_user_profile, get_users_batch, put_channel_add_policy, search_users, }; +pub use webhooks::handle_livekit_webhook; pub use workflows::{ create_workflow, delete_workflow, get_workflow, list_channel_workflows, list_run_approvals, list_workflow_runs, trigger_workflow, update_workflow, workflow_webhook, diff --git a/crates/sprout-relay/src/api/webhooks.rs b/crates/sprout-relay/src/api/webhooks.rs new file mode 100644 index 0000000000..57c6da4f39 --- /dev/null +++ b/crates/sprout-relay/src/api/webhooks.rs @@ -0,0 +1,120 @@ +//! LiveKit webhook handler — server-side huddle presence tracking. +//! +//! Receives webhook events from LiveKit and emits corresponding Nostr +//! huddle lifecycle events (kinds 48100–48103). This provides authoritative +//! presence tracking that survives client crashes — LiveKit fires +//! `participant_left` when the WebRTC connection drops, regardless of +//! whether the client performed a graceful shutdown. +//! +//! ## Route +//! `POST /internal/livekit/webhook` — internal only, not exposed through +//! the public API gateway. + +use std::sync::Arc; + +use axum::{ + extract::State, + http::{HeaderMap, StatusCode}, +}; +use sprout_huddle::WebhookEvent; + +use crate::state::AppState; + +/// Handle a LiveKit webhook event. +/// +/// Verifies the webhook signature, parses the event, and emits the +/// corresponding Nostr huddle lifecycle event to the parent channel. +/// +/// Returns 200 on success (even if the event type is unrecognized — +/// LiveKit expects 2xx for all webhook deliveries). +/// Returns 401 if signature verification fails. +/// Returns 501 if huddles are not configured. +pub async fn handle_livekit_webhook( + State(state): State>, + headers: HeaderMap, + body: String, +) -> StatusCode { + let huddle_service = match state.huddle_service.as_ref() { + Some(svc) => svc, + None => return StatusCode::NOT_IMPLEMENTED, + }; + + // Verify signature and parse the event. + let auth_header = headers + .get("Authorization") + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + + let event = match huddle_service.parse_webhook(body.as_bytes(), auth_header) { + Ok(e) => e, + Err(sprout_huddle::HuddleError::InvalidWebhookSignature) => { + tracing::warn!("LiveKit webhook: signature verification failed"); + return StatusCode::UNAUTHORIZED; + } + Err(e) => { + // Signed but malformed/unknown — log and return 200 so LiveKit + // doesn't retry. Parse failures are not auth failures. + tracing::warn!("LiveKit webhook: parse error (signed OK): {e}"); + return StatusCode::OK; + } + }; + + // Dispatch on the parsed enum variant. + // Room names follow the format "sprout-{channel_uuid}". + match event { + WebhookEvent::RoomStarted { room } => { + tracing::info!("LiveKit: room started {room}"); + // TODO: Emit kind:48100 signed by relay keypair + } + WebhookEvent::RoomFinished { room } => { + tracing::info!("LiveKit: room finished {room}"); + // TODO: Emit kind:48103 + archive ephemeral channel + } + WebhookEvent::ParticipantJoined { room, identity } => { + let Some(channel_id) = parse_channel_id(&room) else { + tracing::debug!("LiveKit webhook for non-sprout room or invalid UUID: {room}"); + return StatusCode::OK; + }; + tracing::info!( + "LiveKit: participant joined room {room} (channel {channel_id}), identity={identity}" + ); + // TODO: Emit kind:48101 signed by relay keypair. + // The client also emits this event; the server-side copy provides + // crash-recovery redundancy. + } + WebhookEvent::ParticipantLeft { room, identity } => { + let Some(channel_id) = parse_channel_id(&room) else { + tracing::debug!("LiveKit webhook for non-sprout room or invalid UUID: {room}"); + return StatusCode::OK; + }; + tracing::info!( + "LiveKit: participant left room {room} (channel {channel_id}), identity={identity}" + ); + // TODO: Emit kind:48102 signed by relay keypair. + // This is the key crash-recovery path — fires even if the client crashed. + } + WebhookEvent::TrackPublished { + room, + identity, + kind, + } => { + tracing::debug!("LiveKit: track published in {room} by {identity} (kind={kind})"); + // TODO: Emit kind:48104 (track published) if/when that event kind is defined. + } + } + + StatusCode::OK +} + +/// Extract the channel UUID from a LiveKit room name. +/// +/// Room names follow the format `sprout-{uuid}`. Returns `None` if the name +/// does not match the expected prefix or contains an invalid UUID. +fn parse_channel_id(room_name: &str) -> Option { + let channel_id = room_name.strip_prefix("sprout-")?; + uuid::Uuid::parse_str(channel_id) + .map_err(|_| { + tracing::warn!("LiveKit webhook: invalid channel UUID in room name: {room_name}"); + }) + .ok() +} diff --git a/crates/sprout-relay/src/handlers/ingest.rs b/crates/sprout-relay/src/handlers/ingest.rs index caf31f29de..2242906213 100644 --- a/crates/sprout-relay/src/handlers/ingest.rs +++ b/crates/sprout-relay/src/handlers/ingest.rs @@ -14,13 +14,15 @@ use sprout_auth::Scope; use sprout_core::kind::{ event_kind_u32, is_parameterized_replaceable, KIND_AUTH, KIND_CANVAS, KIND_CONTACT_LIST, KIND_DELETION, KIND_FORUM_COMMENT, KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_GIFT_WRAP, - KIND_LONG_FORM, KIND_MEMBER_ADDED_NOTIFICATION, KIND_MEMBER_REMOVED_NOTIFICATION, - KIND_NIP29_CREATE_GROUP, KIND_NIP29_DELETE_EVENT, KIND_NIP29_DELETE_GROUP, - KIND_NIP29_EDIT_METADATA, KIND_NIP29_JOIN_REQUEST, KIND_NIP29_LEAVE_REQUEST, - KIND_NIP29_PUT_USER, KIND_NIP29_REMOVE_USER, KIND_PRESENCE_UPDATE, KIND_PROFILE, KIND_REACTION, - KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_BOOKMARKED, KIND_STREAM_MESSAGE_DIFF, - KIND_STREAM_MESSAGE_EDIT, KIND_STREAM_MESSAGE_PINNED, KIND_STREAM_MESSAGE_SCHEDULED, - KIND_STREAM_MESSAGE_V2, KIND_STREAM_REMINDER, KIND_TEXT_NOTE, + KIND_HUDDLE_ENDED, KIND_HUDDLE_GUIDELINES, KIND_HUDDLE_PARTICIPANT_JOINED, + KIND_HUDDLE_PARTICIPANT_LEFT, KIND_HUDDLE_RECORDING_AVAILABLE, KIND_HUDDLE_STARTED, + KIND_HUDDLE_TRACK_PUBLISHED, KIND_LONG_FORM, KIND_MEMBER_ADDED_NOTIFICATION, + KIND_MEMBER_REMOVED_NOTIFICATION, KIND_NIP29_CREATE_GROUP, KIND_NIP29_DELETE_EVENT, + KIND_NIP29_DELETE_GROUP, KIND_NIP29_EDIT_METADATA, KIND_NIP29_JOIN_REQUEST, + KIND_NIP29_LEAVE_REQUEST, KIND_NIP29_PUT_USER, KIND_NIP29_REMOVE_USER, KIND_PRESENCE_UPDATE, + KIND_PROFILE, KIND_REACTION, KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_BOOKMARKED, + KIND_STREAM_MESSAGE_DIFF, KIND_STREAM_MESSAGE_EDIT, KIND_STREAM_MESSAGE_PINNED, + KIND_STREAM_MESSAGE_SCHEDULED, KIND_STREAM_MESSAGE_V2, KIND_STREAM_REMINDER, KIND_TEXT_NOTE, }; use sprout_core::verification::verify_event; @@ -176,8 +178,14 @@ fn required_scope_for_kind(kind: u32, event: &Event) -> Result Ok(Scope::ChannelsWrite), KIND_NIP29_JOIN_REQUEST | KIND_NIP29_LEAVE_REQUEST => Ok(Scope::ChannelsRead), - // Huddle lifecycle events (kind 48100–48103) + guidelines (48106) - 48100..=48106 => Ok(Scope::ChannelsWrite), + // Huddle lifecycle events + guidelines + KIND_HUDDLE_STARTED + | KIND_HUDDLE_PARTICIPANT_JOINED + | KIND_HUDDLE_PARTICIPANT_LEFT + | KIND_HUDDLE_ENDED + | KIND_HUDDLE_TRACK_PUBLISHED + | KIND_HUDDLE_RECORDING_AVAILABLE + | KIND_HUDDLE_GUIDELINES => Ok(Scope::ChannelsWrite), _ => Err("restricted: unknown event kind"), } } @@ -285,12 +293,12 @@ pub(crate) fn requires_h_channel_scope(kind: u32) -> bool { | KIND_NIP29_DELETE_EVENT | KIND_NIP29_DELETE_GROUP | KIND_NIP29_LEAVE_REQUEST - // Huddle lifecycle events (kind 48100–48103) + guidelines (48106) - | 48100 - | 48101 - | 48102 - | 48103 - | 48106 + // Huddle lifecycle events + guidelines + | KIND_HUDDLE_STARTED + | KIND_HUDDLE_PARTICIPANT_JOINED + | KIND_HUDDLE_PARTICIPANT_LEFT + | KIND_HUDDLE_ENDED + | KIND_HUDDLE_GUIDELINES ) } diff --git a/crates/sprout-relay/src/router.rs b/crates/sprout-relay/src/router.rs index 25bd7c52b2..921213cc16 100644 --- a/crates/sprout-relay/src/router.rs +++ b/crates/sprout-relay/src/router.rs @@ -51,6 +51,13 @@ pub fn build_router(state: Arc) -> Router { // ── All other routes: 1 MB body limit ──────────────────────────────────── let api_router = Router::new() + // ── Internal routes (not exposed through the public API gateway) ───── + // LiveKit fires this on participant_joined/left (including crashes), + // providing authoritative presence tracking for crash-orphan recovery. + .route( + "/internal/livekit/webhook", + post(api::handle_livekit_webhook), + ) .route("/", get(nip11_or_ws_handler)) .route("/info", get(relay_info_handler)) .route("/.well-known/nostr.json", get(api::nip05::nostr_nip05)) diff --git a/desktop/scripts/check-file-sizes.mjs b/desktop/scripts/check-file-sizes.mjs index e033168166..d0a91b04dd 100644 --- a/desktop/scripts/check-file-sizes.mjs +++ b/desktop/scripts/check-file-sizes.mjs @@ -57,11 +57,11 @@ const overrides = new Map([ ["src/features/channels/ui/AddChannelBotDialog.tsx", 640], // provider mode: Run on selector, trust warning, probe effect, single-agent enforcement, provider warnings display ["src/shared/api/types.ts", 535], // persona provider/model fields + forum types + workflow type re-exports + ephemeral channel TTL fields + mcpToolsets ["src-tauri/src/events.rs", 530], // event builders + build_huddle_guidelines (kind:48106) + post_event_raw transport helper - ["src-tauri/src/huddle/mod.rs", 1450], // huddle state machine + 16 Tauri commands + STT/TTS pipeline lifecycle + VoiceInputMode (PTT/VAD) + relay membership fetch + session generation guard + multi-human join/leave/auto-end + count_human_members + input validation; split planned post-MVP - ["src-tauri/src/huddle/models.rs", 850], // model download manager for Moonshine STT + Supertonic TTS with streaming downloads + SHA-256 verification + Rust-native tar extraction + version manifest + atomic swap + hot-start signaling + ["src-tauri/src/huddle/kokoro.rs", 890], // Kokoro ONNX TTS engine + three-tier G2P + ARPAbet→IPA + CoreML + 20 G2P unit tests + ["src-tauri/src/huddle/mod.rs", 1000], // huddle state machine + Tauri commands + sync protocol doc; state/relay/pipeline extracted + ["src-tauri/src/huddle/models.rs", 850], // model download manager for Moonshine STT + Kokoro TTS with streaming downloads + SHA-256 verification + Rust-native tar extraction + version manifest + atomic swap + hot-start signaling ["src-tauri/src/huddle/stt.rs", 580], // STT pipeline + PTT edge-detection flush + PTT gating (is_speech AND ptt_active) + barge-in for VAD mode + rubato resampler + earshot VAD + sherpa-onnx transcription ["src-tauri/src/huddle/preprocessing.rs", 620], // TTS text preprocessing pipeline + unified split_sentences (consolidated from tts.rs + supertonic.rs) + int_to_words 0-999999 + 18 unit tests - ["src-tauri/src/huddle/supertonic.rs", 780], // Supertonic 4-ONNX-session TTS engine wrapper + Unicode text processor + LazyLock regex patterns + text chunking ]); async function walkFiles(directory) { diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 57cc2e363e..9d86ce1c31 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -3441,9 +3441,9 @@ dependencies = [ [[package]] name = "ort" -version = "2.0.0-rc.11" +version = "2.0.0-rc.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5df903c0d2c07b56950f1058104ab0c8557159f2741782223704de9be73c3c" +checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" dependencies = [ "ndarray", "ort-sys", @@ -3454,9 +3454,9 @@ dependencies = [ [[package]] name = "ort-sys" -version = "2.0.0-rc.11" +version = "2.0.0-rc.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06503bb33f294c5f1ba484011e053bfa6ae227074bdb841e9863492dc5960d4b" +checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" dependencies = [ "hmac-sha256", "lzma-rust2", @@ -4189,16 +4189,6 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" -[[package]] -name = "rand_distr" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" -dependencies = [ - "num-traits", - "rand 0.8.5", -] - [[package]] name = "rand_distr" version = "0.6.0" @@ -4476,7 +4466,7 @@ dependencies = [ "dasp_sample", "num-rational", "rand 0.10.1", - "rand_distr 0.6.0", + "rand_distr", "rtrb", "symphonia", "thiserror 2.0.18", @@ -5190,8 +5180,6 @@ dependencies = [ "nostr 0.37.0", "ort", "png 0.18.1", - "rand 0.8.5", - "rand_distr 0.4.3", "regex", "reqwest 0.13.2", "rodio", @@ -5214,7 +5202,6 @@ dependencies = [ "tauri-plugin-window-state", "tempfile", "tokio", - "unicode-normalization", "url", "uuid", "windows-sys 0.61.2", diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index 4315ee9060..7ddc478e50 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -53,12 +53,9 @@ uuid = { version = "1", features = ["v4"] } png = "0.18" zip = "2" sherpa-onnx = "1.12" -ort = { version = "=2.0.0-rc.11", features = ["ndarray"] } +ort = { version = "=2.0.0-rc.12", default-features = false, features = ["std", "ndarray", "tracing", "download-binaries", "tls-native", "copy-dylibs", "api-23", "coreml"] } ndarray = { version = "0.17", features = ["rayon"] } -unicode-normalization = "0.1" regex = "1" -rand = "0.8" -rand_distr = "0.4" rodio = "0.22" earshot = "1.0" rubato = "2.0" diff --git a/desktop/src-tauri/src/huddle/kokoro.rs b/desktop/src-tauri/src/huddle/kokoro.rs new file mode 100644 index 0000000000..bbb0a4148d --- /dev/null +++ b/desktop/src-tauri/src/huddle/kokoro.rs @@ -0,0 +1,880 @@ +//! Kokoro-82M ONNX TTS engine — single-session inference with IPA G2P. +//! +//! Mental model: +//! +//! load_text_to_speech(model_dir) → KokoroTTS +//! load_voice_style(path) → VoiceStyle +//! tts.call(text, lang, &style) → Vec @ 24 kHz +//! +//! ┌──────────┐ G2P ┌──────────┐ tokenize ┌──────────┐ +//! │ raw text │ ──────→ │ IPA str │ ─────────→ │ int64[] │ +//! └──────────┘ lexicon └──────────┘ 115-char └────┬─────┘ +//! │ +//! ┌──────────┐ style ┌──────────┐ ONNX ┌────▼─────┐ +//! │ .bin file│ ──────→ │ [1, 256] │ ─────────→ │ Vec │ +//! └──────────┘ indexed └──────────┘ session └──────────┘ +//! by token count 24 kHz PCM +//! +//! G2P strategy: dictionary lookup (us_gold.json, Apache-2.0 via misaki). +//! OOV words are spelled letter-by-letter using a static IPA table. +//! No espeak dependency — fully GPL-free. + +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use ndarray::{Array1, Array2}; +use ort::{session::Session, value::Value}; + +use super::preprocessing::split_sentences; + +// ── Public constants ────────────────────────────────────────────────────────── + +pub const SAMPLE_RATE: u32 = 24_000; +pub const DEFAULT_VOICE: &str = "af_heart"; + +// Maximum phoneme tokens before padding (model context = 512, minus 2 pad tokens). +const MAX_PHONEME_TOKENS: usize = 510; + +// ── VoiceStyle ──────────────────────────────────────────────────────────────── + +/// Raw f32 voice embedding loaded from a `.bin` file. +/// +/// The binary is a flat array of shape `[-1, 256]` in row-major order. +/// Row `i` is the style vector for an utterance with `i` phoneme tokens. +/// This encodes both speaker identity and sequence-length-dependent prosody. +#[derive(Debug)] +pub struct VoiceStyle { + data: Vec, // flat: row i = data[i*256 .. (i+1)*256] +} + +impl VoiceStyle { + /// Return the 256-dim style vector for a given phoneme token count. + /// Clamps to the last available row if `token_count` is out of range. + fn get(&self, token_count: usize) -> &[f32] { + let max_rows = self.data.len() / 256; + let idx = token_count.min(max_rows.saturating_sub(1)); + &self.data[idx * 256..(idx + 1) * 256] + } +} + +/// Load a voice style from a raw little-endian f32 binary file. +pub fn load_voice_style(path: &Path) -> Result { + let bytes = fs::read(path).map_err(|e| format!("read voice {}: {e}", path.display()))?; + if bytes.len() % 4 != 0 { + return Err(format!( + "voice file {} has non-multiple-of-4 byte count ({})", + path.display(), + bytes.len() + )); + } + let data: Vec = bytes + .chunks_exact(4) + .map(|b| f32::from_le_bytes(b.try_into().unwrap())) + .collect(); + if data.len() < 256 { + return Err(format!( + "voice file {} too small ({} floats, need at least 256)", + path.display(), + data.len() + )); + } + Ok(VoiceStyle { data }) +} + +// ── Tokenizer ───────────────────────────────────────────────────────────────── + +/// Static 115-entry IPA char → int64 lookup table. +/// IDs are non-contiguous (0–177); unknown chars are silently dropped. +/// Pad token '$' = 0 is prepended and appended to every sequence. +fn build_vocab() -> HashMap { + // Source: onnx-community/Kokoro-82M-v1.0-ONNX tokenizer.json + #[rustfmt::skip] + let entries: &[(char, i64)] = &[ + ('$', 0), + (';', 1), (':', 2), (',', 3), ('.', 4), ('!', 5), ('?', 6), + ('—', 9), ('…', 10), ('"', 11), ('(', 12), (')', 13), ('\u{201c}', 14), ('\u{201d}', 15), + (' ', 16), ('\u{0303}', 17), + ('ʣ', 18), ('ʥ', 19), ('ʦ', 20), ('ʨ', 21), ('ᵝ', 22), ('ꭧ', 23), + ('A', 24), ('I', 25), ('O', 31), ('Q', 33), ('S', 35), ('T', 36), + ('W', 39), ('Y', 41), ('ᵊ', 42), + ('a', 43), ('b', 44), ('c', 45), ('d', 46), ('e', 47), ('f', 48), + ('h', 50), ('i', 51), ('j', 52), ('k', 53), ('l', 54), ('m', 55), + ('n', 56), ('o', 57), ('p', 58), ('q', 59), ('r', 60), ('s', 61), + ('t', 62), ('u', 63), ('v', 64), ('w', 65), ('x', 66), ('y', 67), ('z', 68), + ('ɑ', 69), ('ɐ', 70), ('ɒ', 71), ('æ', 72), ('β', 75), ('ɔ', 76), + ('ɕ', 77), ('ç', 78), ('ɖ', 80), ('ð', 81), ('ʤ', 82), ('ə', 83), + ('ɚ', 85), ('ɛ', 86), ('ɜ', 87), ('ɟ', 90), ('ɡ', 92), ('ɥ', 99), + ('ɨ', 101), ('ɪ', 102), ('ʝ', 103), ('ɯ', 110), ('ɰ', 111), + ('ŋ', 112), ('ɳ', 113), ('ɲ', 114), ('ɴ', 115), ('ø', 116), + ('ɸ', 118), ('θ', 119), ('œ', 120), ('ɹ', 123), ('ɾ', 125), + ('ɻ', 126), ('ʁ', 128), ('ɽ', 129), ('ʂ', 130), ('ʃ', 131), + ('ʈ', 132), ('ʧ', 133), ('ʊ', 135), ('ʋ', 136), ('ʌ', 138), + ('ɣ', 139), ('ɤ', 140), ('χ', 142), ('ʎ', 143), ('ʒ', 147), + ('ʔ', 148), ('ˈ', 156), ('ˌ', 157), ('ː', 158), ('ʰ', 162), + ('ʲ', 164), ('↓', 169), ('→', 171), ('↗', 172), ('↘', 173), ('ᵻ', 177), + ]; + entries.iter().copied().collect() +} + +/// Convert an IPA phoneme string to a padded int64 token sequence. +/// Returns `[0, id1, id2, ..., idN, 0]` clamped to MAX_PHONEME_TOKENS+2. +/// The pre-pad token count (ids.len() - 2) is used to index the style vector. +fn tokenize(phonemes: &str, vocab: &HashMap) -> Vec { + let mut ids: Vec = vec![0]; // BOS pad + for id in phonemes + .chars() + .filter_map(|c| vocab.get(&c).copied()) + .take(MAX_PHONEME_TOKENS) + { + ids.push(id); + } + ids.push(0); // EOS pad + ids +} + +// ── G2P Lexicon ─────────────────────────────────────────────────────────────── + +/// Grapheme-to-phoneme engine with a four-tier fallback chain: +/// +/// 1. Misaki gold+silver dicts (183K words, Kokoro-native IPA) +/// 2. CMUdict (135K words, ARPAbet→Kokoro IPA) — covers inflected forms +/// 3. Morphological suffix stripping (-s/-ed/-ing) + retry tiers 1-2 +/// 4. Letter-by-letter spelling +/// +/// All dictionaries are Apache-2.0 or BSD licensed. No GPL. +struct Lexicon { + /// Misaki gold+silver merged dictionary (Kokoro-native IPA). + misaki: HashMap, + /// CMU Pronouncing Dictionary (ARPAbet converted to Kokoro IPA at load time). + cmudict: HashMap, +} + +/// IPA pronunciations for individual letter names (used for OOV words). +fn letter_ipa(c: char) -> &'static str { + match c { + 'a' => "ˈeɪ", + 'b' => "bˈiː", + 'c' => "sˈiː", + 'd' => "dˈiː", + 'e' => "ˈiː", + 'f' => "ˈɛf", + 'g' => "dʒˈiː", + 'h' => "ˈeɪtʃ", + 'i' => "ˈaɪ", + 'j' => "dʒˈeɪ", + 'k' => "kˈeɪ", + 'l' => "ˈɛl", + 'm' => "ˈɛm", + 'n' => "ˈɛn", + 'o' => "ˈoʊ", + 'p' => "pˈiː", + 'q' => "kjˈuː", + 'r' => "ˈɑːɹ", + 's' => "ˈɛs", + 't' => "tˈiː", + 'u' => "jˈuː", + 'v' => "vˈiː", + 'w' => "dˈʌbəljˌuː", + 'x' => "ˈɛks", + 'y' => "wˈaɪ", + 'z' => "zˈiː", + _ => "", + } +} + +/// Punctuation chars that are valid Kokoro vocab tokens and should pass through. +fn is_passthrough_punct(c: char) -> bool { + matches!(c, ';' | ':' | ',' | '.' | '!' | '?' | '—' | '…' | ' ') +} + +/// Vowels that trigger US English /t/→/ɾ/ flapping (misaki's US_TAUS). +const US_TAUS: &str = "AIOWYiuæɑəɛɪɹʊʌ"; + +/// ARPAbet → Kokoro IPA conversion. Stress digit is stripped before lookup. +fn arpabet_to_ipa(phoneme: &str) -> &'static str { + match phoneme { + "AA" => "ɑ", + "AE" => "æ", + "AH" => "ʌ", + "AO" => "ɔ", + "AW" => "W", + "AY" => "I", + "EH" => "ɛ", + "ER" => "ɜɹ", + "EY" => "A", + "IH" => "ɪ", + "IY" => "i", + "OW" => "O", + "OY" => "Y", + "UH" => "ʊ", + "UW" => "u", + "B" => "b", + "CH" => "ʧ", + "D" => "d", + "DH" => "ð", + "F" => "f", + "G" => "ɡ", + "HH" => "h", + "JH" => "ʤ", + "K" => "k", + "L" => "l", + "M" => "m", + "N" => "n", + "NG" => "ŋ", + "P" => "p", + "R" => "ɹ", + "S" => "s", + "SH" => "ʃ", + "T" => "t", + "TH" => "θ", + "V" => "v", + "W" => "w", + "Y" => "j", + "Z" => "z", + "ZH" => "ʒ", + _ => "", + } +} + +/// Convert a CMUdict ARPAbet pronunciation line to Kokoro IPA. +/// Input: "K R IY0 EY1 T AH0 D" → Output: "kɹiˈAtəd" +fn arpabet_line_to_ipa(arpabet: &str) -> String { + let mut out = String::new(); + for token in arpabet.split_whitespace() { + // Split phoneme from stress digit (e.g., "EY1" → "EY", Some('1')) + let (base, stress) = if token.ends_with(|c: char| c.is_ascii_digit()) { + (&token[..token.len() - 1], token.as_bytes().last().copied()) + } else { + (token, None) + }; + // Stress marker goes BEFORE the vowel's IPA + match stress { + Some(b'1') => out.push('ˈ'), // primary + Some(b'2') => out.push('ˌ'), // secondary + _ => {} + } + // AH with stress=0 is schwa (ə), not ʌ + if base == "AH" && stress == Some(b'0') { + out.push('ə'); + } else if base == "ER" && stress == Some(b'0') { + // Unstressed ER is just əɹ + out.push_str("əɹ"); + } else { + out.push_str(arpabet_to_ipa(base)); + } + } + out +} + +impl Lexicon { + /// Load misaki gold+silver dicts and CMUdict. + fn load(gold_path: &Path, silver_path: &Path, cmudict_path: &Path) -> Result { + let mut misaki = Self::load_json(silver_path)?; + let gold = Self::load_json(gold_path)?; + misaki.extend(gold); + + let cmudict = if cmudict_path.exists() { + Self::load_cmudict(cmudict_path)? + } else { + eprintln!( + "sprout-desktop: CMUdict not found at {} — inflected forms may be spelled out", + cmudict_path.display() + ); + HashMap::new() + }; + + eprintln!( + "sprout-desktop: G2P loaded — misaki: {} words, cmudict: {} words", + misaki.len(), + cmudict.len() + ); + Ok(Lexicon { misaki, cmudict }) + } + + fn load_json(path: &Path) -> Result, String> { + let content = + fs::read_to_string(path).map_err(|e| format!("read {}: {e}", path.display()))?; + let raw: serde_json::Value = + serde_json::from_str(&content).map_err(|e| format!("parse {}: {e}", path.display()))?; + let obj = raw + .as_object() + .ok_or_else(|| format!("{}: expected JSON object", path.display()))?; + let mut dict = HashMap::with_capacity(obj.len()); + for (word, val) in obj { + let ipa = match val { + serde_json::Value::String(s) => s.clone(), + serde_json::Value::Object(m) => m + .get("DEFAULT") + .or_else(|| m.values().next()) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(), + _ => continue, + }; + if !ipa.is_empty() { + dict.insert(word.to_lowercase(), ipa); + } + } + Ok(dict) + } + + /// Load CMUdict and convert ARPAbet → Kokoro IPA at load time. + /// Format: "WORD PH1 PH2 PH3\n" (single space between word and phonemes). + /// Variant pronunciations like "WORD(2)" are skipped — we take the first. + fn load_cmudict(path: &Path) -> Result, String> { + let content = fs::read_to_string(path).map_err(|e| format!("read cmudict: {e}"))?; + let mut dict = HashMap::with_capacity(140_000); + for line in content.lines() { + // Skip comments and blank lines + if line.starts_with(";;;") || line.is_empty() { + continue; + } + // Split on first space + let (word, phonemes) = match line.find(' ') { + Some(i) => (&line[..i], line[i + 1..].trim()), + None => continue, + }; + // Skip variant pronunciations like "WORD(2)" + if word.contains('(') { + continue; + } + let key = word.to_lowercase(); + let ipa = arpabet_line_to_ipa(phonemes); + if !ipa.is_empty() { + dict.entry(key).or_insert(ipa); + } + } + Ok(dict) + } + + /// Look up a word across all tiers. Returns None if not found anywhere. + fn lookup(&self, word: &str) -> Option { + self.misaki + .get(word) + .cloned() + .or_else(|| self.cmudict.get(word).cloned()) + } + + /// Apply English -s/-es/-ies suffix phoneme rules (misaki's `_s`). + fn apply_s(stem_ipa: &str) -> String { + let last = stem_ipa.chars().last().unwrap_or(' '); + if "ptkfθ".contains(last) { + format!("{stem_ipa}s") + } else if "szʃʒʧʤ".contains(last) { + format!("{stem_ipa}ᵻz") + } else { + format!("{stem_ipa}z") + } + } + + /// Apply English -ed suffix phoneme rules (misaki's `_ed`). + fn apply_ed(stem_ipa: &str) -> String { + let chars: Vec = stem_ipa.chars().collect(); + let last = *chars.last().unwrap_or(&' '); + if "pkfθʃsʧ".contains(last) { + format!("{stem_ipa}t") + } else if last == 'd' { + format!("{stem_ipa}ᵻd") + } else if last != 't' { + format!("{stem_ipa}d") + } else if chars.len() >= 2 && US_TAUS.contains(chars[chars.len() - 2]) { + // US flap: "created" → kɹiˈAɾᵻd + let mut out: String = chars[..chars.len() - 1].iter().collect(); + out.push_str("ɾᵻd"); + out + } else { + format!("{stem_ipa}ᵻd") + } + } + + /// Apply English -ing suffix phoneme rules (misaki's `_ing`). + fn apply_ing(stem_ipa: &str) -> String { + let chars: Vec = stem_ipa.chars().collect(); + let last = *chars.last().unwrap_or(&' '); + if last == 't' && chars.len() >= 2 && US_TAUS.contains(chars[chars.len() - 2]) { + // US flap: "creating" → kɹiˈAɾɪŋ + let mut out: String = chars[..chars.len() - 1].iter().collect(); + out.push_str("ɾɪŋ"); + out + } else { + format!("{stem_ipa}ɪŋ") + } + } + + /// Try stripping -s/-ed/-ing suffix, look up the base, and re-apply phonetically. + fn try_morphological(&self, word: &str) -> Option { + // Try -s / -es / -ies + if word.len() >= 3 && word.ends_with('s') { + // -ies → base + y + if word.len() > 4 && word.ends_with("ies") { + if let Some(stem) = self.lookup(&format!("{}y", &word[..word.len() - 3])) { + return Some(Self::apply_s(&stem)); + } + } + // -es → base + if word.len() > 4 && word.ends_with("es") && !word.ends_with("ies") { + if let Some(stem) = self.lookup(&word[..word.len() - 2]) { + return Some(Self::apply_s(&stem)); + } + } + // -s → base + if !word.ends_with("ss") { + if let Some(stem) = self.lookup(&word[..word.len() - 1]) { + return Some(Self::apply_s(&stem)); + } + } + } + // Try -ed / -d + if word.len() >= 4 && word.ends_with('d') { + // -ed → base (not -eed) + if word.len() > 4 && word.ends_with("ed") && !word.ends_with("eed") { + if let Some(stem) = self.lookup(&word[..word.len() - 2]) { + return Some(Self::apply_ed(&stem)); + } + // -ed where base ends in e: "created" → "create" + if let Some(stem) = self.lookup(&format!("{}e", &word[..word.len() - 2])) { + return Some(Self::apply_ed(&stem)); + } + } + // -d → base (e.g., "discovered" → strip "d" → "discovere" fails, + // but "configured" → strip "d" → "configure" works) + if !word.ends_with("dd") { + if let Some(stem) = self.lookup(&word[..word.len() - 1]) { + return Some(Self::apply_ed(&stem)); + } + } + } + // Try -ing + if word.len() >= 5 && word.ends_with("ing") { + let base = &word[..word.len() - 3]; + // -ing → base (e.g., "running" base = "runn" — won't match, need double-consonant) + if let Some(stem) = self.lookup(base) { + return Some(Self::apply_ing(&stem)); + } + // -ing + e → base+e (e.g., "creating" → "creat" + "e" = "create") + if let Some(stem) = self.lookup(&format!("{base}e")) { + return Some(Self::apply_ing(&stem)); + } + // Double consonant: "running" → "run" + if base.len() >= 2 { + let bytes = base.as_bytes(); + if bytes[bytes.len() - 1] == bytes[bytes.len() - 2] { + if let Some(stem) = self.lookup(&base[..base.len() - 1]) { + return Some(Self::apply_ing(&stem)); + } + } + } + } + None + } + + /// Convert a single word to IPA using the full fallback chain. + fn word_to_ipa(&self, word: &str) -> String { + // Normalize curly quotes to straight apostrophes. + let normalized = word.replace('\u{2019}', "'").replace('\u{2018}', "'"); + let stripped: String = normalized + .chars() + .filter(|c| c.is_alphabetic() || *c == '\'') + .collect::() + .to_lowercase(); + + // Tier 1+2: misaki + CMUdict direct lookup + if let Some(ipa) = self.lookup(&stripped) { + return ipa; + } + + // Contractions: "don't" → "don" + "'t" + if let Some(apos_idx) = stripped.find('\'') { + let base = &stripped[..apos_idx]; + let suffix = &stripped[apos_idx..]; + if let Some(base_ipa) = self.lookup(base) { + let suffix_ipa = self.lookup(suffix).unwrap_or_else(|| match suffix { + "'ve" => "v".to_string(), + "'re" => "ɹ".to_string(), + _ => String::new(), + }); + if !suffix_ipa.is_empty() { + return format!("{base_ipa}{suffix_ipa}"); + } + } + } + + // Tier 3: morphological suffix stripping + if let Some(ipa) = self.try_morphological(&stripped) { + return ipa; + } + + // Tier 4: letter-by-letter spelling + stripped + .chars() + .filter(|c| c.is_alphabetic()) + .map(letter_ipa) + .collect() + } + + /// Convert a full text chunk to an IPA phoneme string. + fn text_to_ipa(&self, text: &str) -> String { + let mut out = String::new(); + for token in text.split_whitespace() { + if !out.is_empty() { + out.push(' '); + } + let leading: String = token + .chars() + .take_while(|c| is_passthrough_punct(*c)) + .collect(); + let trailing: String = token + .chars() + .rev() + .take_while(|c| is_passthrough_punct(*c)) + .collect::() + .chars() + .rev() + .collect(); + let word = &token[leading.len()..token.len() - trailing.len()]; + out.push_str(&leading); + if !word.is_empty() { + out.push_str(&self.word_to_ipa(word)); + } + out.push_str(&trailing); + } + out + } +} + +// ── KokoroTTS ───────────────────────────────────────────────────────────────── + +pub struct KokoroTTS { + session: Session, + vocab: HashMap, + lexicon: Lexicon, + // Retained for potential future use (e.g., hot-reloading voices by path). + #[allow(dead_code)] + model_dir: PathBuf, +} + +/// Load the Kokoro TTS engine from a model directory. +/// +/// Expects: +/// `/model.onnx` (or model_quantized.onnx — tries both) +/// `/us_gold.json` (G2P dictionary) +/// +/// CoreML execution provider is registered with auto-fallback to CPU. +/// The compiled CoreML model is cached in `/.coreml_cache/`. +pub fn load_text_to_speech(model_dir: &str) -> Result { + let model_dir_path = PathBuf::from(model_dir); + + // Try quantized model first for speed, fall back to full-precision. + let model_path = ["model_quantized.onnx", "model_q8f16.onnx", "model.onnx"] + .iter() + .map(|name| model_dir_path.join(name)) + .find(|p| p.exists()) + .ok_or_else(|| format!("no model.onnx found in {model_dir}"))?; + + // Try CoreML first (zero binary cost — macOS system framework). + // If the model has ops CoreML can't handle (common with quantized models), + // the EP registers fine but commit_from_file fails. Catch that and retry + // with CPU-only. This is the expected path for model_q8f16.onnx. + let session = { + let mut builder_with_coreml = Session::builder() + .map_err(|e| format!("session builder: {e}"))? + .with_execution_providers([ort::ep::CoreML::default() + .with_compute_units(ort::ep::coreml::ComputeUnits::All) + .with_model_format(ort::ep::coreml::ModelFormat::MLProgram) + .with_model_cache_dir(model_dir_path.join(".coreml_cache").to_string_lossy()) + .build()]) + .map_err(|e| format!("execution provider: {e}"))?; + + match builder_with_coreml.commit_from_file(&model_path) { + Ok(s) => { + eprintln!("sprout-desktop: Kokoro loaded with CoreML acceleration"); + s + } + Err(coreml_err) => { + eprintln!( + "sprout-desktop: CoreML failed for {}, falling back to CPU: {coreml_err}", + model_path.display() + ); + // Retry without any execution providers — pure CPU. + Session::builder() + .map_err(|e| format!("session builder (CPU fallback): {e}"))? + .commit_from_file(&model_path) + .map_err(|e| format!("load model {} (CPU): {e}", model_path.display()))? + } + } + }; + + let gold_path = model_dir_path.join("us_gold.json"); + let silver_path = model_dir_path.join("us_silver.json"); + let cmudict_path = model_dir_path.join("cmudict.dict"); + let lexicon = Lexicon::load(&gold_path, &silver_path, &cmudict_path)?; + + Ok(KokoroTTS { + session, + vocab: build_vocab(), + lexicon, + model_dir: model_dir_path, + }) +} + +impl KokoroTTS { + /// Synthesize `text` to 24 kHz mono PCM. + /// + /// - `_total_step` is ignored — Kokoro is not diffusion-based. + /// - `speed` controls speech rate (0.5–2.0; 1.0 = normal). + /// - `silence_secs` of silence is inserted between sentence chunks. + /// - `lang` is accepted for API compatibility but currently unused + /// (Kokoro v1.0 language is selected by voice name prefix, e.g. `af_*`). + pub fn call( + &mut self, + text: &str, + _lang: &str, + style: &VoiceStyle, + _total_step: usize, + speed: f32, + silence_secs: f32, + ) -> Result, String> { + let silence_samples = (silence_secs * SAMPLE_RATE as f32) as usize; + let silence = vec![0.0f32; silence_samples]; + + let sentences = split_sentences(text); + let mut output: Vec = Vec::new(); + + for (i, sentence) in sentences.iter().enumerate() { + let chunk_audio = self.synth_chunk(sentence, style, speed)?; + + if i > 0 && !output.is_empty() { + output.extend_from_slice(&silence); + } + output.extend(chunk_audio); + } + + Ok(output) + } + + /// Synthesize a single text chunk: G2P → tokenize → ONNX → PCM. + fn synth_chunk( + &mut self, + text: &str, + style: &VoiceStyle, + speed: f32, + ) -> Result, String> { + // G2P: text → IPA phoneme string + let ipa = self.lexicon.text_to_ipa(text); + + // Tokenize: IPA → int64 ids with BOS/EOS pad tokens + let token_ids = tokenize(&ipa, &self.vocab); + + // Style vector is indexed by phoneme count (excluding the 2 pad tokens). + // Shape expected by model: [1, 256] (kokoro-js uses [1, 256], not [1, 1, 256]). + let phoneme_count = token_ids.len() - 2; + let style_slice = style.get(phoneme_count); + + // Build ONNX input tensors. + let seq_len = token_ids.len(); + let input_ids_arr = Array2::from_shape_vec((1, seq_len), token_ids) + .map_err(|e| format!("input_ids shape: {e}"))?; + let input_ids_val = + Value::from_array(input_ids_arr).map_err(|e| format!("input_ids Value: {e}"))?; + + // Style: [1, 256]. Research notes [1, 1, 256] but kokoro-js uses [1, 256]. + let style_arr = Array2::from_shape_vec((1, 256), style_slice.to_vec()) + .map_err(|e| format!("style shape: {e}"))?; + let style_val = Value::from_array(style_arr).map_err(|e| format!("style Value: {e}"))?; + + let speed_arr = Array1::from_vec(vec![speed]); + let speed_val = Value::from_array(speed_arr).map_err(|e| format!("speed Value: {e}"))?; + + // Run inference. Output[0] = waveform float32[1, N_samples]. + let outputs = self + .session + .run(ort::inputs! { + "input_ids" => &input_ids_val, + "style" => &style_val, + "speed" => &speed_val, + }) + .map_err(|e| format!("onnx run: {e}"))?; + + let (_, waveform) = outputs[0] + .try_extract_tensor::() + .map_err(|e| format!("extract waveform: {e}"))?; + + Ok(waveform.to_vec()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // ── Tokenizer ───────────────────────────────────────────────────────── + + #[test] + fn tokenize_empty_produces_bos_eos() { + let vocab = build_vocab(); + let ids = tokenize("", &vocab); + assert_eq!(ids, vec![0, 0]); // BOS + EOS only + } + + #[test] + fn tokenize_known_chars() { + let vocab = build_vocab(); + let ids = tokenize("a", &vocab); + // 'a' maps to 43 in the vocab + assert_eq!(ids, vec![0, 43, 0]); + } + + #[test] + fn tokenize_unknown_chars_dropped() { + let vocab = build_vocab(); + let ids = tokenize("🎉", &vocab); + // Emoji not in vocab — should be dropped, leaving only BOS+EOS + assert_eq!(ids, vec![0, 0]); + } + + #[test] + fn tokenize_respects_max_length() { + let vocab = build_vocab(); + let long_input: String = "a".repeat(600); // exceeds MAX_PHONEME_TOKENS (510) + let ids = tokenize(&long_input, &vocab); + // Should be clamped: BOS + 510 tokens + EOS = 512 + assert_eq!(ids.len(), 512); + assert_eq!(ids[0], 0); // BOS + assert_eq!(*ids.last().unwrap(), 0); // EOS + } + + // ── ARPAbet conversion ──────────────────────────────────────────────── + + #[test] + fn arpabet_simple_word() { + // "HH AH0 L OW1" = hello + let ipa = arpabet_line_to_ipa("HH AH0 L OW1"); + assert_eq!(ipa, "həlˈO"); + } + + #[test] + fn arpabet_stress_markers() { + // Primary stress before vowel, secondary stress before vowel + let ipa = arpabet_line_to_ipa("K R IY0 EY1 T"); + // IY0 = unstressed 'i', EY1 = primary 'A' + assert!(ipa.contains('ˈ'), "should contain primary stress: {ipa}"); + } + + #[test] + fn arpabet_schwa() { + // AH0 should produce schwa (ə), not ʌ + let ipa = arpabet_line_to_ipa("AH0"); + assert_eq!(ipa, "ə"); + } + + #[test] + fn arpabet_unstressed_er() { + // ER0 should produce əɹ + let ipa = arpabet_line_to_ipa("ER0"); + assert_eq!(ipa, "əɹ"); + } + + // ── Letter IPA ──────────────────────────────────────────────────────── + + #[test] + fn letter_ipa_covers_alphabet() { + for c in 'a'..='z' { + let ipa = letter_ipa(c); + assert!(!ipa.is_empty(), "letter_ipa('{c}') returned empty"); + } + } + + #[test] + fn letter_ipa_non_alpha_empty() { + assert_eq!(letter_ipa('1'), ""); + assert_eq!(letter_ipa('!'), ""); + } + + // ── Punctuation passthrough ─────────────────────────────────────────── + + #[test] + fn passthrough_punct_includes_expected() { + assert!(is_passthrough_punct('.')); + assert!(is_passthrough_punct('!')); + assert!(is_passthrough_punct('?')); + assert!(is_passthrough_punct(' ')); + assert!(is_passthrough_punct(',')); + } + + #[test] + fn passthrough_punct_excludes_alpha() { + assert!(!is_passthrough_punct('a')); + assert!(!is_passthrough_punct('Z')); + } + + // ── VoiceStyle ──────────────────────────────────────────────────────── + + #[test] + fn voice_style_get_clamps_to_last_row() { + // 2 rows of 256 floats + let data: Vec = (0..512).map(|i| i as f32).collect(); + let style = VoiceStyle { data }; + // Row 0 + assert_eq!(style.get(0)[0], 0.0); + // Row 1 + assert_eq!(style.get(1)[0], 256.0); + // Row 999 should clamp to row 1 (last available) + assert_eq!(style.get(999)[0], 256.0); + } + + #[test] + fn load_voice_style_rejects_too_small() { + use std::io::Write; + let dir = std::env::temp_dir().join("kokoro_test_small"); + let _ = std::fs::create_dir_all(&dir); + let path = dir.join("tiny.bin"); + let mut f = std::fs::File::create(&path).unwrap(); + // Write only 100 floats (need at least 256) + for i in 0..100u32 { + f.write_all(&(i as f32).to_le_bytes()).unwrap(); + } + drop(f); + let result = load_voice_style(&path); + assert!(result.is_err(), "should reject file with < 256 floats"); + assert!(result.unwrap_err().contains("too small")); + let _ = std::fs::remove_dir_all(&dir); + } + + // ── Suffix rules ────────────────────────────────────────────────────── + + #[test] + fn apply_s_voiceless() { + // After voiceless consonants: +s + assert!(Lexicon::apply_s("kæt").ends_with('s')); + } + + #[test] + fn apply_s_sibilant() { + // After sibilants: +ᵻz + assert!(Lexicon::apply_s("bʌz").ends_with("ᵻz")); + } + + #[test] + fn apply_s_voiced() { + // After voiced consonants: +z + assert!(Lexicon::apply_s("dɔɡ").ends_with('z')); + } + + #[test] + fn apply_ed_voiceless() { + // After voiceless: +t + assert!(Lexicon::apply_ed("wɔk").ends_with('t')); + } + + #[test] + fn apply_ed_d_ending() { + // After d: +ᵻd + assert!(Lexicon::apply_ed("æd").ends_with("ᵻd")); + } + + #[test] + fn apply_ing_basic() { + assert!(Lexicon::apply_ing("rʌn").ends_with("ɪŋ")); + } +} diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index cee0843631..dc5cebf043 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -7,12 +7,30 @@ //! creator → end_huddle → archive ephemeral channel, clear state //! //! HuddleState is stored in AppState and serialized for get_huddle_state. +//! +//! ## Synchronization Protocol +//! +//! `HuddleState` lives behind a single `Mutex` in `AppState`. Rules: +//! +//! 1. **Never hold the outer lock across `.await`** — acquire, read/write, release. +//! 2. **Pipeline construction happens outside the lock** — the `stt_starting` / +//! `tts_starting` sentinels prevent TOCTOU races during the ~200ms window. +//! 3. **`agent_pubkeys` has its own inner `Arc`** — the transcription task +//! clones the `Arc` and reads at post time without the outer lock. +//! 4. **Atomics for cross-thread signaling** — `tts_active`, `tts_cancel`, +//! `ptt_active`, `session_generation` are shared with pipeline worker threads. +//! 5. **Pipeline teardown extracts handles before dropping** — `teardown_huddle` +//! takes `stt_pipeline`/`tts_pipeline` out of the lock, then calls `shutdown()` +//! and drops them outside the lock (thread joins can block ~200ms). pub mod agents; +pub mod kokoro; pub mod models; +pub mod pipeline; pub mod preprocessing; +pub mod relay_api; +pub mod state; pub mod stt; -pub mod supertonic; pub mod tts; // ── Shared utilities ────────────────────────────────────────────────────────── @@ -35,524 +53,24 @@ pub(super) fn drain_until_shutdown( } } -use reqwest::Method; -use serde::{Deserialize, Serialize}; -use std::sync::{ - atomic::{AtomicBool, AtomicU64, Ordering}, - Arc, Mutex, -}; -use tauri::State; -use uuid::Uuid; - -use nostr::JsonUtil; - -use crate::{ - app_state::AppState, - events, - relay::{api_path, build_authed_request, send_json_request, submit_event}, -}; - -// ── State types ─────────────────────────────────────────────────────────────── - -/// Voice input mode: push-to-talk (PTT) or voice-activity detection (VAD). -/// -/// PTT (default): mic is gated by a global shortcut (Ctrl+Space). Pressing the -/// key sets `ptt_active` and immediately cancels any playing TTS. Releasing -/// the key (after a 200 ms delay) stops mic capture and flushes the utterance. -/// -/// VAD: the earshot VAD runs continuously and speech is accumulated whenever -/// the probability exceeds the threshold. Barge-in is enabled in this mode. -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "snake_case")] -pub enum VoiceInputMode { - #[default] - PushToTalk, - VoiceActivity, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "snake_case")] -pub enum HuddlePhase { - Idle, - Creating, - Connecting, - Connected, // Backend ready, waiting for frontend media confirmation. - Active, - Leaving, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct HuddleState { - pub phase: HuddlePhase, - pub parent_channel_id: Option, - pub ephemeral_channel_id: Option, - /// Skipped from serialization — the frontend gets the token from - /// start_huddle/join_huddle return values. Exposing the LiveKit JWT - /// on every 2-second get_huddle_state poll is unnecessary attack surface. - #[serde(skip)] - pub livekit_token: Option, - /// Skipped from serialization — same rationale as livekit_token. - #[serde(skip)] - pub livekit_url: Option, - pub livekit_room: Option, - /// Participant pubkey hex strings (all members, including humans). - pub participants: Vec, - /// Agent pubkeys only — used as p-tags on transcribed messages. - /// - /// Stored as `Arc>>` so the transcription task can clone - /// the `Arc` and read the current list at post time without holding the - /// outer `HuddleState` lock across an await point. - /// - /// Populated from `member_pubkeys` in `start_huddle` (the UI sends agent - /// pubkeys specifically). Joiners don't add agents — they were already - /// added by the creator. Serialized as a plain `Vec` for the - /// frontend via the custom `Serialize`/`Deserialize` impls below. - #[serde( - serialize_with = "serialize_agent_pubkeys", - deserialize_with = "deserialize_agent_pubkeys" - )] - pub agent_pubkeys: Arc>>, - /// Active STT pipeline — not serialized, not cloned. - #[serde(skip)] - pub stt_pipeline: Option>, - /// Active TTS pipeline — not serialized, not cloned. - #[serde(skip)] - pub tts_pipeline: Option>, - /// Whether this client created the huddle (vs. joined it). - /// Used to enforce that only the creator can end/archive the huddle. - pub is_creator: bool, - /// Whether TTS output is enabled (user-toggled). - pub tts_enabled: bool, - /// Shared flag: true while TTS is playing audio. - /// Shared with the STT pipeline for barge-in / echo gating. - #[serde(skip)] - pub tts_active: Arc, - /// Shared barge-in cancel flag. Set by STT when it detects speech during TTS. - /// Read by TTS to stop playback. Lives in HuddleState so it survives pipeline - /// restarts — both STT and TTS reference the same flag for the entire huddle. - #[serde(skip)] - pub tts_cancel: Arc, - /// Timestamp of the last agent pubkey refresh from the relay. - /// Used to throttle the refresh in check_pipeline_hotstart to every 15 s. - #[serde(skip)] - pub last_agent_refresh: Option, - /// Session generation — incremented on every teardown. The transcription - /// task captures this at spawn time and checks before each POST. If the - /// generation has changed, the task silently drops the transcript. - #[serde(skip)] - pub session_generation: Arc, - /// Voice input mode: push-to-talk or voice-activity detection. - pub voice_input_mode: VoiceInputMode, - /// True while the PTT key is held (+ 200 ms release delay). - /// Shared with the STT pipeline for mic gating. - #[serde(skip)] - pub ptt_active: Arc, -} - -fn serialize_agent_pubkeys(v: &Arc>>, s: S) -> Result -where - S: serde::Serializer, -{ - use serde::ser::SerializeSeq; - let guard = v.lock().unwrap_or_else(|e| e.into_inner()); - let mut seq = s.serialize_seq(Some(guard.len()))?; - for item in guard.iter() { - seq.serialize_element(item)?; - } - seq.end() -} - -fn deserialize_agent_pubkeys<'de, D>(d: D) -> Result>>, D::Error> -where - D: serde::Deserializer<'de>, -{ - let v: Vec = serde::Deserialize::deserialize(d)?; - Ok(Arc::new(Mutex::new(v))) -} - -impl Clone for HuddleState { - fn clone(&self) -> Self { - let agent_pubkeys_snapshot = self - .agent_pubkeys - .lock() - .unwrap_or_else(|e| e.into_inner()) - .clone(); - Self { - phase: self.phase.clone(), - parent_channel_id: self.parent_channel_id.clone(), - ephemeral_channel_id: self.ephemeral_channel_id.clone(), - livekit_token: self.livekit_token.clone(), - livekit_url: self.livekit_url.clone(), - livekit_room: self.livekit_room.clone(), - participants: self.participants.clone(), - agent_pubkeys: Arc::new(Mutex::new(agent_pubkeys_snapshot)), - stt_pipeline: None, // Never clone the pipeline handle. - tts_pipeline: None, // Never clone the pipeline handle. - is_creator: self.is_creator, - tts_enabled: self.tts_enabled, - tts_active: Arc::clone(&self.tts_active), - tts_cancel: Arc::clone(&self.tts_cancel), - last_agent_refresh: self.last_agent_refresh, - session_generation: Arc::clone(&self.session_generation), - voice_input_mode: self.voice_input_mode.clone(), - ptt_active: Arc::clone(&self.ptt_active), - } - } -} - -impl Default for HuddleState { - fn default() -> Self { - Self { - phase: HuddlePhase::Idle, - parent_channel_id: None, - ephemeral_channel_id: None, - livekit_token: None, - livekit_url: None, - livekit_room: None, - participants: Vec::new(), - agent_pubkeys: Arc::new(Mutex::new(Vec::new())), - stt_pipeline: None, - tts_pipeline: None, - is_creator: false, - tts_enabled: true, - tts_active: Arc::new(AtomicBool::new(false)), - tts_cancel: Arc::new(AtomicBool::new(false)), - last_agent_refresh: None, - session_generation: Arc::new(AtomicU64::new(0)), - voice_input_mode: VoiceInputMode::default(), - ptt_active: Arc::new(AtomicBool::new(false)), - } - } -} +// ── Re-exports ──────────────────────────────────────────────────────────────── -impl HuddleState { - /// Reset to default state while preserving the session generation counter. - /// Used by start_huddle rollback, join_huddle rollback, and teardown_huddle - /// to invalidate in-flight transcription tasks without losing the generation. - fn reset_preserving_generation(&mut self) { - let gen = Arc::clone(&self.session_generation); - *self = Self::default(); - self.session_generation = gen; - } -} +pub use state::{HuddleJoinInfo, HuddlePhase, HuddleState, VoiceInputMode}; -// ── Response types ──────────────────────────────────────────────────────────── - -/// Returned by start_huddle and join_huddle. -#[derive(Debug, Serialize, Deserialize)] -pub struct HuddleJoinInfo { - pub ephemeral_channel_id: String, - pub livekit_token: String, - pub livekit_url: String, - pub livekit_room: String, -} - -/// Raw response from `POST /api/huddles/{channel_id}/token`. -#[derive(Debug, Deserialize)] -struct LiveKitTokenResponse { - pub token: String, - pub url: String, - pub room: String, -} - -// ── Helpers ─────────────────────────────────────────────────────────────────── - -/// Maximum number of agents that can be invited to a single huddle. -const MAX_HUDDLE_AGENTS: usize = 20; - -/// Validate that a string looks like a Nostr pubkey hex (64 hex chars). -fn validate_pubkey_hex(pubkey: &str) -> Result<(), String> { - if pubkey.len() != 64 || !pubkey.chars().all(|c| c.is_ascii_hexdigit()) { - let preview: String = pubkey.chars().take(16).collect(); - return Err(format!("invalid pubkey hex: {preview}")); - } - Ok(()) -} - -fn parse_channel_uuid(channel_id: &str) -> Result { - Uuid::parse_str(channel_id).map_err(|_| format!("invalid channel UUID: {channel_id}")) -} - -/// Fetch a LiveKit token from the relay for the given channel. -async fn fetch_livekit_token( - channel_id: &str, - state: &AppState, -) -> Result { - let path = api_path(&["huddles", channel_id, "token"]); - let request = build_authed_request(&state.http_client, Method::POST, &path, state)?; - send_json_request(request).await -} - -/// Fetch channel members from the relay. If `role_filter` is Some, only return -/// members with that role (e.g., "bot" for agents). Returns all members if None. -async fn fetch_channel_members( - channel_id: &str, - role_filter: Option<&str>, - state: &AppState, -) -> Result, String> { - #[derive(Deserialize)] - struct Member { - pubkey: String, - role: Option, - } - #[derive(Deserialize)] - struct MembersResponse { - members: Vec, - } - - let path = api_path(&["channels", channel_id, "members"]); - let request = build_authed_request(&state.http_client, Method::GET, &path, state)?; - let resp: MembersResponse = send_json_request(request).await.map_err(|e| { - eprintln!("sprout-desktop: fetch channel members failed: {e}"); - e - })?; - - Ok(resp - .members - .into_iter() - .filter(|m| role_filter.map_or(true, |r| m.role.as_deref() == Some(r))) - .map(|m| m.pubkey) - .collect()) -} - -/// Count human (non-bot) members remaining in a channel. -/// Used by leave_huddle to detect "last human left" and auto-end. -async fn count_human_members(channel_id: &str, state: &AppState) -> Result { - #[derive(Deserialize)] - struct Member { - role: Option, - } - #[derive(Deserialize)] - struct MembersResponse { - members: Vec, - } - - let path = api_path(&["channels", channel_id, "members"]); - let request = build_authed_request(&state.http_client, Method::GET, &path, state)?; - let resp: MembersResponse = send_json_request(request).await?; - - Ok(resp - .members - .iter() - .filter(|m| m.role.as_deref() != Some("bot")) - .count()) -} - -/// Common setup after a huddle connection is established (both start and join). -/// Hydrates participants from relay, ensures model downloads, starts pipelines. -async fn post_connect_setup(state: &AppState, ephemeral_channel_id: &str) -> Result<(), String> { - // Hydrate agent pubkeys from relay (authoritative — overrides local guess). - if let Ok(agents) = fetch_channel_members(ephemeral_channel_id, Some("bot"), state).await { - let hs = state.huddle()?; - *hs.agent_pubkeys.lock().unwrap_or_else(|e| e.into_inner()) = agents; - } - - // Hydrate participants from relay (authoritative state). - if let Ok(all_members) = fetch_channel_members(ephemeral_channel_id, None, state).await { - if !all_members.is_empty() { - let mut hs = state.huddle()?; - hs.participants = all_members; - } - } - - // Ensure voice models are downloading (idempotent). - if let Some(mgr) = models::global_model_manager() { - mgr.start_moonshine_download(state.http_client.clone()); - mgr.start_supertonic_download(state.http_client.clone()); - } - - // Start pipelines: TTS first (so STT can capture tts_cancel for barge-in). - if let Err(e) = maybe_start_tts_pipeline(state).await { - eprintln!("sprout-desktop: TTS pipeline failed to start: {e}"); - } - if let Err(e) = maybe_start_stt_pipeline(state, ephemeral_channel_id).await { - eprintln!("sprout-desktop: STT pipeline failed to start: {e}"); - } - - Ok(()) -} +// ── Imports ─────────────────────────────────────────────────────────────────── -/// Attempt to start the STT pipeline if models are present. -/// -/// Returns `Ok(true)` if the pipeline was started, `Ok(false)` if models are -/// not ready (voice-only mode), or `Err` on a real failure. -/// -/// Creates the shared `tts_active` flag and passes it to the STT pipeline -/// for barge-in / echo gating. The same flag is later passed to the TTS -/// pipeline so it can signal when audio is playing. -async fn maybe_start_stt_pipeline( - state: &AppState, - ephemeral_channel_id: &str, -) -> Result { - if !models::is_moonshine_ready() { - return Ok(false); // Models not downloaded yet — voice-only mode. - } - let model_dir = - models::moonshine_model_dir().ok_or_else(|| "Moonshine model directory not found")?; - - let channel_uuid = parse_channel_uuid(ephemeral_channel_id)?; - - // Grab shared flags, agent pubkeys, and session generation from HuddleState. - // If replacing an existing pipeline, bump generation first so the old - // transcription task's next POST sees a stale generation and exits. - let (tts_active, tts_cancel, agent_pubkeys_arc, session_gen, ptt_active_for_stt) = { - let hs = state.huddle()?; - // Invalidate any existing transcription task before replacing the pipeline. - if hs.stt_pipeline.is_some() { - hs.session_generation.fetch_add(1, Ordering::Release); - } - if let Some(ref old) = hs.stt_pipeline { - old.shutdown(); - } - let ptt = if hs.voice_input_mode == VoiceInputMode::PushToTalk { - Some(Arc::clone(&hs.ptt_active)) - } else { - None - }; - ( - Arc::clone(&hs.tts_active), - Some(Arc::clone(&hs.tts_cancel)), - Arc::clone(&hs.agent_pubkeys), - Arc::clone(&hs.session_generation), - ptt, - ) - }; - - let (pipeline, text_rx) = - stt::SttPipeline::new(model_dir, tts_active, tts_cancel, ptt_active_for_stt)?; - let pipeline = Arc::new(pipeline); - - { - let mut hs = state.huddle()?; - hs.stt_pipeline = Some(Arc::clone(&pipeline)); - } - - spawn_transcription_task(text_rx, channel_uuid, agent_pubkeys_arc, session_gen, state); - Ok(true) -} - -/// Attempt to start the TTS pipeline if Supertonic models are present and TTS is enabled. -/// -/// Returns `Ok(true)` if the pipeline was started, `Ok(false)` if preconditions -/// aren't met (model not ready, pipeline exists, TTS disabled), or `Err` on failure. -async fn maybe_start_tts_pipeline(state: &AppState) -> Result { - if !models::is_supertonic_ready() { - return Ok(false); // Supertonic not downloaded yet — TTS unavailable. - } - - // Don't create a duplicate pipeline if one is already running. - { - let hs = state.huddle()?; - if hs.tts_pipeline.is_some() { - return Ok(false); - } - } - - let model_dir = match models::supertonic_model_dir() { - Some(d) => d, - None => return Ok(false), - }; - - let (tts_active, tts_enabled, tts_cancel) = { - let hs = state.huddle()?; - ( - Arc::clone(&hs.tts_active), - hs.tts_enabled, - Arc::clone(&hs.tts_cancel), - ) - }; - - if !tts_enabled { - return Ok(false); - } - - let pipeline = Arc::new(tts::TtsPipeline::new(model_dir, tts_active, tts_cancel)?); - - { - let mut hs = state.huddle()?; - // Re-check: another call may have created a pipeline while we were building ours. - if hs.tts_pipeline.is_some() { - return Ok(false); // The existing one wins. - } - hs.tts_pipeline = Some(pipeline); - } - - Ok(true) -} - -/// Spawn a tokio task that reads text_rx and posts kind:9 events. -/// -/// Fix 1: `agent_pubkeys_arc` is an `Arc>>` cloned from -/// `HuddleState` — the task reads it at post time so p-tags are always -/// current, not a stale snapshot. -/// Fix 3: no `.unwrap()` on mutex — poisoned locks are recovered gracefully. -/// Fix 4: `text_rx` is a `tokio::sync::mpsc::Receiver` — fully async `.recv().await` -/// never blocks a Tokio worker thread (unlike std `recv_timeout`). -fn spawn_transcription_task( - mut text_rx: tokio::sync::mpsc::Receiver, - channel_uuid: Uuid, - agent_pubkeys_arc: Arc>>, - session_generation: Arc, - state: &AppState, -) { - // Capture the current generation at spawn time. - let spawned_gen = session_generation.load(Ordering::Acquire); - - let http_client = state.http_client.clone(); - let keys = match state.keys.lock() { - Ok(k) => k.clone(), - Err(_) => return, - }; - let configured_api_token = state.configured_api_token.clone(); - - tauri::async_runtime::spawn(async move { - // recv().await yields (not blocks) until text arrives or sender is dropped. - // When the STT worker exits and drops its Sender, recv() returns None → loop ends. - while let Some(t) = text_rx.recv().await { - if t.is_empty() { - continue; - } - - // Session guard: if the generation has changed, this task is stale. - // Drop the transcript silently — the huddle has ended or been replaced. - if session_generation.load(Ordering::Acquire) != spawned_gen { - break; // Exit the loop entirely — no more posts from this task. - } +use std::sync::{atomic::Ordering, Arc}; +use tauri::State; +use uuid::Uuid; - // Fix 1: read current agent pubkeys at post time. - let agent_pubkeys: Vec = agent_pubkeys_arc - .lock() - .unwrap_or_else(|e| e.into_inner()) - .clone(); +use crate::{app_state::AppState, events, relay::submit_event}; - let p_tags: Vec<&str> = agent_pubkeys.iter().map(|s| s.as_str()).collect(); - let builder = match events::build_message(channel_uuid, &t, None, &p_tags, &[]) { - Ok(b) => b, - Err(e) => { - eprintln!("sprout-desktop: STT build_message: {e}"); - continue; - } - }; - let event = match builder.sign_with_keys(&keys) { - Ok(e) => e, - Err(e) => { - eprintln!("sprout-desktop: STT sign event: {e}"); - continue; - } - }; - let event_json = event.as_json(); - let api_token_ref = configured_api_token.as_deref(); - let pubkey_hex = keys.public_key().to_hex(); - - if let Err(e) = - crate::events::post_event_raw(&http_client, api_token_ref, &pubkey_hex, event_json) - .await - { - eprintln!("sprout-desktop: STT kind:9 post failed: {e}"); - } - } - }); -} +use pipeline::{maybe_start_stt_pipeline, maybe_start_tts_pipeline, post_connect_setup}; +use relay_api::{ + count_human_members, fetch_channel_members, fetch_livekit_token, parse_channel_uuid, + validate_pubkey_hex, MAX_HUDDLE_AGENTS, +}; +use state::LiveKitTokenResponse; // ── Tauri commands ──────────────────────────────────────────────────────────── @@ -807,6 +325,9 @@ pub async fn join_huddle( } // All steps wrapped so we can roll back on ANY failure after phase transition. + // Tracks whether THIS attempt added membership (vs "already a member"). + // On rollback, only leave the channel if we actually added ourselves. + let mut membership_added = false; let result: Result<(LiveKitTokenResponse, String), String> = async { // 0. Add the joining human to the ephemeral channel. let own_pubkey = state @@ -816,16 +337,25 @@ pub async fn join_huddle( .map_err(|_| "keys unavailable".to_string())?; let eph_uuid = parse_channel_uuid(&ephemeral_channel_id)?; let add_self = events::build_add_member(eph_uuid, &own_pubkey, None)?; - if let Err(e) = submit_event(add_self, &state).await { - // Idempotent: "already a member" is fine (rejoining after disconnect). - // Any other error means we can't post STT transcripts — fail the join. - let is_already_member = e.to_lowercase().contains("already"); - if !is_already_member { - eprintln!("sprout-desktop: join_huddle add self to ephemeral channel failed: {e}"); - return Err(format!("failed to join ephemeral channel: {e}")); + match submit_event(add_self, &state).await { + Ok(_) => { + membership_added = true; + } // Newly added. + Err(e) => { + // Idempotent: "already a member" is expected on rejoin. + let is_idempotent = { + let lower = e.to_lowercase(); + lower.contains("already a member") || lower.contains("already_member") + }; + if !is_idempotent { + eprintln!( + "sprout-desktop: join_huddle add self to ephemeral channel failed: {e}" + ); + return Err(format!("failed to join ephemeral channel: {e}")); + } + // Already a member — don't leave on rollback. } - // Already a member — continue normally. - } + }; // 1. Fetch LiveKit token. let lk = fetch_livekit_token(&ephemeral_channel_id, &state).await?; @@ -871,7 +401,19 @@ pub async fn join_huddle( }) } Err(e) => { - // Rollback: reset state to Idle so the user can retry. + // Rollback: remove self from ephemeral channel to avoid orphaned + // membership (e.g. token fetch failed after membership succeeded). + // Only leave if THIS attempt added us — don't kick a pre-existing member. + if membership_added { + if let Ok(eph_uuid) = parse_channel_uuid(&ephemeral_channel_id) { + if let Ok(leave_builder) = events::build_leave(eph_uuid) { + if let Err(le) = submit_event(leave_builder, &state).await { + eprintln!("sprout-desktop: join_huddle rollback leave failed: {le}"); + } + } + } + } + // Reset state to Idle so the user can retry. if let Ok(mut hs) = state.huddle_state.lock() { hs.reset_preserving_generation(); } @@ -998,15 +540,19 @@ pub async fn leave_huddle(state: State<'_, AppState>) -> Result<(), String> { /// 3. Shut down the STT pipeline (Fix 5). /// 4. Clear local huddle state. #[tauri::command] -pub async fn end_huddle(state: State<'_, AppState>) -> Result<(), String> { +pub async fn end_huddle(force: Option, state: State<'_, AppState>) -> Result<(), String> { let (parent_channel_id, ephemeral_channel_id) = { let mut hs = state.huddle()?; if hs.phase == HuddlePhase::Idle { return Ok(()); // Nothing to end. } - // Any participant can end the huddle — if the creator disconnects - // ungracefully (crash, network loss), other participants need a way - // to archive the ephemeral channel and emit huddle_ended. + // Only the creator can end the huddle for everyone. Non-creators + // should use leave_huddle (which auto-ends if they're the last human). + // The `force` flag allows recovery when the creator has disconnected + // ungracefully — the UI should gate this behind a confirmation dialog. + if !hs.is_creator && !force.unwrap_or(false) { + return Err("only the huddle creator can end it — use leave_huddle instead".into()); + } hs.phase = HuddlePhase::Leaving; ( hs.parent_channel_id.clone().unwrap_or_default(), @@ -1159,12 +705,12 @@ pub async fn check_pipeline_hotstart(state: State<'_, AppState>) -> Result<(), S let moonshine_ready = models::global_model_manager() .map(|m| m.take_moonshine_ready()) .unwrap_or(false); - let supertonic_ready = models::global_model_manager() - .map(|m| m.take_supertonic_ready()) + let kokoro_ready = models::global_model_manager() + .map(|m| m.take_kokoro_ready()) .unwrap_or(false); // Start TTS first (so STT can capture tts_cancel). - if !has_tts && (supertonic_ready || models::is_supertonic_ready()) { + if !has_tts && (kokoro_ready || models::is_kokoro_ready()) { if let Err(e) = maybe_start_tts_pipeline(&state).await { eprintln!("sprout-desktop: TTS hotstart failed: {e}"); } @@ -1249,7 +795,7 @@ pub async fn start_stt_pipeline(state: State<'_, AppState>) -> Result<(), String } } -/// Trigger a background download of voice models (Moonshine STT + Supertonic TTS). +/// Trigger a background download of voice models (Moonshine STT + Kokoro TTS). /// /// Returns immediately — downloads run in tokio background tasks. /// Poll `get_model_status` to track progress. @@ -1259,7 +805,7 @@ pub async fn download_voice_models(state: State<'_, AppState>) -> Result<(), Str let manager = models::global_model_manager() .ok_or("model manager unavailable (home directory could not be resolved)")?; manager.start_moonshine_download(state.http_client.clone()); - manager.start_supertonic_download(state.http_client.clone()); + manager.start_kokoro_download(state.http_client.clone()); Ok(()) } @@ -1270,14 +816,15 @@ pub fn get_model_status(_state: State<'_, AppState>) -> Result Option { - self.supertonic.dir_if_ready(&self.models_dir) + /// Path to the Kokoro model directory, or `None` if not ready. + pub fn kokoro_model_dir(&self) -> Option { + self.kokoro.dir_if_ready(&self.models_dir) } - /// `true` if all Supertonic files are present and the manifest version matches. - pub fn is_supertonic_ready(&self) -> bool { - self.supertonic.is_ready(&self.models_dir) + /// `true` if all Kokoro files are present and the manifest version matches. + pub fn is_kokoro_ready(&self) -> bool { + self.kokoro.is_ready(&self.models_dir) } - /// Current Supertonic download status. - pub fn supertonic_status(&self) -> ModelStatus { - self.supertonic.status() + /// Current Kokoro download status. + pub fn kokoro_status(&self) -> ModelStatus { + self.kokoro.status() } - /// Returns `true` once when Supertonic just became ready. Resets the flag. - pub fn take_supertonic_ready(&self) -> bool { - self.supertonic.take_ready() + /// Returns `true` once when Kokoro just became ready. Resets the flag. + pub fn take_kokoro_ready(&self) -> bool { + self.kokoro.take_ready() } // ── Download triggers ───────────────────────────────────────────────────── @@ -508,14 +525,14 @@ impl ModelManager { ); } - /// Start a background Supertonic download (~253 MB). No-op if already ready or downloading. - pub fn start_supertonic_download(&self, http_client: reqwest::Client) { + /// Start a background Kokoro download (~87 MB). No-op if already ready or downloading. + pub fn start_kokoro_download(&self, http_client: reqwest::Client) { let manager = self.clone(); - self.supertonic.start_download( + self.kokoro.start_download( &self.models_dir, http_client, - "supertonic", - move |client| async move { manager.download_supertonic_model(client).await }, + "kokoro", + move |client| async move { manager.download_kokoro_model(client).await }, ); } @@ -600,46 +617,53 @@ impl ModelManager { Ok(()) } - /// Download and verify the Supertonic TTS model files from HuggingFace. + /// Download and verify the Kokoro TTS model files from HuggingFace and GitHub. + /// + /// Downloads files into `~/.sprout/models/kokoro/`: + /// - `model.onnx` — Kokoro-82M mixed-precision ONNX (86 MB) + /// - `af_heart.bin` — best-quality American English voice embedding (510 KB) + /// - `us_gold.json` — Misaki G2P lexicon, pinned to commit fba1236 (3 MB) /// - /// Downloads 7 files into `~/.sprout/models/supertonic/`. /// Files are written to a temp directory first, then moved atomically. - async fn download_supertonic_model(&self, http_client: reqwest::Client) -> Result<(), String> { + async fn download_kokoro_model(&self, http_client: reqwest::Client) -> Result<(), String> { tokio::fs::create_dir_all(&self.models_dir) .await .map_err(|e| format!("create models dir: {e}"))?; - let temp_dir = self.models_dir.join("supertonic.tmp"); + let temp_dir = self.models_dir.join("kokoro.tmp"); fresh_temp_dir(&temp_dir).await?; - // (url_suffix, local_filename) + // (url, local_filename) let downloads: &[(&str, &str)] = &[ - ("onnx/duration_predictor.onnx", "duration_predictor.onnx"), - ("onnx/text_encoder.onnx", "text_encoder.onnx"), - ("onnx/vector_estimator.onnx", "vector_estimator.onnx"), - ("onnx/vocoder.onnx", "vocoder.onnx"), - ("onnx/tts.json", "tts.json"), - ("onnx/unicode_indexer.json", "unicode_indexer.json"), - ("voice_styles/F1.json", "F1.json"), + ( + &format!("{KOKORO_HF_BASE}/onnx/model_q8f16.onnx"), + "model.onnx", + ), + ( + &format!("{KOKORO_HF_BASE}/voices/af_heart.bin"), + "af_heart.bin", + ), + (KOKORO_LEXICON_GOLD_URL, "us_gold.json"), + (KOKORO_LEXICON_SILVER_URL, "us_silver.json"), + (KOKORO_CMUDICT_URL, "cmudict.dict"), ]; let total_files = downloads.len() as u32; - for (i, (url_suffix, filename)) in downloads.iter().enumerate() { - let url = format!("{SUPERTONIC_HF_BASE}/{url_suffix}"); - eprintln!("sprout-desktop: downloading Supertonic {filename} from {url}"); + for (i, (url, filename)) in downloads.iter().enumerate() { + eprintln!("sprout-desktop: downloading Kokoro {filename} from {url}"); - let response = fetch_url(&http_client, &url, filename).await.map_err(|e| { + let response = fetch_url(&http_client, url, filename).await.map_err(|e| { let _ = std::fs::remove_dir_all(&temp_dir); e })?; let dest = temp_dir.join(filename); - let slot = self.supertonic.clone(); + let slot = self.kokoro.clone(); let file_index = i as u32; let bytes = download_file( response, &dest, - MAX_SUPERTONIC_FILE_BYTES, + MAX_KOKORO_FILE_BYTES, filename, |downloaded, content_length| { if let Some(total) = content_length { @@ -663,31 +687,29 @@ impl ModelManager { eprintln!("sprout-desktop: downloaded {bytes} bytes ({filename}), wrote to disk"); // Verify file integrity against pinned hash. - if let Some(&(_, expected)) = - SUPERTONIC_FILE_HASHES.iter().find(|(n, _)| *n == *filename) - { + if let Some(&(_, expected)) = KOKORO_FILE_HASHES.iter().find(|(n, _)| *n == *filename) { let actual = sha256_file(&dest).await?; if actual != expected { let _ = tokio::fs::remove_dir_all(&temp_dir).await; return Err(format!( - "Supertonic {filename} integrity check failed: expected {expected}, got {actual}" + "Kokoro {filename} integrity check failed: expected {expected}, got {actual}" )); } } // Ensure progress reflects file completion even without content-length. let pct = (((i as u32 + 1) * 89) / total_files).min(89) as u8; - self.supertonic.set_status(ModelStatus::Downloading { + self.kokoro.set_status(ModelStatus::Downloading { progress_percent: pct, }); } - self.supertonic.set_status(ModelStatus::Downloading { + self.kokoro.set_status(ModelStatus::Downloading { progress_percent: 90, }); if let Err(e) = self - .supertonic + .kokoro .verify_and_install(&self.models_dir, &temp_dir, None) .await { @@ -696,8 +718,8 @@ impl ModelManager { } eprintln!( - "sprout-desktop: Supertonic model ready at {}", - self.supertonic.model_dir(&self.models_dir).display() + "sprout-desktop: Kokoro model ready at {}", + self.kokoro.model_dir(&self.models_dir).display() ); Ok(()) } @@ -726,14 +748,14 @@ pub fn is_moonshine_ready() -> bool { .unwrap_or(false) } -/// Path to the Supertonic model directory, or `None` if not ready. -pub fn supertonic_model_dir() -> Option { - global_model_manager()?.supertonic_model_dir() +/// Path to the Kokoro model directory, or `None` if not ready. +pub fn kokoro_model_dir() -> Option { + global_model_manager()?.kokoro_model_dir() } -/// `true` if all expected Supertonic model files are present on disk. -pub fn is_supertonic_ready() -> bool { +/// `true` if all expected Kokoro model files are present on disk. +pub fn is_kokoro_ready() -> bool { global_model_manager() - .map(|m| m.is_supertonic_ready()) + .map(|m| m.is_kokoro_ready()) .unwrap_or(false) } diff --git a/desktop/src-tauri/src/huddle/pipeline.rs b/desktop/src-tauri/src/huddle/pipeline.rs new file mode 100644 index 0000000000..e98370bcb6 --- /dev/null +++ b/desktop/src-tauri/src/huddle/pipeline.rs @@ -0,0 +1,280 @@ +//! STT/TTS pipeline lifecycle management. +//! +//! Handles starting, hot-starting, and spawning transcription tasks for +//! the voice pipelines. Extracted from mod.rs to keep the command layer thin. + +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex, +}; + +use nostr::JsonUtil; +use uuid::Uuid; + +use crate::app_state::AppState; +use crate::events; + +use super::models; +use super::relay_api::{fetch_channel_members, parse_channel_uuid}; +use super::state::{HuddlePhase, VoiceInputMode}; +use super::stt; +use super::tts; + +pub(crate) async fn post_connect_setup( + state: &AppState, + ephemeral_channel_id: &str, +) -> Result<(), String> { + // Hydrate agent pubkeys from relay (authoritative — overrides local guess). + if let Ok(agents) = fetch_channel_members(ephemeral_channel_id, Some("bot"), state).await { + let hs = state.huddle()?; + *hs.agent_pubkeys.lock().unwrap_or_else(|e| e.into_inner()) = agents; + } + + // Hydrate participants from relay (authoritative state). + if let Ok(all_members) = fetch_channel_members(ephemeral_channel_id, None, state).await { + if !all_members.is_empty() { + let mut hs = state.huddle()?; + hs.participants = all_members; + } + } + + // Ensure voice models are downloading (idempotent). + if let Some(mgr) = models::global_model_manager() { + mgr.start_moonshine_download(state.http_client.clone()); + mgr.start_kokoro_download(state.http_client.clone()); + } + + // Start pipelines: TTS first (so STT can capture tts_cancel for barge-in). + if let Err(e) = maybe_start_tts_pipeline(state).await { + eprintln!("sprout-desktop: TTS pipeline failed to start: {e}"); + } + if let Err(e) = maybe_start_stt_pipeline(state, ephemeral_channel_id).await { + eprintln!("sprout-desktop: STT pipeline failed to start: {e}"); + } + + Ok(()) +} + +/// Attempt to start the STT pipeline if models are present. +/// +/// Returns `Ok(true)` if the pipeline was started, `Ok(false)` if models are +/// not ready (voice-only mode), or `Err` on a real failure. +/// +/// Creates the shared `tts_active` flag and passes it to the STT pipeline +/// for barge-in / echo gating. The same flag is later passed to the TTS +/// pipeline so it can signal when audio is playing. +pub(crate) async fn maybe_start_stt_pipeline( + state: &AppState, + ephemeral_channel_id: &str, +) -> Result { + if !models::is_moonshine_ready() { + return Ok(false); // Models not downloaded yet — voice-only mode. + } + let model_dir = + models::moonshine_model_dir().ok_or_else(|| "Moonshine model directory not found")?; + + let channel_uuid = parse_channel_uuid(ephemeral_channel_id)?; + + // Atomically claim the construction slot (mirrors tts_starting pattern). + { + let hs = state.huddle()?; + if hs.stt_starting.swap(true, Ordering::AcqRel) { + return Ok(false); // Another caller is already constructing. + } + } + + // Grab shared flags, agent pubkeys, and session generation from HuddleState. + // If replacing an existing pipeline, bump generation first so the old + // transcription task's next POST sees a stale generation and exits. + // Take the old pipeline OUT of the lock before dropping — Drop joins + // the worker thread (~200ms) and must not block under the mutex. + let (tts_active, tts_cancel, agent_pubkeys_arc, session_gen, ptt_active_for_stt, old_stt) = { + let mut hs = state.huddle()?; + // Invalidate any existing transcription task before replacing the pipeline. + if hs.stt_pipeline.is_some() { + hs.session_generation.fetch_add(1, Ordering::Release); + } + let old = hs.stt_pipeline.take(); + if let Some(ref p) = old { + p.shutdown(); + } + let ptt = if hs.voice_input_mode == VoiceInputMode::PushToTalk { + Some(Arc::clone(&hs.ptt_active)) + } else { + None + }; + ( + Arc::clone(&hs.tts_active), + Some(Arc::clone(&hs.tts_cancel)), + Arc::clone(&hs.agent_pubkeys), + Arc::clone(&hs.session_generation), + ptt, + old, + ) + }; + // Drop the old pipeline OUTSIDE the lock — thread join happens here. + drop(old_stt); + + let (pipeline, text_rx) = + match stt::SttPipeline::new(model_dir, tts_active, tts_cancel, ptt_active_for_stt) { + Ok(p) => p, + Err(e) => { + let hs = state.huddle()?; + hs.stt_starting.store(false, Ordering::Release); + return Err(e); + } + }; + let pipeline = Arc::new(pipeline); + + { + let mut hs = state.huddle()?; + hs.stt_starting.store(false, Ordering::Release); + // Phase check: huddle may have been torn down during construction. + if !matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active) { + return Ok(false); + } + hs.stt_pipeline = Some(Arc::clone(&pipeline)); + } + + spawn_transcription_task(text_rx, channel_uuid, agent_pubkeys_arc, session_gen, state); + Ok(true) +} + +/// Attempt to start the TTS pipeline if Kokoro models are present and TTS is enabled. +/// +/// Returns `Ok(true)` if the pipeline was started, `Ok(false)` if preconditions +/// aren't met (model not ready, pipeline exists, TTS disabled), or `Err` on failure. +/// +/// Uses `tts_starting` sentinel to prevent TOCTOU races: two concurrent callers +/// (e.g. `check_pipeline_hotstart` + `speak_agent_message` lazy-start) could both +/// pass the `is_some()` check, both construct pipelines, and the loser's thread +/// leaks ~200MB of ONNX sessions. The sentinel is set under the lock before +/// releasing it for the expensive construction step. +pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result { + if !models::is_kokoro_ready() { + return Ok(false); // Kokoro not downloaded yet — TTS unavailable. + } + + let model_dir = match models::kokoro_model_dir() { + Some(d) => d, + None => return Ok(false), + }; + + // Atomically check preconditions and claim the construction slot. + // The sentinel prevents a second caller from starting construction + // while we're building outside the lock. + let (tts_active, tts_cancel) = { + let hs = state.huddle()?; + if hs.tts_pipeline.is_some() { + return Ok(false); + } + if !hs.tts_enabled { + return Ok(false); + } + if hs.tts_starting.swap(true, Ordering::AcqRel) { + return Ok(false); // Another caller is already constructing. + } + (Arc::clone(&hs.tts_active), Arc::clone(&hs.tts_cancel)) + }; + + // Construct outside the lock — this spawns the TTS worker thread and + // loads ONNX sessions (~200ms). If this fails, clear the sentinel. + let pipeline = match tts::TtsPipeline::new(model_dir, tts_active, tts_cancel) { + Ok(p) => Arc::new(p), + Err(e) => { + let hs = state.huddle()?; + hs.tts_starting.store(false, Ordering::Release); + return Err(e); + } + }; + + { + let mut hs = state.huddle()?; + hs.tts_starting.store(false, Ordering::Release); + // Phase check: huddle may have been torn down during construction. + if !matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active) { + return Ok(false); + } + // Final check: another path may have created a pipeline while we were constructing. + if hs.tts_pipeline.is_some() { + return Ok(false); + } + hs.tts_pipeline = Some(pipeline); + } + + Ok(true) +} + +/// Spawn a tokio task that reads text_rx and posts kind:9 events. +/// +/// Fix 1: `agent_pubkeys_arc` is an `Arc>>` cloned from +/// `HuddleState` — the task reads it at post time so p-tags are always +/// current, not a stale snapshot. +/// Fix 3: no `.unwrap()` on mutex — poisoned locks are recovered gracefully. +/// Fix 4: `text_rx` is a `tokio::sync::mpsc::Receiver` — fully async `.recv().await` +/// never blocks a Tokio worker thread (unlike std `recv_timeout`). +pub(crate) fn spawn_transcription_task( + mut text_rx: tokio::sync::mpsc::Receiver, + channel_uuid: Uuid, + agent_pubkeys_arc: Arc>>, + session_generation: Arc, + state: &AppState, +) { + // Capture the current generation at spawn time. + let spawned_gen = session_generation.load(Ordering::Acquire); + + let http_client = state.http_client.clone(); + let keys = match state.keys.lock() { + Ok(k) => k.clone(), + Err(_) => return, + }; + let configured_api_token = state.configured_api_token.clone(); + + tauri::async_runtime::spawn(async move { + // recv().await yields (not blocks) until text arrives or sender is dropped. + // When the STT worker exits and drops its Sender, recv() returns None → loop ends. + while let Some(t) = text_rx.recv().await { + if t.is_empty() { + continue; + } + + // Session guard: if the generation has changed, this task is stale. + // Drop the transcript silently — the huddle has ended or been replaced. + if session_generation.load(Ordering::Acquire) != spawned_gen { + break; // Exit the loop entirely — no more posts from this task. + } + + // Fix 1: read current agent pubkeys at post time. + let agent_pubkeys: Vec = agent_pubkeys_arc + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(); + + let p_tags: Vec<&str> = agent_pubkeys.iter().map(|s| s.as_str()).collect(); + let builder = match events::build_message(channel_uuid, &t, None, &p_tags, &[]) { + Ok(b) => b, + Err(e) => { + eprintln!("sprout-desktop: STT build_message: {e}"); + continue; + } + }; + let event = match builder.sign_with_keys(&keys) { + Ok(e) => e, + Err(e) => { + eprintln!("sprout-desktop: STT sign event: {e}"); + continue; + } + }; + let event_json = event.as_json(); + let api_token_ref = configured_api_token.as_deref(); + let pubkey_hex = keys.public_key().to_hex(); + + if let Err(e) = + crate::events::post_event_raw(&http_client, api_token_ref, &pubkey_hex, event_json) + .await + { + eprintln!("sprout-desktop: STT kind:9 post failed: {e}"); + } + } + }); +} diff --git a/desktop/src-tauri/src/huddle/relay_api.rs b/desktop/src-tauri/src/huddle/relay_api.rs new file mode 100644 index 0000000000..1879f4a583 --- /dev/null +++ b/desktop/src-tauri/src/huddle/relay_api.rs @@ -0,0 +1,98 @@ +//! Relay HTTP helpers for huddle operations. +//! +//! Thin wrappers around the relay REST API for LiveKit token requests, +//! channel membership queries, and human participant counting. + +use reqwest::Method; +use serde::Deserialize; +use uuid::Uuid; + +use crate::app_state::AppState; +use crate::relay::{api_path, build_authed_request, send_json_request}; + +use super::state::LiveKitTokenResponse; + +/// Maximum number of agents that can be invited to a single huddle. +pub(crate) const MAX_HUDDLE_AGENTS: usize = 20; + +/// Validate that a string looks like a Nostr pubkey hex (64 hex chars). +pub(crate) fn validate_pubkey_hex(pubkey: &str) -> Result<(), String> { + if pubkey.len() != 64 || !pubkey.chars().all(|c| c.is_ascii_hexdigit()) { + let preview: String = pubkey.chars().take(16).collect(); + return Err(format!("invalid pubkey hex: {preview}")); + } + Ok(()) +} + +pub(crate) fn parse_channel_uuid(channel_id: &str) -> Result { + Uuid::parse_str(channel_id).map_err(|_| format!("invalid channel UUID: {channel_id}")) +} + +/// Fetch a LiveKit token from the relay for the given channel. +pub(crate) async fn fetch_livekit_token( + channel_id: &str, + state: &AppState, +) -> Result { + let path = api_path(&["huddles", channel_id, "token"]); + let request = build_authed_request(&state.http_client, Method::POST, &path, state)?; + send_json_request(request).await +} + +/// Fetch channel members with their roles from the relay. +/// Returns (pubkey, role) tuples — the authoritative source for both +/// `fetch_channel_members` (filtered by role) and `count_human_members`. +pub(crate) async fn fetch_channel_members_with_roles( + channel_id: &str, + state: &AppState, +) -> Result)>, String> { + #[derive(Deserialize)] + struct Member { + pubkey: String, + role: Option, + } + #[derive(Deserialize)] + struct MembersResponse { + members: Vec, + } + + let path = api_path(&["channels", channel_id, "members"]); + let request = build_authed_request(&state.http_client, Method::GET, &path, state)?; + let resp: MembersResponse = send_json_request(request).await.map_err(|e| { + eprintln!("sprout-desktop: fetch channel members failed: {e}"); + e + })?; + + Ok(resp + .members + .into_iter() + .map(|m| (m.pubkey, m.role)) + .collect()) +} + +/// Fetch channel members from the relay. If `role_filter` is Some, only return +/// members with that role (e.g., "bot" for agents). Returns all members if None. +pub(crate) async fn fetch_channel_members( + channel_id: &str, + role_filter: Option<&str>, + state: &AppState, +) -> Result, String> { + let all = fetch_channel_members_with_roles(channel_id, state).await?; + Ok(all + .into_iter() + .filter(|(_, role)| role_filter.map_or(true, |r| role.as_deref() == Some(r))) + .map(|(pubkey, _)| pubkey) + .collect()) +} + +/// Count human (non-bot) members remaining in a channel. +/// Built on `fetch_channel_members_with_roles` — fetches all members then counts non-bots. +pub(crate) async fn count_human_members( + channel_id: &str, + state: &AppState, +) -> Result { + let all = fetch_channel_members_with_roles(channel_id, state).await?; + Ok(all + .iter() + .filter(|(_, role)| role.as_deref() != Some("bot")) + .count()) +} diff --git a/desktop/src-tauri/src/huddle/state.rs b/desktop/src-tauri/src/huddle/state.rs new file mode 100644 index 0000000000..dfd837bd9d --- /dev/null +++ b/desktop/src-tauri/src/huddle/state.rs @@ -0,0 +1,227 @@ +//! Huddle state types and serialization. +//! +//! Contains `HuddleState` (the god-object behind `AppState.huddle_state`), +//! phase enum, voice input mode, and response types. + +use serde::{Deserialize, Serialize}; +use std::sync::{ + atomic::{AtomicBool, AtomicU64}, + Arc, Mutex, +}; + +use super::{stt, tts}; + +/// Voice input mode: push-to-talk (PTT) or voice-activity detection (VAD). +/// +/// PTT (default): mic is gated by a global shortcut (Ctrl+Space). Pressing the +/// key sets `ptt_active` and immediately cancels any playing TTS. Releasing +/// the key (after a 200 ms delay) stops mic capture and flushes the utterance. +/// +/// VAD: the earshot VAD runs continuously and speech is accumulated whenever +/// the probability exceeds the threshold. Barge-in is enabled in this mode. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum VoiceInputMode { + #[default] + PushToTalk, + VoiceActivity, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum HuddlePhase { + Idle, + Creating, + Connecting, + Connected, // Backend ready, waiting for frontend media confirmation. + Active, + Leaving, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct HuddleState { + pub phase: HuddlePhase, + pub parent_channel_id: Option, + pub ephemeral_channel_id: Option, + /// Skipped from serialization — the frontend gets the token from + /// start_huddle/join_huddle return values. Exposing the LiveKit JWT + /// on every 2-second get_huddle_state poll is unnecessary attack surface. + #[serde(skip)] + pub livekit_token: Option, + /// Skipped from serialization — same rationale as livekit_token. + #[serde(skip)] + pub livekit_url: Option, + pub livekit_room: Option, + /// Participant pubkey hex strings (all members, including humans). + pub participants: Vec, + /// Agent pubkeys only — used as p-tags on transcribed messages. + /// + /// Stored as `Arc>>` so the transcription task can clone + /// the `Arc` and read the current list at post time without holding the + /// outer `HuddleState` lock across an await point. + /// + /// Populated from `member_pubkeys` in `start_huddle` (the UI sends agent + /// pubkeys specifically). Joiners don't add agents — they were already + /// added by the creator. Serialized as a plain `Vec` for the + /// frontend via the custom `Serialize`/`Deserialize` impls below. + #[serde( + serialize_with = "serialize_agent_pubkeys", + deserialize_with = "deserialize_agent_pubkeys" + )] + pub agent_pubkeys: Arc>>, + /// Active STT pipeline — not serialized, not cloned. + #[serde(skip)] + pub stt_pipeline: Option>, + /// Active TTS pipeline — not serialized, not cloned. + #[serde(skip)] + pub tts_pipeline: Option>, + /// Whether this client created the huddle (vs. joined it). + /// Used to enforce that only the creator can end/archive the huddle. + pub is_creator: bool, + /// Whether TTS output is enabled (user-toggled). + pub tts_enabled: bool, + /// Shared flag: true while TTS is playing audio. + /// Shared with the STT pipeline for barge-in / echo gating. + #[serde(skip)] + pub tts_active: Arc, + /// Shared barge-in cancel flag. Set by STT when it detects speech during TTS. + /// Read by TTS to stop playback. Lives in HuddleState so it survives pipeline + /// restarts — both STT and TTS reference the same flag for the entire huddle. + #[serde(skip)] + pub tts_cancel: Arc, + /// Sentinel: true while a TTS pipeline is being constructed (outside the lock). + /// Prevents TOCTOU races where two concurrent callers both pass the `is_some()` + /// check and both spawn TTS worker threads — the loser's thread would leak. + #[serde(skip)] + pub tts_starting: Arc, + /// Sentinel: true while an STT pipeline is being constructed. + /// Mirrors `tts_starting` — prevents TOCTOU races where two concurrent + /// callers both pass the `is_some()` check and both spawn STT workers. + #[serde(skip)] + pub stt_starting: Arc, + /// Timestamp of the last agent pubkey refresh from the relay. + /// Used to throttle the refresh in check_pipeline_hotstart to every 15 s. + #[serde(skip)] + pub last_agent_refresh: Option, + /// Session generation — incremented on every teardown. The transcription + /// task captures this at spawn time and checks before each POST. If the + /// generation has changed, the task silently drops the transcript. + #[serde(skip)] + pub session_generation: Arc, + /// Voice input mode: push-to-talk or voice-activity detection. + pub voice_input_mode: VoiceInputMode, + /// True while the PTT key is held (+ 200 ms release delay). + /// Shared with the STT pipeline for mic gating. + #[serde(skip)] + pub ptt_active: Arc, +} + +fn serialize_agent_pubkeys(v: &Arc>>, s: S) -> Result +where + S: serde::Serializer, +{ + use serde::ser::SerializeSeq; + let guard = v.lock().unwrap_or_else(|e| e.into_inner()); + let mut seq = s.serialize_seq(Some(guard.len()))?; + for item in guard.iter() { + seq.serialize_element(item)?; + } + seq.end() +} + +fn deserialize_agent_pubkeys<'de, D>(d: D) -> Result>>, D::Error> +where + D: serde::Deserializer<'de>, +{ + let v: Vec = serde::Deserialize::deserialize(d)?; + Ok(Arc::new(Mutex::new(v))) +} + +impl Clone for HuddleState { + fn clone(&self) -> Self { + let agent_pubkeys_snapshot = self + .agent_pubkeys + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(); + Self { + phase: self.phase.clone(), + parent_channel_id: self.parent_channel_id.clone(), + ephemeral_channel_id: self.ephemeral_channel_id.clone(), + livekit_token: self.livekit_token.clone(), + livekit_url: self.livekit_url.clone(), + livekit_room: self.livekit_room.clone(), + participants: self.participants.clone(), + agent_pubkeys: Arc::new(Mutex::new(agent_pubkeys_snapshot)), + stt_pipeline: None, // Never clone the pipeline handle. + tts_pipeline: None, // Never clone the pipeline handle. + is_creator: self.is_creator, + tts_enabled: self.tts_enabled, + tts_active: Arc::clone(&self.tts_active), + tts_cancel: Arc::clone(&self.tts_cancel), + tts_starting: Arc::clone(&self.tts_starting), + stt_starting: Arc::clone(&self.stt_starting), + last_agent_refresh: self.last_agent_refresh, + session_generation: Arc::clone(&self.session_generation), + voice_input_mode: self.voice_input_mode.clone(), + ptt_active: Arc::clone(&self.ptt_active), + } + } +} + +impl Default for HuddleState { + fn default() -> Self { + Self { + phase: HuddlePhase::Idle, + parent_channel_id: None, + ephemeral_channel_id: None, + livekit_token: None, + livekit_url: None, + livekit_room: None, + participants: Vec::new(), + agent_pubkeys: Arc::new(Mutex::new(Vec::new())), + stt_pipeline: None, + tts_pipeline: None, + is_creator: false, + tts_enabled: true, + tts_active: Arc::new(AtomicBool::new(false)), + tts_cancel: Arc::new(AtomicBool::new(false)), + tts_starting: Arc::new(AtomicBool::new(false)), + stt_starting: Arc::new(AtomicBool::new(false)), + last_agent_refresh: None, + session_generation: Arc::new(AtomicU64::new(0)), + voice_input_mode: VoiceInputMode::default(), + ptt_active: Arc::new(AtomicBool::new(false)), + } + } +} + +impl HuddleState { + /// Reset to default state while preserving the session generation counter. + /// Used by start_huddle rollback, join_huddle rollback, and teardown_huddle + /// to invalidate in-flight transcription tasks without losing the generation. + pub(crate) fn reset_preserving_generation(&mut self) { + let gen = Arc::clone(&self.session_generation); + *self = Self::default(); + self.session_generation = gen; + } +} + +// ── Response types ──────────────────────────────────────────────────────────── + +/// Returned by start_huddle and join_huddle. +#[derive(Debug, Serialize, Deserialize)] +pub struct HuddleJoinInfo { + pub ephemeral_channel_id: String, + pub livekit_token: String, + pub livekit_url: String, + pub livekit_room: String, +} + +/// Raw response from `POST /api/huddles/{channel_id}/token`. +#[derive(Debug, Deserialize)] +pub(crate) struct LiveKitTokenResponse { + pub token: String, + pub url: String, + pub room: String, +} diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index ce0cfecf02..6566ab553d 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -231,7 +231,6 @@ fn stt_worker( "sprout-desktop: STT models not found at {} — STT disabled", model_dir.display() ); - // Drain the channel so push_audio doesn't block the sender. drain_until_shutdown(audio_rx, &shutdown); return; } @@ -412,7 +411,8 @@ fn process_16k_samples( while leftover.len() >= VAD_FRAME_SAMPLES { let frame: Vec = leftover.drain(..VAD_FRAME_SAMPLES).collect(); - let prob = vad.predict_f32(&frame); + let clamped: Vec = frame.iter().map(|&s| s.clamp(-1.0, 1.0)).collect(); + let prob = vad.predict_f32(&clamped); let is_speech = prob > VAD_THRESHOLD; // PTT gating: when PTT key is not held, treat as silence. diff --git a/desktop/src-tauri/src/huddle/supertonic.rs b/desktop/src-tauri/src/huddle/supertonic.rs deleted file mode 100644 index 69e0ae39e0..0000000000 --- a/desktop/src-tauri/src/huddle/supertonic.rs +++ /dev/null @@ -1,700 +0,0 @@ -//! Supertonic TTS engine — wraps the 4-ONNX-session pipeline from -//! `supertone-inc/supertonic` and exposes a clean `call()` API that returns -//! `Vec` samples at 44.1 kHz. -//! -//! Mental model: -//! load_text_to_speech(onnx_dir) → TextToSpeech -//! load_voice_style(path) → Style -//! tts.call(text, lang, &style) → Vec @ 44.1 kHz - -use ndarray::{Array, Array3}; -use rand_distr::{Distribution, Normal}; -use regex::Regex; -use serde::{Deserialize, Serialize}; -use std::fs::File; -use std::io::BufReader; -use std::path::Path; -use std::sync::LazyLock; -use unicode_normalization::UnicodeNormalization; - -use ort::{session::Session, value::Value}; - -use super::preprocessing::split_sentences; - -// ── Public constants ────────────────────────────────────────────────────────── - -pub const SAMPLE_RATE: u32 = 44_100; - -pub const VOICES: &[&str] = &["F1", "F2", "F3", "F4", "F5", "M1", "M2", "M3", "M4", "M5"]; -pub const DEFAULT_VOICE: &str = "F1"; - -pub const AVAILABLE_LANGS: &[&str] = &["en", "ko", "es", "pt", "fr"]; - -// ── Config ──────────────────────────────────────────────────────────────────── - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct Config { - pub ae: AEConfig, - pub ttl: TTLConfig, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct AEConfig { - pub sample_rate: i32, - pub base_chunk_size: i32, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct TTLConfig { - pub chunk_compress_factor: i32, - pub latent_dim: i32, -} - -fn load_cfgs>(onnx_dir: P) -> Result { - let cfg_path = onnx_dir.as_ref().join("tts.json"); - let file = File::open(&cfg_path).map_err(|e| format!("open tts.json: {e}"))?; - let reader = BufReader::new(file); - serde_json::from_reader(reader).map_err(|e| format!("parse tts.json: {e}")) -} - -// ── Voice style ─────────────────────────────────────────────────────────────── - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct VoiceStyleData { - pub style_ttl: StyleComponent, - pub style_dp: StyleComponent, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct StyleComponent { - pub data: Vec>>, - pub dims: Vec, - #[serde(rename = "type")] - pub dtype: String, -} - -pub(crate) struct Style { - pub ttl: Array3, - pub dp: Array3, -} - -/// Load a single voice style JSON into a batch-1 `Style`. -pub(crate) fn load_voice_style>(path: P) -> Result { - let file = File::open(path.as_ref()) - .map_err(|e| format!("open voice style {}: {e}", path.as_ref().display()))?; - let reader = BufReader::new(file); - let data: VoiceStyleData = - serde_json::from_reader(reader).map_err(|e| format!("parse voice style: {e}"))?; - - let ttl_dims = &data.style_ttl.dims; - let dp_dims = &data.style_dp.dims; - - // Validate dimensions — model JSON must have [batch, dim1, dim2] shape. - if ttl_dims.len() < 3 { - return Err(format!( - "voice style ttl dims too short: expected 3, got {}", - ttl_dims.len() - )); - } - if dp_dims.len() < 3 { - return Err(format!( - "voice style dp dims too short: expected 3, got {}", - dp_dims.len() - )); - } - - // dims = [1, dim1, dim2] — batch dimension is always 1 for a single voice. - let (ttl_d1, ttl_d2) = (ttl_dims[1], ttl_dims[2]); - let (dp_d1, dp_d2) = (dp_dims[1], dp_dims[2]); - - let mut ttl_flat = Vec::with_capacity(ttl_d1 * ttl_d2); - for batch in &data.style_ttl.data { - for row in batch { - ttl_flat.extend_from_slice(row); - } - } - - let mut dp_flat = Vec::with_capacity(dp_d1 * dp_d2); - for batch in &data.style_dp.data { - for row in batch { - dp_flat.extend_from_slice(row); - } - } - - let ttl = Array3::from_shape_vec((1, ttl_d1, ttl_d2), ttl_flat) - .map_err(|e| format!("reshape ttl style: {e}"))?; - let dp = Array3::from_shape_vec((1, dp_d1, dp_d2), dp_flat) - .map_err(|e| format!("reshape dp style: {e}"))?; - - Ok(Style { ttl, dp }) -} - -// ── Unicode text processor ──────────────────────────────────────────────────── - -pub(crate) struct UnicodeProcessor { - indexer: Vec, -} - -impl UnicodeProcessor { - pub fn new>(path: P) -> Result { - let file = - File::open(path.as_ref()).map_err(|e| format!("open unicode_indexer.json: {e}"))?; - let reader = BufReader::new(file); - let indexer: Vec = - serde_json::from_reader(reader).map_err(|e| format!("parse unicode_indexer: {e}"))?; - Ok(UnicodeProcessor { indexer }) - } - - /// Tokenize a single (text, lang) pair into (token_ids, text_mask). - pub fn call( - &self, - text_list: &[String], - lang_list: &[String], - ) -> Result<(Vec>, Array3), String> { - let mut processed: Vec = Vec::with_capacity(text_list.len()); - for (text, lang) in text_list.iter().zip(lang_list.iter()) { - processed.push(preprocess_text(text, lang)?); - } - - let lengths: Vec = processed.iter().map(|t| t.chars().count()).collect(); - let max_len = *lengths.iter().max().unwrap_or(&0); - - let mut text_ids: Vec> = Vec::with_capacity(processed.len()); - for text in &processed { - let mut row = vec![0i64; max_len]; - for (j, c) in text.chars().enumerate() { - let val = c as usize; - row[j] = if val < self.indexer.len() { - self.indexer[val] - } else { - -1 - }; - } - text_ids.push(row); - } - - let text_mask = get_text_mask(&lengths); - Ok((text_ids, text_mask)) - } -} - -// ── Compiled regex patterns (one-time init) ─────────────────────────────── -static RE_SPACE_COMMA: LazyLock = LazyLock::new(|| Regex::new(r" ,").unwrap()); -static RE_SPACE_DOT: LazyLock = LazyLock::new(|| Regex::new(r" \.").unwrap()); -static RE_SPACE_BANG: LazyLock = LazyLock::new(|| Regex::new(r" !").unwrap()); -static RE_SPACE_QUESTION: LazyLock = LazyLock::new(|| Regex::new(r" \?").unwrap()); -static RE_SPACE_SEMI: LazyLock = LazyLock::new(|| Regex::new(r" ;").unwrap()); -static RE_SPACE_COLON: LazyLock = LazyLock::new(|| Regex::new(r" :").unwrap()); -static RE_SPACE_APOS: LazyLock = LazyLock::new(|| Regex::new(r" '").unwrap()); -static RE_ENDS_PUNC: LazyLock = - LazyLock::new(|| Regex::new(r#"[.!?;:,'")\]}…。」』】〉》›»]$"#).unwrap()); -static RE_PARAGRAPH: LazyLock = LazyLock::new(|| Regex::new(r"\n\s*\n").unwrap()); - -// ── Text preprocessing ──────────────────────────────────────────────────────── - -fn preprocess_text(text: &str, lang: &str) -> Result { - let mut s: String = text.nfkd().collect(); - - // Emoji already stripped by preprocessing.rs::preprocess_for_tts. - - // Character replacements. - for (from, to) in &[ - ("–", "-"), - ("‑", "-"), - ("—", "-"), - ("_", " "), - ("\u{201C}", "\""), - ("\u{201D}", "\""), - ("\u{2018}", "'"), - ("\u{2019}", "'"), - ("´", "'"), - ("`", "'"), - ("[", " "), - ("]", " "), - ("|", " "), - ("/", " "), - ("#", " "), - ("→", " "), - ("←", " "), - ] { - s = s.replace(from, to); - } - - for sym in &["♥", "☆", "♡", "©", "\\"] { - s = s.replace(sym, ""); - } - - for (from, to) in &[ - ("@", " at "), - ("e.g.,", "for example, "), - ("i.e.,", "that is, "), - ] { - s = s.replace(from, to); - } - - // Fix spacing around punctuation. - s = RE_SPACE_COMMA.replace_all(&s, ",").to_string(); - s = RE_SPACE_DOT.replace_all(&s, ".").to_string(); - s = RE_SPACE_BANG.replace_all(&s, "!").to_string(); - s = RE_SPACE_QUESTION.replace_all(&s, "?").to_string(); - s = RE_SPACE_SEMI.replace_all(&s, ";").to_string(); - s = RE_SPACE_COLON.replace_all(&s, ":").to_string(); - s = RE_SPACE_APOS.replace_all(&s, "'").to_string(); - - // Collapse duplicate quote pairs. - while s.contains("\"\"") { - s = s.replace("\"\"", "\""); - } - while s.contains("''") { - s = s.replace("''", "'"); - } - while s.contains("``") { - s = s.replace("``", "`"); - } - - // Whitespace already collapsed by preprocessing.rs::preprocess_for_tts. - // Trim only — model-specific transforms above may have introduced leading/trailing space. - s = s.trim().to_string(); - - // Ensure terminal punctuation. - if !s.is_empty() && !RE_ENDS_PUNC.is_match(&s) { - s.push('.'); - } - - if !AVAILABLE_LANGS.contains(&lang) { - return Err(format!( - "invalid lang '{lang}'; available: {AVAILABLE_LANGS:?}" - )); - } - - Ok(format!("<{lang}>{s}")) -} - -// ── Mask / latent helpers ───────────────────────────────────────────────────── - -fn length_to_mask(lengths: &[usize], max_len: usize) -> Array3 { - let bsz = lengths.len(); - let mut mask = Array3::::zeros((bsz, 1, max_len)); - for (i, &len) in lengths.iter().enumerate() { - for j in 0..len.min(max_len) { - mask[[i, 0, j]] = 1.0; - } - } - mask -} - -fn get_text_mask(lengths: &[usize]) -> Array3 { - let max_len = *lengths.iter().max().unwrap_or(&0); - length_to_mask(lengths, max_len) -} - -fn sample_noisy_latent( - duration: &[f32], - sample_rate: i32, - base_chunk_size: i32, - chunk_compress: i32, - latent_dim: i32, -) -> (Array3, Array3) { - let bsz = duration.len(); - let max_dur = duration.iter().cloned().fold(0.0f32, f32::max); - - let wav_len_max = (max_dur * sample_rate as f32) as usize; - let wav_lengths: Vec = duration - .iter() - .map(|&d| (d * sample_rate as f32) as usize) - .collect(); - - let chunk_size = (base_chunk_size * chunk_compress) as usize; - let latent_len = (wav_len_max + chunk_size - 1) / chunk_size; - let latent_dim_val = (latent_dim * chunk_compress) as usize; - - let mut noisy = Array3::::zeros((bsz, latent_dim_val, latent_len)); - let normal = Normal::new(0.0f32, 1.0f32).unwrap(); - let mut rng = rand::thread_rng(); - - for b in 0..bsz { - for d in 0..latent_dim_val { - for t in 0..latent_len { - noisy[[b, d, t]] = normal.sample(&mut rng); - } - } - } - - let latent_lengths: Vec = wav_lengths - .iter() - .map(|&len| ((len + chunk_size - 1) / chunk_size).max(1)) - .collect(); - - let latent_mask = length_to_mask(&latent_lengths, latent_len); - - // Apply mask. - for b in 0..bsz { - for d in 0..latent_dim_val { - for t in 0..latent_len { - noisy[[b, d, t]] *= latent_mask[[b, 0, t]]; - } - } - } - - (noisy, latent_mask) -} - -// ── Text chunking ───────────────────────────────────────────────────────────── - -const MAX_CHUNK_LEN: usize = 300; - -pub(crate) fn chunk_text(text: &str, max_len: Option) -> Vec { - let max_len = max_len.unwrap_or(MAX_CHUNK_LEN); - let text = text.trim(); - if text.is_empty() { - return vec![String::new()]; - } - - let mut chunks: Vec = Vec::new(); - - for para in RE_PARAGRAPH.split(text) { - let para = para.trim(); - if para.is_empty() { - continue; - } - - if para.len() <= max_len { - chunks.push(para.to_string()); - continue; - } - - let sentences = split_sentences(para); - let mut current = String::new(); - let mut current_len = 0usize; - - for sentence in sentences { - let sentence = sentence.trim(); - if sentence.is_empty() { - continue; - } - let slen = sentence.len(); - - if slen > max_len { - if !current.is_empty() { - chunks.push(current.trim().to_string()); - current.clear(); - current_len = 0; - } - // Split by comma, then by space. - for part in sentence.split(',') { - let part = part.trim(); - if part.is_empty() { - continue; - } - let plen = part.len(); - if plen > max_len { - let mut wchunk = String::new(); - let mut wlen = 0usize; - for word in part.split_whitespace() { - let wl = word.len(); - if wlen + wl + 1 > max_len && !wchunk.is_empty() { - chunks.push(wchunk.trim().to_string()); - wchunk.clear(); - wlen = 0; - } - if !wchunk.is_empty() { - wchunk.push(' '); - wlen += 1; - } - wchunk.push_str(word); - wlen += wl; - } - if !wchunk.is_empty() { - chunks.push(wchunk.trim().to_string()); - } - } else { - if current_len + plen + 2 > max_len && !current.is_empty() { - chunks.push(current.trim().to_string()); - current.clear(); - current_len = 0; - } - if !current.is_empty() { - current.push_str(", "); - current_len += 2; - } - current.push_str(part); - current_len += plen; - } - } - continue; - } - - if current_len + slen + 1 > max_len && !current.is_empty() { - chunks.push(current.trim().to_string()); - current.clear(); - current_len = 0; - } - if !current.is_empty() { - current.push(' '); - current_len += 1; - } - current.push_str(sentence); - current_len += slen; - } - - if !current.is_empty() { - chunks.push(current.trim().to_string()); - } - } - - if chunks.is_empty() { - vec![String::new()] - } else { - chunks - } -} - -// ── TextToSpeech ────────────────────────────────────────────────────────────── - -pub(crate) struct TextToSpeech { - cfgs: Config, - text_processor: UnicodeProcessor, - dp_ort: Session, - text_enc_ort: Session, - vector_est_ort: Session, - vocoder_ort: Session, - pub sample_rate: i32, -} - -impl TextToSpeech { - fn new( - cfgs: Config, - text_processor: UnicodeProcessor, - dp_ort: Session, - text_enc_ort: Session, - vector_est_ort: Session, - vocoder_ort: Session, - ) -> Self { - let sample_rate = cfgs.ae.sample_rate; - TextToSpeech { - cfgs, - text_processor, - dp_ort, - text_enc_ort, - vector_est_ort, - vocoder_ort, - sample_rate, - } - } - - fn _infer( - &mut self, - text_list: &[String], - lang_list: &[String], - style: &Style, - total_step: usize, - speed: f32, - ) -> Result<(Vec, Vec), String> { - let bsz = text_list.len(); - - let (text_ids, text_mask) = self.text_processor.call(text_list, lang_list)?; - - let seq_len = text_ids[0].len(); - let flat: Vec = text_ids.into_iter().flatten().collect(); - let text_ids_arr = Array::from_shape_vec((bsz, seq_len), flat) - .map_err(|e| format!("reshape text_ids: {e}"))?; - - let text_ids_val = - Value::from_array(text_ids_arr).map_err(|e| format!("text_ids Value: {e}"))?; - let text_mask_val = - Value::from_array(text_mask.clone()).map_err(|e| format!("text_mask Value: {e}"))?; - let style_dp_val = - Value::from_array(style.dp.clone()).map_err(|e| format!("style_dp Value: {e}"))?; - - // Duration prediction. - let dp_out = self - .dp_ort - .run(ort::inputs! { - "text_ids" => &text_ids_val, - "style_dp" => &style_dp_val, - "text_mask" => &text_mask_val - }) - .map_err(|e| format!("dp_ort run: {e}"))?; - - let (_, dur_data) = dp_out["duration"] - .try_extract_tensor::() - .map_err(|e| format!("extract duration: {e}"))?; - let mut duration: Vec = dur_data.to_vec(); - for d in &mut duration { - *d /= speed; - } - - // Text encoding. - let style_ttl_val = - Value::from_array(style.ttl.clone()).map_err(|e| format!("style_ttl Value: {e}"))?; - let text_enc_out = self - .text_enc_ort - .run(ort::inputs! { - "text_ids" => &text_ids_val, - "style_ttl" => &style_ttl_val, - "text_mask" => &text_mask_val - }) - .map_err(|e| format!("text_enc_ort run: {e}"))?; - - let (emb_shape, emb_data) = text_enc_out["text_emb"] - .try_extract_tensor::() - .map_err(|e| format!("extract text_emb: {e}"))?; - let text_emb = Array3::from_shape_vec( - ( - emb_shape[0] as usize, - emb_shape[1] as usize, - emb_shape[2] as usize, - ), - emb_data.to_vec(), - ) - .map_err(|e| format!("reshape text_emb: {e}"))?; - - // Noisy latent. - let (mut xt, latent_mask) = sample_noisy_latent( - &duration, - self.sample_rate, - self.cfgs.ae.base_chunk_size, - self.cfgs.ttl.chunk_compress_factor, - self.cfgs.ttl.latent_dim, - ); - - let total_step_arr = Array::from_elem(bsz, total_step as f32); - - // Denoising loop. - for step in 0..total_step { - let cur_step_arr = Array::from_elem(bsz, step as f32); - - let xt_val = Value::from_array(xt.clone()).map_err(|e| format!("xt Value: {e}"))?; - let emb_val = - Value::from_array(text_emb.clone()).map_err(|e| format!("emb Value: {e}"))?; - let lmask_val = - Value::from_array(latent_mask.clone()).map_err(|e| format!("lmask Value: {e}"))?; - let tmask_val = - Value::from_array(text_mask.clone()).map_err(|e| format!("tmask Value: {e}"))?; - let cur_val = - Value::from_array(cur_step_arr).map_err(|e| format!("cur_step Value: {e}"))?; - let tot_val = Value::from_array(total_step_arr.clone()) - .map_err(|e| format!("tot_step Value: {e}"))?; - let sttl_val = - Value::from_array(style.ttl.clone()).map_err(|e| format!("sttl Value: {e}"))?; - - let ve_out = self - .vector_est_ort - .run(ort::inputs! { - "noisy_latent" => &xt_val, - "text_emb" => &emb_val, - "style_ttl" => &sttl_val, - "latent_mask" => &lmask_val, - "text_mask" => &tmask_val, - "current_step" => &cur_val, - "total_step" => &tot_val - }) - .map_err(|e| format!("vector_est_ort run step {step}: {e}"))?; - - let (ds, dd) = ve_out["denoised_latent"] - .try_extract_tensor::() - .map_err(|e| format!("extract denoised_latent: {e}"))?; - xt = Array3::from_shape_vec( - (ds[0] as usize, ds[1] as usize, ds[2] as usize), - dd.to_vec(), - ) - .map_err(|e| format!("reshape denoised_latent: {e}"))?; - } - - // Vocoder. - let latent_val = Value::from_array(xt).map_err(|e| format!("final latent Value: {e}"))?; - let voc_out = self - .vocoder_ort - .run(ort::inputs! { - "latent" => &latent_val - }) - .map_err(|e| format!("vocoder_ort run: {e}"))?; - - let (_, wav_data) = voc_out["wav_tts"] - .try_extract_tensor::() - .map_err(|e| format!("extract wav_tts: {e}"))?; - - Ok((wav_data.to_vec(), duration)) - } - - /// Synthesize `text` in `lang` using `style`. - /// - /// Long text is automatically chunked; silence of `silence_secs` seconds - /// is inserted between chunks. Returns raw f32 PCM at `SAMPLE_RATE`. - pub(crate) fn call( - &mut self, - text: &str, - lang: &str, - style: &Style, - total_step: usize, - speed: f32, - silence_secs: f32, - ) -> Result, String> { - let max_len = if lang == "ko" { 120 } else { 300 }; - let chunks = chunk_text(text, Some(max_len)); - - let mut out: Vec = Vec::new(); - - for (i, chunk) in chunks.iter().enumerate() { - let (wav, duration) = self._infer( - &[chunk.clone()], - &[lang.to_string()], - style, - total_step, - speed, - )?; - - let dur = duration.first().copied().unwrap_or(0.0); - let wav_len = (self.sample_rate as f32 * dur) as usize; - let wav_chunk = &wav[..wav_len.min(wav.len())]; - - if i > 0 { - let silence_len = (silence_secs * self.sample_rate as f32) as usize; - out.extend(std::iter::repeat(0.0f32).take(silence_len)); - } - out.extend_from_slice(wav_chunk); - } - - Ok(out) - } -} - -// ── Loader ──────────────────────────────────────────────────────────────────── - -/// Load all four ONNX sessions and the Unicode tokenizer from `onnx_dir`. -/// -/// `onnx_dir` should be `~/.sprout/models/supertonic/`. -pub(crate) fn load_text_to_speech(onnx_dir: &str) -> Result { - let cfgs = load_cfgs(onnx_dir)?; - - let dp_ort = Session::builder() - .map_err(|e| format!("session builder: {e}"))? - .commit_from_file(format!("{onnx_dir}/duration_predictor.onnx")) - .map_err(|e| format!("load duration_predictor: {e}"))?; - - let text_enc_ort = Session::builder() - .map_err(|e| format!("session builder: {e}"))? - .commit_from_file(format!("{onnx_dir}/text_encoder.onnx")) - .map_err(|e| format!("load text_encoder: {e}"))?; - - let vector_est_ort = Session::builder() - .map_err(|e| format!("session builder: {e}"))? - .commit_from_file(format!("{onnx_dir}/vector_estimator.onnx")) - .map_err(|e| format!("load vector_estimator: {e}"))?; - - let vocoder_ort = Session::builder() - .map_err(|e| format!("session builder: {e}"))? - .commit_from_file(format!("{onnx_dir}/vocoder.onnx")) - .map_err(|e| format!("load vocoder: {e}"))?; - - let text_processor = UnicodeProcessor::new(format!("{onnx_dir}/unicode_indexer.json"))?; - - Ok(TextToSpeech::new( - cfgs, - text_processor, - dp_ort, - text_enc_ort, - vector_est_ort, - vocoder_ort, - )) -} diff --git a/desktop/src-tauri/src/huddle/tts.rs b/desktop/src-tauri/src/huddle/tts.rs index 0ddce474b7..4617c1e121 100644 --- a/desktop/src-tauri/src/huddle/tts.rs +++ b/desktop/src-tauri/src/huddle/tts.rs @@ -5,20 +5,20 @@ //! ```text //! caller: pipeline.speak("Hello world. How are you?") //! → bounded sync_channel (TEXT_QUEUE_DEPTH = 8) -//! → tts_worker thread (owns 1 Supertonic engine) +//! → tts_worker thread (owns 1 Kokoro engine) //! 1. Preprocess text //! 2. Split into sentences -//! 3. Batch sentences in groups of BATCH_SIZE → synth_batch() → f32 PCM -//! 4. Apply volume boost + fade in/out to each batch -//! 5. Append all buffers to a single rodio Player (gapless playback) +//! 3. Synthesize each sentence individually → f32 PCM +//! 4. Apply volume boost + fade in/out to each sentence +//! 5. Append each buffer to a single rodio Player (gapless playback) //! 6. Wait for player.empty() before accepting next text item //! → tts_active = true while playing, false when idle //! → cancel flag: player.clear() + drain queue //! ``` //! -//! Supertonic synthesis is ~167× faster than real-time. Batching 3 sentences -//! per engine.call() improves prosody (more context) and eliminates -//! inter-sentence gaps via a single persistent rodio Player. +//! Lookahead pipelining: synthesis of sentence N+1 begins immediately after +//! appending sentence N to the Player. Rodio queues buffers and plays them +//! sequentially — synthesis overlaps with playback for near-zero gaps. //! //! `tts_active` is an `Arc` shared with the STT pipeline so STT //! can gate microphone input while the agent is speaking. @@ -35,10 +35,8 @@ use std::{ time::Duration, }; +use super::kokoro::{load_text_to_speech, load_voice_style, SAMPLE_RATE}; use super::preprocessing::{preprocess_for_tts, split_sentences}; -use super::supertonic::{ - self, load_text_to_speech, load_voice_style, Style, TextToSpeech, SAMPLE_RATE, -}; // ── Constants ───────────────────────────────────────────────────────────────── @@ -50,26 +48,26 @@ const TEXT_QUEUE_DEPTH: usize = 8; /// How long the worker waits on the text channel before checking the shutdown flag. const RECV_TIMEOUT: Duration = Duration::from_millis(100); -/// Supertonic denoising steps. 5 = good quality/speed tradeoff. -/// Lower (2) = fastest; higher (10) = best quality. -const SYNTH_STEPS: usize = 5; +/// Kokoro ignores denoising steps (not a diffusion model). Kept for API compat. +const SYNTH_STEPS: usize = 1; /// Synthesis speed multiplier. Slightly faster than natural speech. const SYNTH_SPEED: f32 = 1.05; -/// Volume boost applied after synthesis — Supertonic output is quiet. -const VOLUME_BOOST: f32 = 2.5; +/// Volume boost applied after synthesis — Kokoro output is normalized. +/// Start at 1.5 and tune empirically. +const VOLUME_BOOST: f32 = 1.5; -/// Fade in/out length in samples (8ms at 44.1kHz ≈ 352 samples). -/// Eliminates clicks/pops at batch boundaries. +/// Fade in/out length in samples (8ms at 24kHz ≈ 192 samples). +/// Eliminates clicks/pops at sentence boundaries. const FADE_SAMPLES: usize = (SAMPLE_RATE as f64 * 0.008) as usize; -/// Number of sentences batched per engine.call() invocation. -/// More context → better prosody; synthesis is fast enough that this is free. -const BATCH_SIZE: usize = 3; +/// Sentence-by-sentence synthesis for lower TTFA (≈200ms vs ≈600ms for 3-sentence batches). +const BATCH_SIZE: usize = 1; -/// Silence inserted between batched sentences by the Supertonic engine (seconds). -const INTER_SENTENCE_SILENCE: f32 = 0.15; +/// Silence inserted between sentences by the Kokoro engine (seconds). +/// Kokoro handles its own inter-sentence silence. +const INTER_SENTENCE_SILENCE: f32 = 0.1; // ── Public pipeline handle ──────────────────────────────────────────────────── @@ -89,7 +87,7 @@ pub struct TtsPipeline { /// Kept alive here so the Arc isn't dropped — the worker holds a clone. #[allow(dead_code)] cancel: Arc, - /// Voice name (e.g. "F1"). Stored for future voice-switching support. + /// Voice name (e.g. "af_heart"). Stored for future voice-switching support. #[allow(dead_code)] voice: String, /// Worker thread handle — taken on drop to join cleanly. @@ -99,10 +97,8 @@ pub struct TtsPipeline { impl TtsPipeline { /// Spawn the TTS pipeline thread using the default voice. /// - /// `model_dir` must contain the Supertonic model files: - /// `duration_predictor.onnx`, `text_encoder.onnx`, - /// `vector_estimator.onnx`, `vocoder.onnx`, - /// `unicode_indexer.json`, `tts.json`, `.json` + /// `model_dir` must contain the Kokoro model files: + /// `model_quantized.onnx`, `tokenizer.json`, `voices/.bin` /// /// `tts_active` is set to `true` while audio is playing and `false` when idle. /// Pass the same `Arc` to the STT pipeline to gate microphone input. @@ -115,10 +111,11 @@ impl TtsPipeline { tts_active: Arc, cancel: Arc, ) -> Result { - Self::new_with_voice(model_dir, tts_active, cancel, supertonic::DEFAULT_VOICE) + use super::kokoro::DEFAULT_VOICE; + Self::new_with_voice(model_dir, tts_active, cancel, DEFAULT_VOICE) } - /// Spawn the TTS pipeline thread with a specific voice name (e.g. `"F1"`, `"M3"`). + /// Spawn the TTS pipeline thread with a specific voice name (e.g. `"af_heart"`, `"am_michael"`). pub fn new_with_voice( model_dir: PathBuf, tts_active: Arc, @@ -203,14 +200,14 @@ fn tts_worker( shutdown: Arc, cancel: Arc, ) { - // ── 1. Initialise Supertonic engine ─────────────────────────────────────── + // ── 1. Initialise Kokoro engine ─────────────────────────────────────────── let model_dir_str = model_dir.to_string_lossy().to_string(); let mut engine = match load_text_to_speech(&model_dir_str) { Ok(e) => e, Err(e) => { eprintln!( - "sprout-desktop: TTS Supertonic init failed (model_dir={}): {e}. TTS disabled.", + "sprout-desktop: TTS Kokoro init failed (model_dir={}): {e}. TTS disabled.", model_dir.display() ); drain_until_shutdown(text_rx, &shutdown); @@ -219,7 +216,7 @@ fn tts_worker( }; // ── 2. Load voice style ─────────────────────────────────────────────────── - let voice_path = model_dir.join(format!("{voice_name}.json")); + let voice_path = model_dir.join(format!("{voice_name}.bin")); let style = match load_voice_style(&voice_path) { Ok(s) => s, Err(e) => { @@ -293,11 +290,12 @@ fn tts_worker( continue; } - // Split into sentences, batch in groups of BATCH_SIZE for better - // prosody and gapless playback via a single persistent Player. + // Split into sentences. Each sentence is synthesized individually and + // appended to the Player immediately — synthesis of sentence N+1 overlaps + // with playback of sentence N (lookahead pipelining). let sentences: Vec = split_sentences(&text) .into_iter() - .filter(|s| !s.is_empty()) + .filter(|s| !s.trim().is_empty()) .collect(); if sentences.is_empty() { @@ -320,19 +318,53 @@ fn tts_worker( } }; - // Single persistent Player — all batches append here, rodio plays - // them gaplessly without per-batch device setup overhead. + // Single persistent Player — all sentences append here, rodio plays + // them gaplessly without per-sentence device setup overhead. let player = Player::connect_new(&sink_handle.mixer()); - tts_active.store(true, Ordering::Release); - - for chunk in sentences.chunks(BATCH_SIZE) { + // NOTE: tts_active is set AFTER the first player.append(), not before. + // Setting it before synthesis would cause STT to discard user speech + // during the synthesis window as "echo" even though no audio is + // actually playing yet. See crossfire review C3. + let mut first_append = true; + + // Lookahead pipeline: synthesize each sentence and append immediately. + // Rodio queues buffers sequentially — synthesis of the next sentence + // overlaps with playback of the current one. + for sentence in &sentences { if handle_cancel_or_shutdown(&cancel, &shutdown, &tts_active, &text_rx, Some(&player)) { break; } - if let Some(samples) = synth_batch(&mut engine, chunk, &style) { - let buf = SamplesBuffer::new(channels, rate, samples); - player.append(buf); + let text = sentence.trim(); + if text.is_empty() { + continue; + } + + match engine.call( + text, + "en", + &style, + SYNTH_STEPS, + SYNTH_SPEED, + INTER_SENTENCE_SILENCE, + ) { + Ok(samples) if !samples.is_empty() => { + let mut boosted: Vec = samples + .iter() + .map(|&s| (s * VOLUME_BOOST).clamp(-1.0, 1.0)) + .collect(); + apply_fades(&mut boosted); + let buf = SamplesBuffer::new(channels, rate, boosted); + player.append(buf); + if first_append { + tts_active.store(true, Ordering::Release); + first_append = false; + } + } + Ok(_) => {} + Err(e) => { + eprintln!("sprout-desktop: TTS synth failed: {e}"); + } } } @@ -387,46 +419,10 @@ fn handle_cancel_or_shutdown( false } -/// Synthesize a batch of sentences in a single engine call. -/// -/// Sentences are joined with a space so Supertonic sees full context for -/// better prosody. `silence_secs=INTER_SENTENCE_SILENCE` lets the engine -/// insert natural pauses between sentences internally. -/// -/// After synthesis: volume boost (×VOLUME_BOOST, clamped) + 8ms fade in/out -/// to eliminate clicks at batch boundaries. -fn synth_batch(engine: &mut TextToSpeech, sentences: &[String], style: &Style) -> Option> { - let text = sentences.join(" "); - match engine.call( - &text, - "en", - style, - SYNTH_STEPS, - SYNTH_SPEED, - INTER_SENTENCE_SILENCE, - ) { - Ok(samples) if !samples.is_empty() => { - // Volume boost — Supertonic output is quiet. - let mut boosted: Vec = samples - .iter() - .map(|&s| (s * VOLUME_BOOST).clamp(-1.0, 1.0)) - .collect(); - // Fade in/out to eliminate clicks at batch boundaries. - apply_fades(&mut boosted); - Some(boosted) - } - Ok(_) => None, - Err(e) => { - eprintln!("sprout-desktop: TTS synth failed: {e}"); - None - } - } -} - /// Apply a short linear fade-in at the start and fade-out at the end of `samples`. /// /// Uses `FADE_SAMPLES` (8ms) or half the buffer length, whichever is smaller. -/// Eliminates clicks/pops at batch boundaries. +/// Eliminates clicks/pops at sentence boundaries. fn apply_fades(samples: &mut Vec) { let len = samples.len(); let fade = FADE_SAMPLES.min(len / 2); @@ -442,3 +438,7 @@ fn apply_fades(samples: &mut Vec) { // drain_until_shutdown lives in super (huddle/mod.rs) — shared with stt.rs. use super::drain_until_shutdown; + +// BATCH_SIZE is used implicitly (one sentence per iteration). Suppress dead_code +// lint since it documents the design intent. +const _: () = assert!(BATCH_SIZE == 1); diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 39cc66dc23..db13caac95 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -483,10 +483,10 @@ pub fn run() { // Pre-download voice models in the background so they're ready // when the user starts their first huddle. Idempotent — no-op if - // already downloaded. ~303 MB total (50 MB Moonshine + 253 MB Supertonic). + // already downloaded. ~87 MB total (50 MB Moonshine + 87 MB Kokoro). if let Some(mgr) = huddle::models::global_model_manager() { mgr.start_moonshine_download(state.http_client.clone()); - mgr.start_supertonic_download(state.http_client.clone()); + mgr.start_kokoro_download(state.http_client.clone()); } // Register PTT global shortcut (Ctrl+Space). diff --git a/desktop/src/features/huddle/HuddleContext.tsx b/desktop/src/features/huddle/HuddleContext.tsx index 271e89a5ec..1ded589065 100644 --- a/desktop/src/features/huddle/HuddleContext.tsx +++ b/desktop/src/features/huddle/HuddleContext.tsx @@ -98,6 +98,10 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { /** Current voice input mode */ const [voiceInputMode, setVoiceInputModeState] = React.useState("push_to_talk"); + /** Ref tracking latest voiceInputMode — read inside connectAndSetupMedia to + * avoid stale closure capture when the user toggles mode mid-start. (Fix I4.) */ + const voiceInputModeRef = React.useRef("push_to_talk"); + voiceInputModeRef.current = voiceInputMode; /** Ephemeral channel ID — set after start_huddle/join_huddle, used for TTS subscription */ const [ephemeralChannelId, setEphemeralChannelId] = React.useState< string | null @@ -140,17 +144,13 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { }, []); // Toggle voice input mode — persists to Rust backend and updates worklet gating. - const setVoiceInputMode = React.useCallback( - async (mode: VoiceInputMode) => { - await invoke("set_voice_input_mode", { mode }); - setVoiceInputModeState(mode); - // Update the worklet's transmit state to match the new mode: - // VAD = always transmitting, PTT = transmit only when key is held. - const transmitting = mode === "voice_activity" || pttActive; - workletRef.current?.setTransmitting(transmitting); - }, - [pttActive], - ); + const setVoiceInputMode = React.useCallback(async (mode: VoiceInputMode) => { + await invoke("set_voice_input_mode", { mode }); + setVoiceInputModeState(mode); + // Use setMode (not setTransmitting) so the worklet tracks the current + // mode and ignores PTT events when in VAD mode. (Crossfire fix I1.) + workletRef.current?.setMode(mode); + }, []); /** Stop AudioWorklet and disconnect LiveKit. Best-effort on both steps. */ const disconnectMedia = React.useCallback(async () => { @@ -324,8 +324,8 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { setLocalAudioTrack(connection.localAudioTrack); setMicConnected(true); - // Setup AudioWorklet - const initialTransmitting = voiceInputMode !== "push_to_talk"; + // Setup AudioWorklet — read mode from ref to avoid stale closure (fix I4). + const initialTransmitting = voiceInputModeRef.current !== "push_to_talk"; const worklet = await setupAudioWorklet( connection.localAudioTrack, initialTransmitting, @@ -341,7 +341,7 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { return { connection, worklet }; }, - [leaveHuddle, voiceInputMode], + [leaveHuddle], ); const startHuddle = React.useCallback( diff --git a/desktop/src/features/huddle/components/HuddleBar.tsx b/desktop/src/features/huddle/components/HuddleBar.tsx index 0e2bb4aadd..f4932dc959 100644 --- a/desktop/src/features/huddle/components/HuddleBar.tsx +++ b/desktop/src/features/huddle/components/HuddleBar.tsx @@ -66,7 +66,7 @@ export function HuddleBar({ className }: HuddleBarProps) { const [agentAddError, setAgentAddError] = React.useState(null); const [modelStatus, setModelStatus] = React.useState<{ moonshine: string; - supertonic: string; + kokoro: string; } | null>(null); // Poll huddle state — replace with event listener once Rust emits events @@ -125,13 +125,13 @@ export function HuddleBar({ className }: HuddleBarProps) { try { const status = await invoke<{ moonshine: unknown; - supertonic: unknown; + kokoro: unknown; }>("get_model_status"); if (cancelled) return; setModelStatus({ moonshine: fmt(status.moonshine), - supertonic: fmt(status.supertonic), + kokoro: fmt(status.kokoro), }); } catch { // best-effort @@ -219,15 +219,15 @@ export function HuddleBar({ className }: HuddleBarProps) { {/* Model download progress */} {modelStatus && (modelStatus.moonshine !== "ready" || - modelStatus.supertonic !== "ready") && ( + modelStatus.kokoro !== "ready") && ( {modelStatus.moonshine !== "ready" && - modelStatus.supertonic !== "ready" - ? `Voice models: STT ${modelStatus.moonshine}, TTS ${modelStatus.supertonic}` + modelStatus.kokoro !== "ready" + ? `Voice models: STT ${modelStatus.moonshine}, TTS ${modelStatus.kokoro}` : modelStatus.moonshine !== "ready" ? `STT model: ${modelStatus.moonshine}` - : `TTS model: ${modelStatus.supertonic}`} + : `TTS model: ${modelStatus.kokoro}`} )} @@ -402,17 +402,19 @@ export function HuddleBar({ className }: HuddleBarProps) { - + {state?.is_creator && ( + + )} {/* Screen reader announcements for huddle state changes */} @@ -425,8 +427,8 @@ export function HuddleBar({ className }: HuddleBarProps) { modelStatus.moonshine !== "ready" && `, STT model ${modelStatus.moonshine}`} {modelStatus && - modelStatus.supertonic !== "ready" && - `, TTS model ${modelStatus.supertonic}`} + modelStatus.kokoro !== "ready" && + `, TTS model ${modelStatus.kokoro}`}
    ); diff --git a/desktop/src/features/huddle/lib/audioWorklet.ts b/desktop/src/features/huddle/lib/audioWorklet.ts index 98c44fcf52..4022e8b859 100644 --- a/desktop/src/features/huddle/lib/audioWorklet.ts +++ b/desktop/src/features/huddle/lib/audioWorklet.ts @@ -16,11 +16,14 @@ function invokeRawBinary(cmd: string, payload: Uint8Array): Promise { return internals.invoke(cmd, payload); } -/** Return type for setupAudioWorklet — stop + PTT control. */ +/** Return type for setupAudioWorklet — stop + mode control. */ export type AudioWorkletHandle = { stop: () => void; /** Send PTT state to the worklet processor. */ setTransmitting: (active: boolean) => void; + /** Switch voice input mode. In VAD mode, always transmitting (PTT events ignored). + * In PTT mode, gated by Ctrl+Space. */ + setMode: (mode: "push_to_talk" | "voice_activity") => void; }; /** @@ -90,12 +93,23 @@ export async function setupAudioWorklet( }); }; + // Track the current mode so PTT events are only forwarded in PTT mode. + // In VAD mode, the worklet stays in transmitting=true regardless of + // Ctrl+Space presses — prevents accidental muting. (Crossfire fix I1.) + let currentMode: "push_to_talk" | "voice_activity" = initialTransmitting + ? "voice_activity" + : "push_to_talk"; + // Listen for PTT state from Rust global shortcut (Ctrl+Space press/release). // Direction: Rust→main→worklet. The Tauri event carries a boolean payload. let pttUnlisten: UnlistenFn | null = null; try { pttUnlisten = await listen("ptt-state", (event) => { - workletNode.port.postMessage({ type: "ptt", active: event.payload }); + // Only forward PTT events to the worklet when in PTT mode. + // In VAD mode, Ctrl+Space is ignored — the worklet stays open. + if (currentMode === "push_to_talk") { + workletNode.port.postMessage({ type: "ptt", active: event.payload }); + } }); } catch { // PTT events not available — worklet stays in current transmit mode. @@ -114,5 +128,14 @@ export async function setupAudioWorklet( setTransmitting: (active: boolean) => { workletNode.port.postMessage({ type: "ptt", active }); }, + setMode: (mode: "push_to_talk" | "voice_activity") => { + currentMode = mode; + // When switching to VAD, immediately open the mic. + // When switching to PTT, immediately gate until key press. + workletNode.port.postMessage({ + type: "ptt", + active: mode === "voice_activity", + }); + }, }; } From fa1491ee329bcf503d02fbdc0fe76e275924e432 Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Tue, 14 Apr 2026 09:52:24 -0400 Subject: [PATCH 37/41] =?UTF-8?q?fix(huddle):=20crossfire=20P0/P1/P2=20fix?= =?UTF-8?q?es=20=E2=80=94=20safety,=20DRY,=20UX,=20accessibility?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P0 (safety): - leave_huddle: unwrap_or(1)→unwrap_or(2) prevents transient REST errors from auto-archiving - End All button now requires window.confirm() before destructive action - Mic permission denial surfaces user-friendly error banner (role=alert) P1 (DRY + correctness): - SDK builders: extracted build_huddle_event_sdk() shared helper (4 builders → thin wrappers) - Participant p-tag: join/leave events now encode which participant (enables roster reconstruction) - emit_end_and_archive() helper deduplicates leave_huddle auto-end + end_huddle - synth_chunk() public API on KokoroTTS eliminates double sentence-splitting - PTT audio cues: 50ms Web Audio tones (880Hz press, 440Hz release) - Agent visual distinction: bot badge on agent participants in ParticipantList P2 (polish + accessibility): - Voice style validation: rejects non-multiple-of-256 float counts - URL strip preserves trailing sentence punctuation (.!?) - HuddleBar: event-driven state updates (listen huddle-state-changed + 10s fallback) - Tauri: app_handle storage + emit_huddle_state() infrastructure - AX: aria-busy, aria-pressed, semantic ul/li, mode toggle announced via aria-live - post_event_raw: documented why consolidation with build_authed_request is incompatible --- crates/sprout-sdk/src/builders.rs | 106 ++++++++------ desktop/scripts/check-file-sizes.mjs | 10 +- desktop/src-tauri/src/app_state.rs | 25 ++++ desktop/src-tauri/src/events.rs | 36 ++++- desktop/src-tauri/src/huddle/kokoro.rs | 112 ++++++++++++++- desktop/src-tauri/src/huddle/mod.rs | 130 +++++++++++------- desktop/src-tauri/src/huddle/preprocessing.rs | 53 ++++++- desktop/src-tauri/src/huddle/state.rs | 33 +++++ desktop/src-tauri/src/huddle/tts.rs | 20 ++- desktop/src-tauri/src/lib.rs | 16 +++ desktop/src/features/huddle/HuddleContext.tsx | 73 +++++----- .../features/huddle/components/HuddleBar.tsx | 53 ++++++- .../huddle/components/ParticipantList.tsx | 79 +++++++---- 13 files changed, 558 insertions(+), 188 deletions(-) diff --git a/crates/sprout-sdk/src/builders.rs b/crates/sprout-sdk/src/builders.rs index 707dc48c42..a3d5e68e76 100644 --- a/crates/sprout-sdk/src/builders.rs +++ b/crates/sprout-sdk/src/builders.rs @@ -563,6 +563,36 @@ pub fn build_contact_list( Ok(EventBuilder::new(Kind::Custom(3), "", tags)) } +// ── Huddle shared helper ────────────────────────────────────────────────────── + +/// Shared builder for huddle lifecycle events (kinds 48100–48103). +/// +/// All huddle events share: an `["h", parent_channel_id]` tag, JSON content +/// with `ephemeral_channel_id`, optional extra content fields, and optional +/// p-tags for participant identity. +fn build_huddle_event_sdk( + kind: u16, + parent_channel_id: Uuid, + ephemeral_channel_id: Uuid, + extra_fields: &[(&str, &str)], + participant_pubkey: Option<&str>, +) -> Result { + let mut tags = vec![tag(&["h", &parent_channel_id.to_string()])?]; + if let Some(pk) = participant_pubkey { + tags.push(tag(&["p", pk])?); + } + let mut map = serde_json::Map::new(); + map.insert( + "ephemeral_channel_id".into(), + serde_json::Value::String(ephemeral_channel_id.to_string()), + ); + for (k, v) in extra_fields { + map.insert((*k).into(), serde_json::Value::String(v.to_string())); + } + let content = serde_json::Value::Object(map).to_string(); + Ok(EventBuilder::new(Kind::Custom(kind), content, tags)) +} + // ── Builder 26: build_huddle_started ───────────────────────────────────────── /// Build a huddle-started event (kind 48100). @@ -576,18 +606,13 @@ pub fn build_huddle_started( ephemeral_channel_id: Uuid, livekit_room: &str, ) -> Result { - let tags = vec![tag(&["h", &parent_channel_id.to_string()])?]; - let mut map = serde_json::Map::new(); - map.insert( - "ephemeral_channel_id".into(), - serde_json::Value::String(ephemeral_channel_id.to_string()), - ); - map.insert( - "livekit_room".into(), - serde_json::Value::String(livekit_room.into()), - ); - let content = serde_json::Value::Object(map).to_string(); - Ok(EventBuilder::new(Kind::Custom(48100), content, tags)) + build_huddle_event_sdk( + 48100, + parent_channel_id, + ephemeral_channel_id, + &[("livekit_room", livekit_room)], + None, + ) } // ── Builder 27: build_huddle_participant_joined ─────────────────────────────── @@ -597,18 +622,19 @@ pub fn build_huddle_started( /// Posted to the parent channel when a participant enters the huddle. /// - `parent_channel_id`: the channel the huddle belongs to (h-tag) /// - `ephemeral_channel_id`: the short-lived channel UUID for the huddle session +/// - `participant_pubkey`: hex pubkey of the joining participant (p-tag) pub fn build_huddle_participant_joined( parent_channel_id: Uuid, ephemeral_channel_id: Uuid, + participant_pubkey: &str, ) -> Result { - let tags = vec![tag(&["h", &parent_channel_id.to_string()])?]; - let mut map = serde_json::Map::new(); - map.insert( - "ephemeral_channel_id".into(), - serde_json::Value::String(ephemeral_channel_id.to_string()), - ); - let content = serde_json::Value::Object(map).to_string(); - Ok(EventBuilder::new(Kind::Custom(48101), content, tags)) + build_huddle_event_sdk( + 48101, + parent_channel_id, + ephemeral_channel_id, + &[], + Some(participant_pubkey), + ) } // ── Builder 28: build_huddle_participant_left ───────────────────────────────── @@ -618,18 +644,19 @@ pub fn build_huddle_participant_joined( /// Posted to the parent channel when a participant exits the huddle. /// - `parent_channel_id`: the channel the huddle belongs to (h-tag) /// - `ephemeral_channel_id`: the short-lived channel UUID for the huddle session +/// - `participant_pubkey`: hex pubkey of the departing participant (p-tag) pub fn build_huddle_participant_left( parent_channel_id: Uuid, ephemeral_channel_id: Uuid, + participant_pubkey: &str, ) -> Result { - let tags = vec![tag(&["h", &parent_channel_id.to_string()])?]; - let mut map = serde_json::Map::new(); - map.insert( - "ephemeral_channel_id".into(), - serde_json::Value::String(ephemeral_channel_id.to_string()), - ); - let content = serde_json::Value::Object(map).to_string(); - Ok(EventBuilder::new(Kind::Custom(48102), content, tags)) + build_huddle_event_sdk( + 48102, + parent_channel_id, + ephemeral_channel_id, + &[], + Some(participant_pubkey), + ) } // ── Builder 29: build_huddle_ended ─────────────────────────────────────────── @@ -643,14 +670,7 @@ pub fn build_huddle_ended( parent_channel_id: Uuid, ephemeral_channel_id: Uuid, ) -> Result { - let tags = vec![tag(&["h", &parent_channel_id.to_string()])?]; - let mut map = serde_json::Map::new(); - map.insert( - "ephemeral_channel_id".into(), - serde_json::Value::String(ephemeral_channel_id.to_string()), - ); - let content = serde_json::Value::Object(map).to_string(); - Ok(EventBuilder::new(Kind::Custom(48103), content, tags)) + build_huddle_event_sdk(48103, parent_channel_id, ephemeral_channel_id, &[], None) } // ── Helper: extract_channel_id ─────────────────────────────────────────────── @@ -1513,9 +1533,11 @@ mod tests { fn huddle_participant_joined_happy_path() { let parent = uuid(); let ephemeral = uuid(); - let ev = sign(build_huddle_participant_joined(parent, ephemeral).unwrap()); + let pubkey = "a".repeat(64); + let ev = sign(build_huddle_participant_joined(parent, ephemeral, &pubkey).unwrap()); assert_eq!(ev.kind.as_u16(), 48101); assert!(has_tag(&ev, "h", &parent.to_string())); + assert!(has_tag(&ev, "p", &pubkey)); let v: serde_json::Value = serde_json::from_str(&ev.content).unwrap(); assert_eq!(v["ephemeral_channel_id"], ephemeral.to_string()); } @@ -1524,7 +1546,8 @@ mod tests { fn huddle_participant_joined_h_tag_is_parent_not_ephemeral() { let parent = uuid(); let ephemeral = uuid(); - let ev = sign(build_huddle_participant_joined(parent, ephemeral).unwrap()); + let pubkey = "b".repeat(64); + let ev = sign(build_huddle_participant_joined(parent, ephemeral, &pubkey).unwrap()); assert!(has_tag(&ev, "h", &parent.to_string())); assert!(!has_tag(&ev, "h", &ephemeral.to_string())); } @@ -1535,9 +1558,11 @@ mod tests { fn huddle_participant_left_happy_path() { let parent = uuid(); let ephemeral = uuid(); - let ev = sign(build_huddle_participant_left(parent, ephemeral).unwrap()); + let pubkey = "c".repeat(64); + let ev = sign(build_huddle_participant_left(parent, ephemeral, &pubkey).unwrap()); assert_eq!(ev.kind.as_u16(), 48102); assert!(has_tag(&ev, "h", &parent.to_string())); + assert!(has_tag(&ev, "p", &pubkey)); let v: serde_json::Value = serde_json::from_str(&ev.content).unwrap(); assert_eq!(v["ephemeral_channel_id"], ephemeral.to_string()); } @@ -1546,7 +1571,8 @@ mod tests { fn huddle_participant_left_h_tag_is_parent_not_ephemeral() { let parent = uuid(); let ephemeral = uuid(); - let ev = sign(build_huddle_participant_left(parent, ephemeral).unwrap()); + let pubkey = "d".repeat(64); + let ev = sign(build_huddle_participant_left(parent, ephemeral, &pubkey).unwrap()); assert!(has_tag(&ev, "h", &parent.to_string())); assert!(!has_tag(&ev, "h", &ephemeral.to_string())); } diff --git a/desktop/scripts/check-file-sizes.mjs b/desktop/scripts/check-file-sizes.mjs index 626d2d083d..81e4036506 100644 --- a/desktop/scripts/check-file-sizes.mjs +++ b/desktop/scripts/check-file-sizes.mjs @@ -42,7 +42,7 @@ const overrides = new Map([ ["src/features/tokens/ui/TokenSettingsCard.tsx", 800], ["src/shared/api/relayClientSession.ts", 835], // durable websocket session manager with reconnect/replay/recovery state + sendTypingIndicator + fetchChannelHistoryBefore + subscribeToChannelLive (huddle TTS) + subscribeToHuddleEvents (huddle indicator) ["src/shared/api/tauri.ts", 1100], // remote agent provider API bindings + canvas API functions - ["src-tauri/src/lib.rs", 700], // sprout-media:// proxy + Range headers + Sprout nest init (ensure_nest) in setup() + huddle command registration + PTT global shortcut handler with generation counter and release delay + persona pack commands + ["src-tauri/src/lib.rs", 710], // sprout-media:// proxy + Range headers + Sprout nest init (ensure_nest) in setup() + huddle command registration + PTT global shortcut handler + persona pack commands + app_handle storage for event emission ["src-tauri/src/commands/media.rs", 720], // ffmpeg video transcode + poster frame extraction + run_ffmpeg_with_timeout (find_ffmpeg, is_video_file, transcode_to_mp4, extract_poster_frame, transcode_and_extract_poster) + spawn_blocking wrappers + tests ["src-tauri/src/commands/agents.rs", 880], // remote agent lifecycle routing (local + provider branches) + scope enforcement + persona pack metadata wiring + mcp_toolsets field ["src-tauri/src/managed_agents/runtime.rs", 690], // KNOWN_AGENT_BINARIES const + process_belongs_to_us FFI (macOS proc_name + Linux /proc/comm) + terminate_process + start/stop/sync lifecycle + pack persona live-read @@ -57,12 +57,12 @@ const overrides = new Map([ ["src/features/agents/ui/CreateAgentDialog.tsx", 685], // provider selector + config form + schema-typed config coercion + required field validation + locked scopes ["src/features/channels/ui/AddChannelBotDialog.tsx", 640], // provider mode: Run on selector, trust warning, probe effect, single-agent enforcement, provider warnings display ["src/shared/api/types.ts", 550], // persona provider/model fields + forum types + workflow type re-exports + ephemeral channel TTL fields + mcpToolsets + sourcePack + UpdateManagedAgentInput edit fields - ["src-tauri/src/events.rs", 530], // event builders + build_huddle_guidelines (kind:48106) + post_event_raw transport helper - ["src-tauri/src/huddle/kokoro.rs", 890], // Kokoro ONNX TTS engine + three-tier G2P + ARPAbet→IPA + CoreML + 20 G2P unit tests - ["src-tauri/src/huddle/mod.rs", 1000], // huddle state machine + Tauri commands + sync protocol doc; state/relay/pipeline extracted + ["src-tauri/src/events.rs", 555], // event builders + build_huddle_guidelines (kind:48106) + post_event_raw transport helper + participant p-tag on join/leave + ["src-tauri/src/huddle/kokoro.rs", 980], // Kokoro ONNX TTS engine + three-tier G2P + ARPAbet→IPA + CoreML + synth_chunk() public API + style validation + hyphenated compound splitting + 23 unit tests + ["src-tauri/src/huddle/mod.rs", 1020], // huddle state machine + Tauri commands + sync protocol doc; state/relay/pipeline extracted + emit_huddle_state_changed wiring ["src-tauri/src/huddle/models.rs", 850], // model download manager for Moonshine STT + Kokoro TTS with streaming downloads + SHA-256 verification + Rust-native tar extraction + version manifest + atomic swap + hot-start signaling ["src-tauri/src/huddle/stt.rs", 580], // STT pipeline + PTT edge-detection flush + PTT gating (is_speech AND ptt_active) + barge-in for VAD mode + rubato resampler + earshot VAD + sherpa-onnx transcription - ["src-tauri/src/huddle/preprocessing.rs", 620], // TTS text preprocessing pipeline + unified split_sentences (consolidated from tts.rs + supertonic.rs) + int_to_words 0-999999 + 18 unit tests + ["src-tauri/src/huddle/preprocessing.rs", 670], // TTS text preprocessing pipeline + unified split_sentences + int_to_words 0-999999 + URL trailing punctuation preservation + 23 unit tests ]); async function walkFiles(directory) { diff --git a/desktop/src-tauri/src/app_state.rs b/desktop/src-tauri/src/app_state.rs index 578386f3bb..a1d8ca6dde 100644 --- a/desktop/src-tauri/src/app_state.rs +++ b/desktop/src-tauri/src/app_state.rs @@ -14,6 +14,12 @@ pub struct AppState { pub managed_agents_store_lock: Mutex<()>, pub managed_agent_processes: Mutex>, pub huddle_state: Mutex, + /// Tauri app handle — stored after setup so huddle commands can emit + /// `huddle-state-changed` events without needing the handle threaded + /// through every call site. + /// + /// Set once during `setup()` in `lib.rs`; never cleared. + pub app_handle: Mutex>, } pub fn build_app_state() -> AppState { @@ -58,6 +64,7 @@ pub fn build_app_state() -> AppState { managed_agents_store_lock: Mutex::new(()), managed_agent_processes: Mutex::new(HashMap::new()), huddle_state: Mutex::new(HuddleState::default()), + app_handle: Mutex::new(None), } } @@ -70,6 +77,24 @@ impl AppState { pub fn huddle(&self) -> Result, String> { self.huddle_state.lock().map_err(|e| e.to_string()) } + + /// Emit the current huddle state to the frontend via Tauri event. + /// + /// Acquires both locks (app_handle + huddle_state), clones a snapshot, + /// releases both, then emits. Best-effort — no-op if either lock is + /// poisoned or the app_handle hasn't been set yet. + pub fn emit_huddle_state_changed(&self) { + let app = match self.app_handle.lock() { + Ok(guard) => guard.clone(), + Err(_) => return, + }; + let Some(app) = app else { return }; + let snapshot = match self.huddle_state.lock() { + Ok(hs) => hs.clone(), + Err(_) => return, + }; + crate::huddle::state::emit_huddle_state(&app, &snapshot); + } } /// Resolve the user's identity key from the app data directory. diff --git a/desktop/src-tauri/src/events.rs b/desktop/src-tauri/src/events.rs index 2750f0865f..4c4dedf1ff 100644 --- a/desktop/src-tauri/src/events.rs +++ b/desktop/src-tauri/src/events.rs @@ -367,12 +367,14 @@ fn validate_channel_id(id: &str) -> Result<(), String> { /// Shared builder for huddle lifecycle events (kinds 48100–48103). /// All huddle events share: validate two channel IDs, JSON content with -/// `ephemeral_channel_id`, and an `["h", parent_channel_id]` tag. +/// `ephemeral_channel_id`, an `["h", parent_channel_id]` tag, and an +/// optional `["p", participant_pubkey]` tag for join/leave identity. fn build_huddle_event( kind: u16, parent_channel_id: &str, ephemeral_channel_id: &str, extra_fields: &[(&str, &str)], + participant_pubkey: Option<&str>, ) -> Result { validate_channel_id(parent_channel_id)?; validate_channel_id(ephemeral_channel_id)?; @@ -382,7 +384,10 @@ fn build_huddle_event( for (k, v) in extra_fields { content[*k] = serde_json::Value::String(v.to_string()); } - let tags = vec![tag(vec!["h", parent_channel_id])?]; + let mut tags = vec![tag(vec!["h", parent_channel_id])?]; + if let Some(pk) = participant_pubkey { + tags.push(tag(vec!["p", pk])?); + } Ok(EventBuilder::new(Kind::Custom(kind), content.to_string()).tags(tags)) } @@ -397,23 +402,44 @@ pub fn build_huddle_started( parent_channel_id, ephemeral_channel_id, &[("livekit_room", livekit_room)], + None, ) } /// Kind 48101 — participant joined a huddle, posted to the parent channel. +/// +/// `participant_pubkey`: when provided, adds a `["p", pubkey]` tag so +/// consumers can identify who joined without parsing the event's author. pub fn build_huddle_participant_joined( parent_channel_id: &str, ephemeral_channel_id: &str, + participant_pubkey: Option<&str>, ) -> Result { - build_huddle_event(48101, parent_channel_id, ephemeral_channel_id, &[]) + build_huddle_event( + 48101, + parent_channel_id, + ephemeral_channel_id, + &[], + participant_pubkey, + ) } /// Kind 48102 — participant left a huddle, posted to the parent channel. +/// +/// `participant_pubkey`: when provided, adds a `["p", pubkey]` tag so +/// consumers can identify who left without parsing the event's author. pub fn build_huddle_participant_left( parent_channel_id: &str, ephemeral_channel_id: &str, + participant_pubkey: Option<&str>, ) -> Result { - build_huddle_event(48102, parent_channel_id, ephemeral_channel_id, &[]) + build_huddle_event( + 48102, + parent_channel_id, + ephemeral_channel_id, + &[], + participant_pubkey, + ) } /// Kind 48103 — huddle ended, posted to the parent channel. @@ -421,7 +447,7 @@ pub fn build_huddle_ended( parent_channel_id: &str, ephemeral_channel_id: &str, ) -> Result { - build_huddle_event(48103, parent_channel_id, ephemeral_channel_id, &[]) + build_huddle_event(48103, parent_channel_id, ephemeral_channel_id, &[], None) } /// Kind 48106 — voice-mode guidelines for agents in a huddle. diff --git a/desktop/src-tauri/src/huddle/kokoro.rs b/desktop/src-tauri/src/huddle/kokoro.rs index bbb0a4148d..0c5bf7224e 100644 --- a/desktop/src-tauri/src/huddle/kokoro.rs +++ b/desktop/src-tauri/src/huddle/kokoro.rs @@ -79,6 +79,13 @@ pub fn load_voice_style(path: &Path) -> Result { data.len() )); } + if data.len() % 256 != 0 { + return Err(format!( + "voice style has {} floats — expected a multiple of 256 (got {} remainder)", + data.len(), + data.len() % 256, + )); + } Ok(VoiceStyle { data }) } @@ -471,6 +478,19 @@ impl Lexicon { /// Convert a single word to IPA using the full fallback chain. fn word_to_ipa(&self, word: &str) -> String { + // Compound words: split on hyphens and underscores, process each part + // independently. "short-and-natural" → "short" + "and" + "natural", + // "parent_event_id" → "parent" + "event" + "id". + // Each part gets full dict lookup. Joined with a space (brief TTS pause). + if word.contains('-') || word.contains('_') { + let parts: Vec = word + .split(|c: char| c == '-' || c == '_') + .filter(|p| !p.is_empty()) + .map(|p| self.word_to_ipa(p)) + .collect(); + return parts.join(" "); + } + // Normalize curly quotes to straight apostrophes. let normalized = word.replace('\u{2019}', "'").replace('\u{2018}', "'"); let stripped: String = normalized @@ -642,7 +662,7 @@ impl KokoroTTS { let mut output: Vec = Vec::new(); for (i, sentence) in sentences.iter().enumerate() { - let chunk_audio = self.synth_chunk(sentence, style, speed)?; + let chunk_audio = self.synth_chunk(sentence, _lang, style, _total_step, speed)?; if i > 0 && !output.is_empty() { output.extend_from_slice(&silence); @@ -653,11 +673,19 @@ impl KokoroTTS { Ok(output) } - /// Synthesize a single text chunk: G2P → tokenize → ONNX → PCM. - fn synth_chunk( + /// Synthesize a single pre-split text chunk. Caller is responsible for sentence splitting. + /// This avoids double-splitting when the TTS pipeline has already split the text. + /// + /// - `_lang` is accepted for API compatibility but currently unused + /// (Kokoro v1.0 language is selected by voice name prefix, e.g. `af_*`). + /// - `_steps` is accepted for API compatibility but currently unused + /// (Kokoro is not diffusion-based). + pub fn synth_chunk( &mut self, text: &str, + _lang: &str, style: &VoiceStyle, + _steps: usize, speed: f32, ) -> Result, String> { // G2P: text → IPA phoneme string @@ -841,6 +869,35 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + #[test] + fn load_voice_style_rejects_non_multiple_of_256() { + use std::io::Write; + let dir = std::env::temp_dir().join("kokoro_test_nonaligned"); + let _ = std::fs::create_dir_all(&dir); + let path = dir.join("nonaligned.bin"); + let mut f = std::fs::File::create(&path).unwrap(); + // Write 257 floats — not a multiple of 256 (remainder = 1) + for i in 0..257u32 { + f.write_all(&(i as f32).to_le_bytes()).unwrap(); + } + drop(f); + let result = load_voice_style(&path); + assert!( + result.is_err(), + "should reject file with non-multiple-of-256 floats" + ); + let err = result.unwrap_err(); + assert!( + err.contains("257"), + "error should mention float count: {err}" + ); + assert!( + err.contains("remainder"), + "error should mention remainder: {err}" + ); + let _ = std::fs::remove_dir_all(&dir); + } + // ── Suffix rules ────────────────────────────────────────────────────── #[test] @@ -877,4 +934,53 @@ mod tests { fn apply_ing_basic() { assert!(Lexicon::apply_ing("rʌn").ends_with("ɪŋ")); } + + #[test] + fn hyphenated_word_splits_into_parts() { + let lex = Lexicon { + misaki: HashMap::new(), + cmudict: HashMap::new(), + }; + let result = lex.word_to_ipa("short-and-natural"); + let space_count = result.matches(' ').count(); + assert_eq!( + space_count, 2, + "expected 2 spaces for 3 hyphenated parts, got {space_count}: {result}" + ); + } + + #[test] + fn underscored_word_splits_into_parts() { + let lex = Lexicon { + misaki: HashMap::new(), + cmudict: HashMap::new(), + }; + let result = lex.word_to_ipa("parent_event_id"); + let space_count = result.matches(' ').count(); + assert_eq!( + space_count, 2, + "expected 2 spaces for 3 underscored parts, got {space_count}: {result}" + ); + } + + #[test] + fn compound_word_with_dict_lookup() { + let mut dict = HashMap::new(); + dict.insert("parent".to_string(), "pɛɹənt".to_string()); + dict.insert("event".to_string(), "ɪvɛnt".to_string()); + dict.insert("id".to_string(), "aɪdiː".to_string()); + let lex = Lexicon { + misaki: dict, + cmudict: HashMap::new(), + }; + // Underscore compound + let result = lex.word_to_ipa("parent_event_id"); + assert!(result.contains("pɛɹənt"), "parent not resolved: {result}"); + assert!(result.contains("ɪvɛnt"), "event not resolved: {result}"); + assert!(result.contains("aɪdiː"), "id not resolved: {result}"); + + // Hyphen compound + let result = lex.word_to_ipa("short-and-sweet"); + assert_eq!(result.matches(' ').count(), 2, "hyphen split: {result}"); + } } diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index dc5cebf043..039d3a88a9 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -260,7 +260,10 @@ pub async fn start_huddle( hs.participants = participants; } - // 6. Hydrate members, download models, start pipelines. + // 6. Notify frontend of state change. + state.emit_huddle_state_changed(); + + // 7. Hydrate members, download models, start pipelines. if let Err(e) = post_connect_setup(&state, &ephemeral_channel_id).await { eprintln!("sprout-desktop: post_connect_setup failed (degraded mode): {e}"); // Non-fatal: huddle works without STT/TTS pipelines. @@ -360,10 +363,12 @@ pub async fn join_huddle( // 1. Fetch LiveKit token. let lk = fetch_livekit_token(&ephemeral_channel_id, &state).await?; - // 2. Emit PARTICIPANT_JOINED (best-effort). - if let Ok(joined_builder) = - events::build_huddle_participant_joined(&parent_channel_id, &ephemeral_channel_id) - { + // 2. Emit PARTICIPANT_JOINED (best-effort) — include own pubkey as p-tag. + if let Ok(joined_builder) = events::build_huddle_participant_joined( + &parent_channel_id, + &ephemeral_channel_id, + Some(&own_pubkey), + ) { if let Err(e) = submit_event(joined_builder, &state).await { eprintln!("sprout-desktop: huddle_participant_joined event failed: {e}"); } @@ -388,7 +393,10 @@ pub async fn join_huddle( } } - // 4. Hydrate members, download models, start pipelines. + // 4. Notify frontend of state change. + state.emit_huddle_state_changed(); + + // 5. Hydrate members, download models, start pipelines. if let Err(e) = post_connect_setup(&state, &ephemeral_channel_id).await { eprintln!("sprout-desktop: post_connect_setup failed (degraded mode): {e}"); } @@ -450,9 +458,40 @@ fn teardown_huddle(state: &AppState) -> Result<(), String> { // Drop the Arcs here (implicit) — triggers thread join via Drop. drop(old_stt); drop(old_tts); + // Notify frontend that we're back to Idle. + state.emit_huddle_state_changed(); Ok(()) } +/// Emit HUDDLE_ENDED to the parent channel and archive the ephemeral channel. +/// +/// Both steps are best-effort — failures are logged but do not propagate. +/// Called from `leave_huddle` (auto-end path) and `end_huddle`. +async fn emit_end_and_archive( + parent_channel_id: &str, + ephemeral_channel_id: &str, + state: &AppState, +) { + if !parent_channel_id.is_empty() && !ephemeral_channel_id.is_empty() { + if let Ok(ended_builder) = + events::build_huddle_ended(parent_channel_id, ephemeral_channel_id) + { + if let Err(e) = submit_event(ended_builder, state).await { + eprintln!("sprout-desktop: huddle_ended event failed: {e}"); + } + } + } + if !ephemeral_channel_id.is_empty() { + if let Ok(uuid) = parse_channel_uuid(ephemeral_channel_id) { + if let Ok(archive_builder) = events::build_archive(uuid) { + if let Err(e) = submit_event(archive_builder, state).await { + eprintln!("sprout-desktop: archive ephemeral channel failed: {e}"); + } + } + } + } +} + /// Leave the current huddle. /// /// Steps: @@ -473,11 +512,18 @@ pub async fn leave_huddle(state: State<'_, AppState>) -> Result<(), String> { ) }; - // Emit PARTICIPANT_LEFT (best-effort). + // Emit PARTICIPANT_LEFT (best-effort) — include own pubkey as p-tag. + let own_pubkey = state + .keys + .lock() + .ok() + .map(|k| k.public_key().to_hex()); if !parent_channel_id.is_empty() && !ephemeral_channel_id.is_empty() { - if let Ok(left_builder) = - events::build_huddle_participant_left(&parent_channel_id, &ephemeral_channel_id) - { + if let Ok(left_builder) = events::build_huddle_participant_left( + &parent_channel_id, + &ephemeral_channel_id, + own_pubkey.as_deref(), + ) { if let Err(e) = submit_event(left_builder, &state).await { eprintln!("sprout-desktop: huddle_participant_left event failed: {e}"); } @@ -493,7 +539,12 @@ pub async fn leave_huddle(state: State<'_, AppState>) -> Result<(), String> { if !parent_channel_id.is_empty() && !ephemeral_channel_id.is_empty() { let humans_remaining = count_human_members(&ephemeral_channel_id, &state) .await - .unwrap_or(1); // On fetch failure, assume someone is still there (safe default). + // On fetch failure, assume 2 humans remain (safe default). + // unwrap_or(1) would mean "I'm the last human" → triggers auto-archive, + // ending the huddle for everyone on a transient REST failure. Using 2 + // means we skip the auto-end path and just remove ourselves — the huddle + // stays alive and the next real leave will clean up correctly. + .unwrap_or(2); if humans_remaining <= 1 { // We're the last human — end the huddle entirely. @@ -501,20 +552,7 @@ pub async fn leave_huddle(state: State<'_, AppState>) -> Result<(), String> { // This avoids the "cannot remove the last owner" relay error that // build_leave hits when the creator is the sole remaining member. eprintln!("sprout-desktop: last human left huddle — auto-ending"); - if let Ok(ended_builder) = - events::build_huddle_ended(&parent_channel_id, &ephemeral_channel_id) - { - if let Err(e) = submit_event(ended_builder, &state).await { - eprintln!("sprout-desktop: auto-end huddle_ended event failed: {e}"); - } - } - if let Ok(eph_uuid) = parse_channel_uuid(&ephemeral_channel_id) { - if let Ok(archive_builder) = events::build_archive(eph_uuid) { - if let Err(e) = submit_event(archive_builder, &state).await { - eprintln!("sprout-desktop: auto-end archive ephemeral channel failed: {e}"); - } - } - } + emit_end_and_archive(&parent_channel_id, &ephemeral_channel_id, &state).await; } else { // Other humans still in the huddle — just remove self from membership. if let Ok(eph_uuid) = parse_channel_uuid(&ephemeral_channel_id) { @@ -560,27 +598,7 @@ pub async fn end_huddle(force: Option, state: State<'_, AppState>) -> Resu ) }; - // Emit HUDDLE_ENDED (best-effort). - if !parent_channel_id.is_empty() && !ephemeral_channel_id.is_empty() { - if let Ok(ended_builder) = - events::build_huddle_ended(&parent_channel_id, &ephemeral_channel_id) - { - if let Err(e) = submit_event(ended_builder, &state).await { - eprintln!("sprout-desktop: huddle_ended event failed: {e}"); - } - } - } - - // Archive the ephemeral channel (best-effort). - if !ephemeral_channel_id.is_empty() { - if let Ok(uuid) = parse_channel_uuid(&ephemeral_channel_id) { - if let Ok(archive_builder) = events::build_archive(uuid) { - if let Err(e) = submit_event(archive_builder, &state).await { - eprintln!("sprout-desktop: huddle archive ephemeral channel failed: {e}"); - } - } - } - } + emit_end_and_archive(&parent_channel_id, &ephemeral_channel_id, &state).await; teardown_huddle(&state)?; @@ -591,15 +609,21 @@ pub async fn end_huddle(force: Option, state: State<'_, AppState>) -> Resu /// Transitions from Connected → Active. No-op if already Active. #[tauri::command] pub async fn confirm_huddle_active(state: State<'_, AppState>) -> Result<(), String> { - let mut hs = state.huddle()?; - match hs.phase { - HuddlePhase::Connected => { - hs.phase = HuddlePhase::Active; - Ok(()) + let transitioned = { + let mut hs = state.huddle()?; + match hs.phase { + HuddlePhase::Connected => { + hs.phase = HuddlePhase::Active; + true + } + HuddlePhase::Active => false, // Already active — idempotent. + ref other => return Err(format!("cannot confirm active: phase is {:?}", other)), } - HuddlePhase::Active => Ok(()), // Already active — idempotent. - ref other => Err(format!("cannot confirm active: phase is {:?}", other)), + }; + if transitioned { + state.emit_huddle_state_changed(); } + Ok(()) } /// Return the current HuddleState (serialized for the frontend). diff --git a/desktop/src-tauri/src/huddle/preprocessing.rs b/desktop/src-tauri/src/huddle/preprocessing.rs index fd21418601..2f69ac5dd6 100644 --- a/desktop/src-tauri/src/huddle/preprocessing.rs +++ b/desktop/src-tauri/src/huddle/preprocessing.rs @@ -205,6 +205,12 @@ fn strip_inline_code(text: &str) -> String { } /// Replace http/https URLs with "link omitted". +/// +/// Trailing sentence-ending punctuation (`.`, `!`, `?`) that immediately follows +/// a URL and is at end-of-string or followed by whitespace is preserved so that +/// sentence splitting and TTS prosody are not degraded. +/// +/// Example: `"See https://x.y/z."` → `"See link omitted."` fn strip_urls(text: &str) -> String { let mut out = String::with_capacity(text.len()); let mut rest = text; @@ -223,12 +229,31 @@ fn strip_urls(text: &str) -> String { }; out.push_str(&rest[..url_start]); rest = &rest[url_start..]; - // Consume until whitespace or end. + // Consume until whitespace or structural delimiter. let url_end = rest .find(|c: char| c.is_whitespace() || c == ')' || c == ']' || c == '"' || c == '\'') .unwrap_or(rest.len()); + let url_token = &rest[..url_end]; rest = &rest[url_end..]; + + // Check if the URL token ends with sentence-ending punctuation that + // belongs to the surrounding sentence rather than the URL itself. + // A trailing `.`, `!`, or `?` is preserved when it is at end-of-string + // or followed by whitespace (i.e. it is a sentence boundary). + let trailing_punct = if url_token.ends_with(|c: char| matches!(c, '.' | '!' | '?')) { + let after = rest; // rest is already past url_end + if after.is_empty() || after.starts_with(|c: char| c.is_whitespace()) { + // Preserve the trailing punctuation. + &url_token[url_token.len() - 1..] + } else { + "" + } + } else { + "" + }; + out.push_str("link omitted"); + out.push_str(trailing_punct); } out } @@ -492,6 +517,32 @@ mod tests { assert!(!out.contains("example.com"), "got: {out}"); } + #[test] + fn strips_url_preserves_trailing_period() { + // Trailing `.` at end of sentence must be preserved for sentence splitting. + let out = strip_urls("See https://x.y/z."); + assert_eq!(out, "See link omitted.", "got: {out}"); + } + + #[test] + fn strips_url_preserves_trailing_exclamation() { + let out = strip_urls("Visit https://example.com!"); + assert_eq!(out, "Visit link omitted!", "got: {out}"); + } + + #[test] + fn strips_url_preserves_trailing_question() { + let out = strip_urls("Did you see https://example.com?"); + assert_eq!(out, "Did you see link omitted?", "got: {out}"); + } + + #[test] + fn strips_url_mid_sentence_no_punct_preserved() { + // URL in the middle of a sentence — no trailing punct to preserve. + let out = strip_urls("Check https://example.com for more info."); + assert_eq!(out, "Check link omitted for more info.", "got: {out}"); + } + #[test] fn strips_bold_italic() { let out = preprocess_for_tts("**bold** and *italic* and _under_"); diff --git a/desktop/src-tauri/src/huddle/state.rs b/desktop/src-tauri/src/huddle/state.rs index dfd837bd9d..68d105337d 100644 --- a/desktop/src-tauri/src/huddle/state.rs +++ b/desktop/src-tauri/src/huddle/state.rs @@ -207,6 +207,39 @@ impl HuddleState { } } +// ── Event emission ──────────────────────────────────────────────────────────── + +/// Emit the current huddle state to the frontend via a Tauri event. +/// +/// The frontend listens for `"huddle-state-changed"` and updates its UI +/// immediately, replacing the previous 2-second polling loop. +/// +/// **Call sites** — call this in `huddle/mod.rs` after every state transition +/// that the frontend needs to observe: +/// - After `phase` changes (Creating → Connecting → Connected → Active → Leaving → Idle) +/// - After `participants` is updated (join/leave) +/// - After `tts_enabled` is toggled +/// +/// **Usage in mod.rs:** +/// ```rust +/// // After mutating huddle state, while still holding the lock: +/// let snapshot = hs.clone(); +/// drop(hs); // release lock before emitting +/// if let Ok(guard) = state.app_handle.lock() { +/// if let Some(app) = guard.as_ref() { +/// emit_huddle_state(app, &snapshot); +/// } +/// } +/// ``` +/// +/// Best-effort — silently ignores errors (e.g., no listeners attached yet). +// Not yet called from mod.rs — suppress dead_code until that integration lands. +#[allow(dead_code)] +pub fn emit_huddle_state(app: &tauri::AppHandle, state: &HuddleState) { + use tauri::Emitter; + let _ = app.emit("huddle-state-changed", state); +} + // ── Response types ──────────────────────────────────────────────────────────── /// Returned by start_huddle and join_huddle. diff --git a/desktop/src-tauri/src/huddle/tts.rs b/desktop/src-tauri/src/huddle/tts.rs index 4617c1e121..054121cb46 100644 --- a/desktop/src-tauri/src/huddle/tts.rs +++ b/desktop/src-tauri/src/huddle/tts.rs @@ -65,8 +65,8 @@ const FADE_SAMPLES: usize = (SAMPLE_RATE as f64 * 0.008) as usize; /// Sentence-by-sentence synthesis for lower TTFA (≈200ms vs ≈600ms for 3-sentence batches). const BATCH_SIZE: usize = 1; -/// Silence inserted between sentences by the Kokoro engine (seconds). -/// Kokoro handles its own inter-sentence silence. +/// Silence inserted between sentences by the TTS pipeline (seconds). +/// Injected as a silent buffer between each synthesized sentence chunk. const INTER_SENTENCE_SILENCE: f32 = 0.1; // ── Public pipeline handle ──────────────────────────────────────────────────── @@ -330,6 +330,8 @@ fn tts_worker( // Lookahead pipeline: synthesize each sentence and append immediately. // Rodio queues buffers sequentially — synthesis of the next sentence // overlaps with playback of the current one. + let silence_samples = (INTER_SENTENCE_SILENCE * SAMPLE_RATE as f32) as usize; + let silence_buf = vec![0.0f32; silence_samples]; for sentence in &sentences { if handle_cancel_or_shutdown(&cancel, &shutdown, &tts_active, &text_rx, Some(&player)) { break; @@ -340,22 +342,16 @@ fn tts_worker( continue; } - match engine.call( - text, - "en", - &style, - SYNTH_STEPS, - SYNTH_SPEED, - INTER_SENTENCE_SILENCE, - ) { + match engine.synth_chunk(text, "en", &style, SYNTH_STEPS, SYNTH_SPEED) { Ok(samples) if !samples.is_empty() => { let mut boosted: Vec = samples .iter() .map(|&s| (s * VOLUME_BOOST).clamp(-1.0, 1.0)) .collect(); apply_fades(&mut boosted); - let buf = SamplesBuffer::new(channels, rate, boosted); - player.append(buf); + player.append(SamplesBuffer::new(channels, rate, boosted)); + // Insert inter-sentence silence after each synthesized chunk. + player.append(SamplesBuffer::new(channels, rate, silence_buf.clone())); if first_append { tts_active.store(true, Ordering::Release); first_append = false; diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 0f78e5625f..b04648366a 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -401,6 +401,13 @@ pub fn run() { .store(true, std::sync::atomic::Ordering::Release); } } + // Emit ptt-state=true to the frontend. + // The React side plays the press audio cue on this event + // (Web Audio API via HuddleContext). Rust-side rodio audio + // was considered but rejected: the rodio OutputStream must + // outlive the handler and sharing it across the shortcut + // closure adds lifecycle complexity for marginal gain. + // The React implementation is sufficient and simpler. let _ = app.emit("ptt-state", true); } ShortcutState::Released => { @@ -425,6 +432,7 @@ pub fn run() { .store(false, std::sync::atomic::Ordering::Release); } } + // Emit ptt-state=false — React plays the release audio cue. let _ = app_handle.emit("ptt-state", false); }); } @@ -471,6 +479,14 @@ pub fn run() { // that will be lost on restart, as that silently breaks channel // memberships, DMs, and relay identity. let state = app_handle.state::(); + + // Store the AppHandle so huddle commands can emit `huddle-state-changed` + // events via `huddle::emit_huddle_state` without threading the handle + // through every call site. + if let Ok(mut guard) = state.app_handle.lock() { + *guard = Some(app_handle.clone()); + } + resolve_persisted_identity(&app_handle, &state) .map_err(|e| -> Box { e.into() })?; diff --git a/desktop/src/features/huddle/HuddleContext.tsx b/desktop/src/features/huddle/HuddleContext.tsx index 1ded589065..57221df2a0 100644 --- a/desktop/src/features/huddle/HuddleContext.tsx +++ b/desktop/src/features/huddle/HuddleContext.tsx @@ -8,29 +8,10 @@ import { setupAudioWorklet, type AudioWorkletHandle } from "./lib/audioWorklet"; /** * Huddle lifecycle (React context): - * - * startHuddle(channelId, agents) - * → invoke("start_huddle") [Rust: ephemeral channel + LiveKit token] - * → connectToHuddle(url, token) [LiveKit: WebRTC room + mic] - * → setupAudioWorklet(track, init) [AudioWorklet: mic PCM → Rust STT, PTT gating] - * → invoke("confirm_huddle_active") [Rust: Connected → Active] - * → setEphemeralChannelId(...) [triggers TTS subscription + hotstart polling] - * - * joinHuddle(parentChannelId, ephemeralChannelId, livekitRoom) - * → invoke("join_huddle") [Rust: get LiveKit token for existing room] - * → connectToHuddle(url, token) [LiveKit: WebRTC room + mic] - * → setupAudioWorklet(track, init) [AudioWorklet: mic PCM → Rust STT, PTT gating] - * → invoke("confirm_huddle_active") [Rust: Connected → Active] - * → setEphemeralChannelId(...) [triggers TTS subscription + hotstart polling] - * - * TTS subscription (on ephemeralChannelId change): - * → relayClient.subscribeToChannelLive(ephId, callback) - * → live-only (since: now) — no historical backlog - * → filter: agent pubkeys only (fail-closed), skip self - * → invoke("speak_agent_message", { text }) - * - * leaveHuddle() - * → stop AudioWorklet → disconnect LiveKit → invoke("leave_huddle") + * startHuddle/joinHuddle → invoke(start/join_huddle) → connectToHuddle (LiveKit+mic) + * → setupAudioWorklet (PCM→STT, PTT gating) → confirm_huddle_active + * TTS subscription: subscribeToChannelLive → filter agent pubkeys → speak_agent_message + * leaveHuddle: stop worklet → disconnect LiveKit → invoke(leave_huddle) */ type HuddleJoinInfo = { @@ -47,6 +28,10 @@ interface HuddleContextValue { localAudioTrack: MediaStreamTrack | null; /** Whether a huddle is being started (for button disabled state) */ isStarting: boolean; + /** Last start/join error message — display in UI and clear with clearHuddleError */ + huddleError: string | null; + /** Dismiss the current huddleError */ + clearHuddleError: () => void; /** Whether the LiveKit + mic connection is live */ micConnected: boolean; /** Current mic input level 0–1 (updated via requestAnimationFrame) */ @@ -91,6 +76,8 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { const [localAudioTrack, setLocalAudioTrack] = React.useState(null); const [isStarting, setIsStarting] = React.useState(false); + const [huddleError, setHuddleError] = React.useState(null); + const clearHuddleError = React.useCallback(() => setHuddleError(null), []); const [micConnected, setMicConnected] = React.useState(false); const [micLevel, setMicLevel] = React.useState(0); /** Whether the PTT key is currently held */ @@ -123,25 +110,39 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { }); }, []); - // Listen for PTT state from Rust global shortcut (Ctrl+Space). - // Updates pttActive for UI feedback (green indicator in HuddleBar). - // The actual audio gating happens in audioWorklet.ts → worklet.js. + // PTT state from Rust (Ctrl+Space). UI feedback + 50ms audio cue when mic active. + // Actual audio gating is in audioWorklet.ts → worklet.js. React.useEffect(() => { let cancelled = false; let unlisten: (() => void) | null = null; - listen("ptt-state", (event) => { - if (!cancelled) setPttActive(event.payload); + if (cancelled) return; + setPttActive(event.payload); + if (micConnected) { + try { + const ac = new AudioContext(); + const osc = ac.createOscillator(); + const g = ac.createGain(); + osc.connect(g); + g.connect(ac.destination); + osc.frequency.value = event.payload ? 880 : 440; + g.gain.value = 0.05; + osc.start(); + osc.stop(ac.currentTime + 0.05); + osc.onended = () => void ac.close(); + } catch { + /* best-effort */ + } + } }).then((fn) => { if (cancelled) fn(); else unlisten = fn; }); - return () => { cancelled = true; unlisten?.(); }; - }, []); + }, [micConnected]); // Toggle voice input mode — persists to Rust backend and updates worklet gating. const setVoiceInputMode = React.useCallback(async (mode: VoiceInputMode) => { @@ -361,8 +362,6 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { memberPubkeys, }); rustActiveRef.current = true; - // Do NOT set ephemeralChannelId yet — wait until fully established (LiveKit + Worklet) - // Step 2-4: Connect LiveKit, setup AudioWorklet, confirm active try { await connectAndSetupMedia(joinInfo, myToken); @@ -378,11 +377,12 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { throw e; } } catch (e) { - // Pass workletRef.current — it may have been assigned before the error - // (e.g. confirm_huddle_active rejects after worklet setup succeeded). + // workletRef.current may have been assigned before the error const w = workletRef.current; workletRef.current = null; await cleanupFailedStart(connectionRef.current, w, true); + const msg = e instanceof Error ? e.message : String(e); + setHuddleError(msg); console.error("Failed to start huddle:", e); throw e; } finally { @@ -432,6 +432,8 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { const w = workletRef.current; workletRef.current = null; await cleanupFailedStart(connectionRef.current, w, false); + const msg = e instanceof Error ? e.message : String(e); + setHuddleError(msg); console.error("Failed to join huddle:", e); throw e; } finally { @@ -516,7 +518,6 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { if (event.content.trim().length <= 1) return; // Legacy: skip [System]-prefixed messages from before kind:48106. if (event.content.startsWith("[System]")) return; - invoke("speak_agent_message", { text: event.content }).catch((err) => { console.warn( "[huddle] TTS speak failed (backpressure or pipeline unavailable):", @@ -599,6 +600,8 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { value={{ localAudioTrack, isStarting, + huddleError, + clearHuddleError, micConnected, micLevel, pttActive, diff --git a/desktop/src/features/huddle/components/HuddleBar.tsx b/desktop/src/features/huddle/components/HuddleBar.tsx index f4932dc959..aca1acf1aa 100644 --- a/desktop/src/features/huddle/components/HuddleBar.tsx +++ b/desktop/src/features/huddle/components/HuddleBar.tsx @@ -1,4 +1,5 @@ import { invoke } from "@tauri-apps/api/core"; +import { listen } from "@tauri-apps/api/event"; import { Mic, MicOff, @@ -53,6 +54,8 @@ export function HuddleBar({ className }: HuddleBarProps) { setVoiceInputMode, activeSpeakers, isReconnecting, + huddleError, + clearHuddleError, } = useHuddle(); const isPttMode = voiceInputMode === "push_to_talk"; @@ -69,17 +72,17 @@ export function HuddleBar({ className }: HuddleBarProps) { kokoro: string; } | null>(null); - // Poll huddle state — replace with event listener once Rust emits events + // Huddle state: event-driven primary path + 10s fallback poll. React.useEffect(() => { let cancelled = false; + let unlisten: (() => void) | null = null; - async function poll() { + async function fetchState() { try { const s = await invoke("get_huddle_state"); if (!cancelled) setState(s); } catch { // Only clear state if we never had an active huddle. - // Transient errors shouldn't remove the control bar. if (!cancelled) { setState((prev) => prev?.phase === "active" || prev?.phase === "connected" @@ -90,11 +93,23 @@ export function HuddleBar({ className }: HuddleBarProps) { } } - void poll(); - const id = window.setInterval(() => void poll(), 2_000); + // Initial fetch + void fetchState(); + + // Primary: listen for Rust-emitted state change events + listen("huddle-state-changed", (event) => { + if (!cancelled) setState(event.payload); + }).then((fn) => { + if (cancelled) fn(); + else unlisten = fn; + }); + + // Fallback: 10s poll in case events are missed + const id = window.setInterval(() => void fetchState(), 10_000); return () => { cancelled = true; + unlisten?.(); window.clearInterval(id); }; }, []); @@ -177,6 +192,10 @@ export function HuddleBar({ className }: HuddleBarProps) { async function handleEnd() { if (isLeaving) return; + const confirmed = window.confirm( + "End the huddle for everyone? This will disconnect all participants.", + ); + if (!confirmed) return; setIsLeaving(true); try { const backendClean = await endHuddle(); @@ -200,6 +219,24 @@ export function HuddleBar({ className }: HuddleBarProps) { className, )} > + {/* Error banner — dismissible, shown when start/join fails */} + {huddleError && ( +
    + {huddleError} + +
    + )} + {/* Room label */} Huddle @@ -237,6 +274,7 @@ export function HuddleBar({ className }: HuddleBarProps) { )} @@ -290,6 +328,7 @@ export function HuddleBar({ className }: HuddleBarProps) { ? "Switch to voice activity mode" : "Switch to push-to-talk mode" } + aria-pressed={isPttMode} className="h-6 px-1.5 text-[10px]" onClick={() => void setVoiceInputMode(isPttMode ? "voice_activity" : "push_to_talk") @@ -394,10 +433,11 @@ export function HuddleBar({ className }: HuddleBarProps) { aria-label="Leave huddle" className="h-8 w-8" disabled={isLeaving} + aria-busy={isLeaving} onClick={() => void handleLeave()} size="icon" variant="destructive" - title="Leave huddle" + title="Leave huddle (press Escape to dismiss dialogs first)" > @@ -423,6 +463,7 @@ export function HuddleBar({ className }: HuddleBarProps) { : micConnected ? "In huddle, microphone connected" : "In huddle, no microphone"} + {`, voice input: ${isPttMode ? "push to talk, press Ctrl+Space to transmit" : "voice activity detection"}`} {modelStatus && modelStatus.moonshine !== "ready" && `, STT model ${modelStatus.moonshine}`} diff --git a/desktop/src/features/huddle/components/ParticipantList.tsx b/desktop/src/features/huddle/components/ParticipantList.tsx index ad12b4e484..f53666c858 100644 --- a/desktop/src/features/huddle/components/ParticipantList.tsx +++ b/desktop/src/features/huddle/components/ParticipantList.tsx @@ -1,3 +1,5 @@ +import * as React from "react"; + import { cn } from "@/shared/lib/cn"; import { useUsersBatchQuery } from "@/features/profile/hooks"; import { ProfileAvatar } from "@/features/profile/ui/ProfileAvatar"; @@ -6,54 +8,73 @@ type ParticipantListProps = { /** Pubkey hex strings from the Rust huddle state */ participants: string[]; activeSpeakers?: string[]; + /** Pubkeys of agent participants — rendered with a bot badge */ + agentPubkeys?: string[]; className?: string; }; export function ParticipantList({ participants, activeSpeakers, + agentPubkeys, className, }: ParticipantListProps) { const { data } = useUsersBatchQuery(participants); const profiles = data?.profiles ?? {}; + const agentSet = React.useMemo( + () => new Set(agentPubkeys ?? []), + [agentPubkeys], + ); if (participants.length === 0) return null; return ( -
    +
      {participants.map((pubkey) => { const profile = profiles[pubkey.toLowerCase()]; const hasProfile = profile?.displayName || profile?.avatarUrl; const isActive = activeSpeakers?.includes(pubkey); - const ariaLabel = - profile?.displayName || `Participant ${pubkey.slice(0, 8)}`; + const isAgent = agentSet.has(pubkey); + const ariaLabel = `${profile?.displayName || `Participant ${pubkey.slice(0, 8)}`}${isAgent ? " (agent)" : ""}`; - return hasProfile ? ( -
      - -
      - ) : ( - + return ( +
    • + {hasProfile ? ( +
      + +
      + ) : ( + + )} + {isAgent && ( + + )} +
      • ); })} -
    +
); } @@ -61,9 +82,11 @@ export function ParticipantList({ function HexAvatar({ pubkey, activeSpeakers, + ariaLabel, }: { pubkey: string; activeSpeakers?: string[]; + ariaLabel?: string; }) { const shortId = pubkey.slice(0, 6).toUpperCase(); const parsed = parseInt(pubkey.slice(0, 4), 16); @@ -73,7 +96,7 @@ function HexAvatar({ return (
Date: Tue, 14 Apr 2026 10:52:02 -0400 Subject: [PATCH 38/41] feat(huddle): relay-side auto-add for huddle joiners (Option D) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a second human joins a huddle, they need membership in the private ephemeral channel to get a LiveKit token. Previously join_huddle tried to self-add via kind:9000, which the relay correctly rejected (only owners/admins can add members to private channels). Now the relay's huddle_token endpoint handles this automatically: - Client passes parent_channel_id as a query parameter - Relay verifies caller is a member of the parent channel - Relay verifies target is a private ephemeral channel (ttl_seconds IS NOT NULL) - Relay auto-adds caller via db.add_member() with the channel creator as invited_by - Then issues the LiveKit token normally Safety gates (ALL must be true for auto-add): 1. parent_channel_id query param is present 2. Caller IS a member of the parent channel 3. Target channel visibility == private 4. Target channel ttl_seconds IS NOT NULL (ephemeral) 5. Only fires inside /api/huddles/ endpoint (not generic channel API) Client changes: - join_huddle: removed self-add block, passes parent_channel_id to token endpoint - start_huddle: passes None (creator is already owner, no auto-add needed) - Simplified rollback (no channel leave needed — TTL handles cleanup) --- crates/sprout-relay/src/api/huddles.rs | 63 +++++++++++++++++- desktop/src-tauri/src/huddle/mod.rs | 81 +++++++---------------- desktop/src-tauri/src/huddle/relay_api.rs | 11 ++- 3 files changed, 96 insertions(+), 59 deletions(-) diff --git a/crates/sprout-relay/src/api/huddles.rs b/crates/sprout-relay/src/api/huddles.rs index 3ef0bba37a..d83ca2c000 100644 --- a/crates/sprout-relay/src/api/huddles.rs +++ b/crates/sprout-relay/src/api/huddles.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use axum::{ - extract::{Path, State}, + extract::{Path, Query, State}, http::{HeaderMap, StatusCode}, response::Json, }; @@ -19,6 +19,17 @@ use super::{ }; use crate::state::AppState; +/// Query parameters for [`huddle_token`]. +#[derive(serde::Deserialize)] +pub struct HuddleTokenQuery { + /// The parent (non-ephemeral) channel this huddle belongs to. + /// + /// When provided and the caller is a member of the parent channel, the relay + /// will auto-add them to the private ephemeral huddle channel so they can + /// obtain a token without requiring an explicit invite. + pub parent_channel_id: Option, +} + /// `POST /api/huddles/{channel_id}/token` — generate a LiveKit access token. /// /// Returns `{ "token": "", "url": "", "room": "sprout-" }`. @@ -28,6 +39,7 @@ pub async fn huddle_token( State(state): State>, headers: HeaderMap, Path(channel_id): Path, + Query(query): Query, ) -> Result, (StatusCode, Json)> { let ctx = extract_auth_context(&headers, &state).await?; sprout_auth::require_scope(&ctx.scopes, sprout_auth::Scope::MessagesRead) @@ -46,7 +58,54 @@ pub async fn huddle_token( })?; // Verify the caller is a member of the channel (or it's an open channel). - check_channel_membership(&state, channel_id, &ctx.pubkey_bytes).await?; + // If they're not a member, attempt relay-side auto-add when: + // 1. parent_channel_id was provided + // 2. The target channel is private + ephemeral (ttl_seconds IS NOT NULL) + // 3. The caller IS a member of the parent channel + let membership_result = check_channel_membership(&state, channel_id, &ctx.pubkey_bytes).await; + + if let Err(ref membership_err) = membership_result { + if let Some(parent_id) = query.parent_channel_id { + // Gate 1: caller must be a member of the parent channel. + check_channel_membership(&state, parent_id, &ctx.pubkey_bytes).await?; + + // Gate 2: target channel must be private + ephemeral. + let channel = state + .db + .get_channel(channel_id) + .await + .map_err(|e| internal_error(&format!("db error: {e}")))?; + + if channel.visibility == "private" && channel.ttl_seconds.is_some() { + // Auto-add: use the channel creator as invited_by (truthful attribution). + // The creator is always an active owner, satisfying add_member's invite check. + state + .db + .add_member( + channel_id, + &ctx.pubkey_bytes, + sprout_db::channel::MemberRole::Member, + Some(&channel.created_by), + ) + .await + .map_err(|e| internal_error(&format!("auto-add failed: {e}")))?; + + tracing::info!( + "Huddle auto-add: added {} to ephemeral channel {} (parent: {})", + ctx.pubkey.to_hex(), + channel_id, + parent_id + ); + // Fall through to token generation. + } else { + // Not a private ephemeral channel — return the original 403. + return Err(membership_err.clone()); + } + } else { + // No parent_channel_id provided — return the original 403. + membership_result?; + } + } // Generate the LiveKit token. let room = sprout_huddle::HuddleService::create_room_name(channel_id); diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index 039d3a88a9..8c3032be58 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -220,7 +220,8 @@ pub async fn start_huddle( // 4. Fetch LiveKit token BEFORE emitting HUDDLE_STARTED. // This prevents a phantom announcement if the token fetch fails. - let lk = fetch_livekit_token(&ephemeral_channel_id, &state).await?; + // Creator is already the channel owner — no auto-add needed (None). + let lk = fetch_livekit_token(&ephemeral_channel_id, None, &state).await?; // 5. Emit HUDDLE_STARTED to parent channel — only now that token is confirmed. let started_builder = @@ -301,10 +302,12 @@ pub async fn start_huddle( /// Join an existing huddle in the given parent channel. /// /// Steps: -/// 0. Add the joining human to the ephemeral channel (required for STT transcript posting). -/// 1. Fetch a LiveKit token from the relay for the ephemeral channel. -/// 2. Emit KIND_HUDDLE_PARTICIPANT_JOINED to the parent channel (best-effort). -/// 3. Store state and return join info. +/// 1. Transition to Connecting. +/// 2. Fetch a LiveKit token from the relay (relay auto-adds caller as member +/// of the ephemeral channel via `parent_channel_id` query param). +/// 3. Emit KIND_HUDDLE_PARTICIPANT_JOINED to the parent channel (best-effort). +/// 4. Store state and return join info. +/// 5. Post-connect setup (pipelines, model hydration). #[tauri::command] pub async fn join_huddle( parent_channel_id: String, @@ -327,47 +330,28 @@ pub async fn join_huddle( hs.livekit_room = Some(livekit_room.clone()); } - // All steps wrapped so we can roll back on ANY failure after phase transition. - // Tracks whether THIS attempt added membership (vs "already a member"). - // On rollback, only leave the channel if we actually added ourselves. - let mut membership_added = false; let result: Result<(LiveKitTokenResponse, String), String> = async { - // 0. Add the joining human to the ephemeral channel. + // 1. Fetch LiveKit token — relay auto-adds caller to the ephemeral channel + // when parent_channel_id is provided (caller must be a parent member). + let lk = + fetch_livekit_token(&ephemeral_channel_id, Some(&parent_channel_id), &state).await?; + + // 2. Emit PARTICIPANT_JOINED (best-effort) — include own pubkey as p-tag. + // Fetch own_pubkey AFTER the token call so a key-lock failure doesn't + // block the join; the event is best-effort anyway. let own_pubkey = state .keys .lock() .map(|k| k.public_key().to_hex()) - .map_err(|_| "keys unavailable".to_string())?; - let eph_uuid = parse_channel_uuid(&ephemeral_channel_id)?; - let add_self = events::build_add_member(eph_uuid, &own_pubkey, None)?; - match submit_event(add_self, &state).await { - Ok(_) => { - membership_added = true; - } // Newly added. - Err(e) => { - // Idempotent: "already a member" is expected on rejoin. - let is_idempotent = { - let lower = e.to_lowercase(); - lower.contains("already a member") || lower.contains("already_member") - }; - if !is_idempotent { - eprintln!( - "sprout-desktop: join_huddle add self to ephemeral channel failed: {e}" - ); - return Err(format!("failed to join ephemeral channel: {e}")); - } - // Already a member — don't leave on rollback. - } - }; - - // 1. Fetch LiveKit token. - let lk = fetch_livekit_token(&ephemeral_channel_id, &state).await?; - - // 2. Emit PARTICIPANT_JOINED (best-effort) — include own pubkey as p-tag. + .unwrap_or_default(); if let Ok(joined_builder) = events::build_huddle_participant_joined( &parent_channel_id, &ephemeral_channel_id, - Some(&own_pubkey), + if own_pubkey.is_empty() { + None + } else { + Some(own_pubkey.as_str()) + }, ) { if let Err(e) = submit_event(joined_builder, &state).await { eprintln!("sprout-desktop: huddle_participant_joined event failed: {e}"); @@ -409,19 +393,8 @@ pub async fn join_huddle( }) } Err(e) => { - // Rollback: remove self from ephemeral channel to avoid orphaned - // membership (e.g. token fetch failed after membership succeeded). - // Only leave if THIS attempt added us — don't kick a pre-existing member. - if membership_added { - if let Ok(eph_uuid) = parse_channel_uuid(&ephemeral_channel_id) { - if let Ok(leave_builder) = events::build_leave(eph_uuid) { - if let Err(le) = submit_event(leave_builder, &state).await { - eprintln!("sprout-desktop: join_huddle rollback leave failed: {le}"); - } - } - } - } - // Reset state to Idle so the user can retry. + // Rollback: just reset state to Idle — the relay auto-added us and + // the ephemeral channel has a TTL, so no manual leave is needed. if let Ok(mut hs) = state.huddle_state.lock() { hs.reset_preserving_generation(); } @@ -513,11 +486,7 @@ pub async fn leave_huddle(state: State<'_, AppState>) -> Result<(), String> { }; // Emit PARTICIPANT_LEFT (best-effort) — include own pubkey as p-tag. - let own_pubkey = state - .keys - .lock() - .ok() - .map(|k| k.public_key().to_hex()); + let own_pubkey = state.keys.lock().ok().map(|k| k.public_key().to_hex()); if !parent_channel_id.is_empty() && !ephemeral_channel_id.is_empty() { if let Ok(left_builder) = events::build_huddle_participant_left( &parent_channel_id, diff --git a/desktop/src-tauri/src/huddle/relay_api.rs b/desktop/src-tauri/src/huddle/relay_api.rs index 1879f4a583..c3ccc54c34 100644 --- a/desktop/src-tauri/src/huddle/relay_api.rs +++ b/desktop/src-tauri/src/huddle/relay_api.rs @@ -29,11 +29,20 @@ pub(crate) fn parse_channel_uuid(channel_id: &str) -> Result { } /// Fetch a LiveKit token from the relay for the given channel. +/// +/// When `parent_channel_id` is `Some`, appends `?parent_channel_id={id}` to +/// the URL so the relay can auto-add the caller as a member of the ephemeral +/// channel (used by joiners — creators are already owners and pass `None`). pub(crate) async fn fetch_livekit_token( channel_id: &str, + parent_channel_id: Option<&str>, state: &AppState, ) -> Result { - let path = api_path(&["huddles", channel_id, "token"]); + let base = api_path(&["huddles", channel_id, "token"]); + let path = match parent_channel_id { + Some(pid) => format!("{base}?parent_channel_id={pid}"), + None => base, + }; let request = build_authed_request(&state.http_client, Method::POST, &path, state)?; send_json_request(request).await } From 57be7e7fb706598f3b5e6bb0f333a3905c05bc42 Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Tue, 14 Apr 2026 12:07:24 -0400 Subject: [PATCH 39/41] =?UTF-8?q?fix(huddle):=20crossfire=20round=204=20?= =?UTF-8?q?=E2=80=94=20integrity,=20perf,=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crossfire review: opus 8/10 APPROVE ×2, codex 3/10 triaged (2 real, 3 Phase 2 scope). QF1: Add cmudict.dict SHA-256 to KOKORO_FILE_HASHES — closes the one unverified model artifact in the integrity pipeline. QF2: Reduce LiveKit token TTL from 6h to 1h — match ephemeral channel TTL. A 6-hour token for a 1-hour channel is unnecessary attack surface. QF3: Remove stale #[allow(dead_code)] on emit_huddle_state and the broken doctest example. Function IS called from AppState. Updated doc comment to reference the actual call site. QF4: Reuse one AudioContext for PTT audio cues instead of creating a new one per press/release. Prevents exhausting Chrome's ~6 concurrent AudioContext limit during long huddles. QF5: Throttle micLevel updates from 60fps to ~10fps. Voice meters don't need 60fps visual fidelity, and setMicLevel was re-rendering the entire HuddleBar on every requestAnimationFrame tick. Bonus: Remove dead KokoroTTS::call() method (-34 lines) and its unused split_sentences import. The TTS pipeline uses synth_chunk() directly for lookahead pipelining — call() was never invoked. Fix: Replace block-artifacts.com tarball URLs in pnpm-lock.yaml with public registry.npmjs.org (local ~/.npmrc leaked corporate registry into lockfile resolution). --- crates/sprout-huddle/src/token.rs | 2 +- desktop/pnpm-lock.yaml | 172 +++++++++--------- desktop/scripts/check-file-sizes.mjs | 2 +- desktop/src-tauri/src/huddle/kokoro.rs | 38 +--- desktop/src-tauri/src/huddle/models.rs | 1 + desktop/src-tauri/src/huddle/state.rs | 22 +-- desktop/src/features/huddle/HuddleContext.tsx | 27 ++- 7 files changed, 116 insertions(+), 148 deletions(-) diff --git a/crates/sprout-huddle/src/token.rs b/crates/sprout-huddle/src/token.rs index 611a62cda0..ac286f87f8 100644 --- a/crates/sprout-huddle/src/token.rs +++ b/crates/sprout-huddle/src/token.rs @@ -50,7 +50,7 @@ pub fn generate_token( ttl: Option, ) -> Result { let now = Utc::now(); - let ttl = ttl.unwrap_or_else(|| Duration::hours(6)); + let ttl = ttl.unwrap_or_else(|| Duration::hours(1)); let expires_at = now + ttl; let claims = LiveKitClaims { diff --git a/desktop/pnpm-lock.yaml b/desktop/pnpm-lock.yaml index ef04f07109..83a60e7343 100644 --- a/desktop/pnpm-lock.yaml +++ b/desktop/pnpm-lock.yaml @@ -264,59 +264,59 @@ packages: hasBin: true '@biomejs/cli-darwin-arm64@2.4.6': - resolution: {integrity: sha512-NW18GSyxr+8sJIqgoGwVp5Zqm4SALH4b4gftIA0n62PTuBs6G2tHlwNAOj0Vq0KKSs7Sf88VjjmHh0O36EnzrQ==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-2.4.6.tgz} + resolution: {integrity: sha512-NW18GSyxr+8sJIqgoGwVp5Zqm4SALH4b4gftIA0n62PTuBs6G2tHlwNAOj0Vq0KKSs7Sf88VjjmHh0O36EnzrQ==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [darwin] '@biomejs/cli-darwin-x64@2.4.6': - resolution: {integrity: sha512-4uiE/9tuI7cnjtY9b07RgS7gGyYOAfIAGeVJWEfeCnAarOAS7qVmuRyX6d7JTKw28/mt+rUzMasYeZ+0R/U1Mw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@biomejs/cli-darwin-x64/-/cli-darwin-x64-2.4.6.tgz} + resolution: {integrity: sha512-4uiE/9tuI7cnjtY9b07RgS7gGyYOAfIAGeVJWEfeCnAarOAS7qVmuRyX6d7JTKw28/mt+rUzMasYeZ+0R/U1Mw==} engines: {node: '>=14.21.3'} cpu: [x64] os: [darwin] '@biomejs/cli-linux-arm64-musl@2.4.6': - resolution: {integrity: sha512-F/JdB7eN22txiTqHM5KhIVt0jVkzZwVYrdTR1O3Y4auBOQcXxHK4dxULf4z43QyZI5tsnQJrRBHZy7wwtL+B3A==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.4.6.tgz} + resolution: {integrity: sha512-F/JdB7eN22txiTqHM5KhIVt0jVkzZwVYrdTR1O3Y4auBOQcXxHK4dxULf4z43QyZI5tsnQJrRBHZy7wwtL+B3A==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] libc: [musl] '@biomejs/cli-linux-arm64@2.4.6': - resolution: {integrity: sha512-kMLaI7OF5GN1Q8Doymjro1P8rVEoy7BKQALNz6fiR8IC1WKduoNyteBtJlHT7ASIL0Cx2jR6VUOBIbcB1B8pew==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@biomejs/cli-linux-arm64/-/cli-linux-arm64-2.4.6.tgz} + resolution: {integrity: sha512-kMLaI7OF5GN1Q8Doymjro1P8rVEoy7BKQALNz6fiR8IC1WKduoNyteBtJlHT7ASIL0Cx2jR6VUOBIbcB1B8pew==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] libc: [glibc] '@biomejs/cli-linux-x64-musl@2.4.6': - resolution: {integrity: sha512-C9s98IPDu7DYarjlZNuzJKTjVHN03RUnmHV5htvqsx6vEUXCDSJ59DNwjKVD5XYoSS4N+BYhq3RTBAL8X6svEg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-2.4.6.tgz} + resolution: {integrity: sha512-C9s98IPDu7DYarjlZNuzJKTjVHN03RUnmHV5htvqsx6vEUXCDSJ59DNwjKVD5XYoSS4N+BYhq3RTBAL8X6svEg==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] libc: [musl] '@biomejs/cli-linux-x64@2.4.6': - resolution: {integrity: sha512-oHXmUFEoH8Lql1xfc3QkFLiC1hGR7qedv5eKNlC185or+o4/4HiaU7vYODAH3peRCfsuLr1g6v2fK9dFFOYdyw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@biomejs/cli-linux-x64/-/cli-linux-x64-2.4.6.tgz} + resolution: {integrity: sha512-oHXmUFEoH8Lql1xfc3QkFLiC1hGR7qedv5eKNlC185or+o4/4HiaU7vYODAH3peRCfsuLr1g6v2fK9dFFOYdyw==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] libc: [glibc] '@biomejs/cli-win32-arm64@2.4.6': - resolution: {integrity: sha512-xzThn87Pf3YrOGTEODFGONmqXpTwUNxovQb72iaUOdcw8sBSY3+3WD8Hm9IhMYLnPi0n32s3L3NWU6+eSjfqFg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@biomejs/cli-win32-arm64/-/cli-win32-arm64-2.4.6.tgz} + resolution: {integrity: sha512-xzThn87Pf3YrOGTEODFGONmqXpTwUNxovQb72iaUOdcw8sBSY3+3WD8Hm9IhMYLnPi0n32s3L3NWU6+eSjfqFg==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [win32] '@biomejs/cli-win32-x64@2.4.6': - resolution: {integrity: sha512-7++XhnsPlr1HDbor5amovPjOH6vsrFOCdp93iKXhFn6bcMUI6soodj3WWKfgEO6JosKU1W5n3uky3WW9RlRjTg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@biomejs/cli-win32-x64/-/cli-win32-x64-2.4.6.tgz} + resolution: {integrity: sha512-7++XhnsPlr1HDbor5amovPjOH6vsrFOCdp93iKXhFn6bcMUI6soodj3WWKfgEO6JosKU1W5n3uky3WW9RlRjTg==} engines: {node: '>=14.21.3'} cpu: [x64] os: [win32] '@bufbuild/protobuf@1.10.1': - resolution: {integrity: sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@bufbuild/protobuf/-/protobuf-1.10.1.tgz} + resolution: {integrity: sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ==, tarball: https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-1.10.1.tgz} '@emoji-mart/data@1.2.1': resolution: {integrity: sha512-no2pQMWiBy6gpBEiqGeU77/bFejDqUTRY7KX+0+iur13op3bqUsXdnwoZs6Xb1zbv0gAj5VvS1PWoUUckSr5Dw==} @@ -328,157 +328,157 @@ packages: react: ^16.8 || ^17 || ^18 '@esbuild/aix-ppc64@0.27.3': - resolution: {integrity: sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz} + resolution: {integrity: sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==} engines: {node: '>=18'} cpu: [ppc64] os: [aix] '@esbuild/android-arm64@0.27.3': - resolution: {integrity: sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/android-arm64/-/android-arm64-0.27.3.tgz} + resolution: {integrity: sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==} engines: {node: '>=18'} cpu: [arm64] os: [android] '@esbuild/android-arm@0.27.3': - resolution: {integrity: sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/android-arm/-/android-arm-0.27.3.tgz} + resolution: {integrity: sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==} engines: {node: '>=18'} cpu: [arm] os: [android] '@esbuild/android-x64@0.27.3': - resolution: {integrity: sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/android-x64/-/android-x64-0.27.3.tgz} + resolution: {integrity: sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==} engines: {node: '>=18'} cpu: [x64] os: [android] '@esbuild/darwin-arm64@0.27.3': - resolution: {integrity: sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/darwin-arm64/-/darwin-arm64-0.27.3.tgz} + resolution: {integrity: sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==} engines: {node: '>=18'} cpu: [arm64] os: [darwin] '@esbuild/darwin-x64@0.27.3': - resolution: {integrity: sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/darwin-x64/-/darwin-x64-0.27.3.tgz} + resolution: {integrity: sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==} engines: {node: '>=18'} cpu: [x64] os: [darwin] '@esbuild/freebsd-arm64@0.27.3': - resolution: {integrity: sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.3.tgz} + resolution: {integrity: sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==} engines: {node: '>=18'} cpu: [arm64] os: [freebsd] '@esbuild/freebsd-x64@0.27.3': - resolution: {integrity: sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/freebsd-x64/-/freebsd-x64-0.27.3.tgz} + resolution: {integrity: sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==} engines: {node: '>=18'} cpu: [x64] os: [freebsd] '@esbuild/linux-arm64@0.27.3': - resolution: {integrity: sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/linux-arm64/-/linux-arm64-0.27.3.tgz} + resolution: {integrity: sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==} engines: {node: '>=18'} cpu: [arm64] os: [linux] '@esbuild/linux-arm@0.27.3': - resolution: {integrity: sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/linux-arm/-/linux-arm-0.27.3.tgz} + resolution: {integrity: sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==} engines: {node: '>=18'} cpu: [arm] os: [linux] '@esbuild/linux-ia32@0.27.3': - resolution: {integrity: sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/linux-ia32/-/linux-ia32-0.27.3.tgz} + resolution: {integrity: sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==} engines: {node: '>=18'} cpu: [ia32] os: [linux] '@esbuild/linux-loong64@0.27.3': - resolution: {integrity: sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/linux-loong64/-/linux-loong64-0.27.3.tgz} + resolution: {integrity: sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==} engines: {node: '>=18'} cpu: [loong64] os: [linux] '@esbuild/linux-mips64el@0.27.3': - resolution: {integrity: sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/linux-mips64el/-/linux-mips64el-0.27.3.tgz} + resolution: {integrity: sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==} engines: {node: '>=18'} cpu: [mips64el] os: [linux] '@esbuild/linux-ppc64@0.27.3': - resolution: {integrity: sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/linux-ppc64/-/linux-ppc64-0.27.3.tgz} + resolution: {integrity: sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==} engines: {node: '>=18'} cpu: [ppc64] os: [linux] '@esbuild/linux-riscv64@0.27.3': - resolution: {integrity: sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/linux-riscv64/-/linux-riscv64-0.27.3.tgz} + resolution: {integrity: sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==} engines: {node: '>=18'} cpu: [riscv64] os: [linux] '@esbuild/linux-s390x@0.27.3': - resolution: {integrity: sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/linux-s390x/-/linux-s390x-0.27.3.tgz} + resolution: {integrity: sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==} engines: {node: '>=18'} cpu: [s390x] os: [linux] '@esbuild/linux-x64@0.27.3': - resolution: {integrity: sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/linux-x64/-/linux-x64-0.27.3.tgz} + resolution: {integrity: sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==} engines: {node: '>=18'} cpu: [x64] os: [linux] '@esbuild/netbsd-arm64@0.27.3': - resolution: {integrity: sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.3.tgz} + resolution: {integrity: sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==} engines: {node: '>=18'} cpu: [arm64] os: [netbsd] '@esbuild/netbsd-x64@0.27.3': - resolution: {integrity: sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/netbsd-x64/-/netbsd-x64-0.27.3.tgz} + resolution: {integrity: sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==} engines: {node: '>=18'} cpu: [x64] os: [netbsd] '@esbuild/openbsd-arm64@0.27.3': - resolution: {integrity: sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.3.tgz} + resolution: {integrity: sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==} engines: {node: '>=18'} cpu: [arm64] os: [openbsd] '@esbuild/openbsd-x64@0.27.3': - resolution: {integrity: sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/openbsd-x64/-/openbsd-x64-0.27.3.tgz} + resolution: {integrity: sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==} engines: {node: '>=18'} cpu: [x64] os: [openbsd] '@esbuild/openharmony-arm64@0.27.3': - resolution: {integrity: sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.3.tgz} + resolution: {integrity: sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==} engines: {node: '>=18'} cpu: [arm64] os: [openharmony] '@esbuild/sunos-x64@0.27.3': - resolution: {integrity: sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/sunos-x64/-/sunos-x64-0.27.3.tgz} + resolution: {integrity: sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==} engines: {node: '>=18'} cpu: [x64] os: [sunos] '@esbuild/win32-arm64@0.27.3': - resolution: {integrity: sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/win32-arm64/-/win32-arm64-0.27.3.tgz} + resolution: {integrity: sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==} engines: {node: '>=18'} cpu: [arm64] os: [win32] '@esbuild/win32-ia32@0.27.3': - resolution: {integrity: sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/win32-ia32/-/win32-ia32-0.27.3.tgz} + resolution: {integrity: sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==} engines: {node: '>=18'} cpu: [ia32] os: [win32] '@esbuild/win32-x64@0.27.3': - resolution: {integrity: sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@esbuild/win32-x64/-/win32-x64-0.27.3.tgz} + resolution: {integrity: sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==} engines: {node: '>=18'} cpu: [x64] os: [win32] @@ -515,10 +515,10 @@ packages: resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} '@livekit/mutex@1.1.1': - resolution: {integrity: sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@livekit/mutex/-/mutex-1.1.1.tgz} + resolution: {integrity: sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==, tarball: https://registry.npmjs.org/@livekit/mutex/-/mutex-1.1.1.tgz} '@livekit/protocol@1.44.0': - resolution: {integrity: sha512-/vfhDUGcUKO8Q43r6i+5FrDhl5oZjm/X3U4x2Iciqvgn5C8qbj+57YPcWSJ1kyIZm5Cm6AV2nAPjMm3ETD/iyg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@livekit/protocol/-/protocol-1.44.0.tgz} + resolution: {integrity: sha512-/vfhDUGcUKO8Q43r6i+5FrDhl5oZjm/X3U4x2Iciqvgn5C8qbj+57YPcWSJ1kyIZm5Cm6AV2nAPjMm3ETD/iyg==, tarball: https://registry.npmjs.org/@livekit/protocol/-/protocol-1.44.0.tgz} '@noble/ciphers@2.1.1': resolution: {integrity: sha512-bysYuiVfhxNJuldNXlFEitTVdNnYUc+XNJZd7Qm2a5j1vZHgY+fazadNFWFaMK/2vye0JVlxV3gHmC0WDfAOQw==} @@ -954,140 +954,140 @@ packages: resolution: {integrity: sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==} '@rollup/rollup-android-arm-eabi@4.59.0': - resolution: {integrity: sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.59.0.tgz} + resolution: {integrity: sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg==} cpu: [arm] os: [android] '@rollup/rollup-android-arm64@4.59.0': - resolution: {integrity: sha512-hZ+Zxj3SySm4A/DylsDKZAeVg0mvi++0PYVceVyX7hemkw7OreKdCvW2oQ3T1FMZvCaQXqOTHb8qmBShoqk69Q==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.59.0.tgz} + resolution: {integrity: sha512-hZ+Zxj3SySm4A/DylsDKZAeVg0mvi++0PYVceVyX7hemkw7OreKdCvW2oQ3T1FMZvCaQXqOTHb8qmBShoqk69Q==} cpu: [arm64] os: [android] '@rollup/rollup-darwin-arm64@4.59.0': - resolution: {integrity: sha512-W2Psnbh1J8ZJw0xKAd8zdNgF9HRLkdWwwdWqubSVk0pUuQkoHnv7rx4GiF9rT4t5DIZGAsConRE3AxCdJ4m8rg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.59.0.tgz} + resolution: {integrity: sha512-W2Psnbh1J8ZJw0xKAd8zdNgF9HRLkdWwwdWqubSVk0pUuQkoHnv7rx4GiF9rT4t5DIZGAsConRE3AxCdJ4m8rg==} cpu: [arm64] os: [darwin] '@rollup/rollup-darwin-x64@4.59.0': - resolution: {integrity: sha512-ZW2KkwlS4lwTv7ZVsYDiARfFCnSGhzYPdiOU4IM2fDbL+QGlyAbjgSFuqNRbSthybLbIJ915UtZBtmuLrQAT/w==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.59.0.tgz} + resolution: {integrity: sha512-ZW2KkwlS4lwTv7ZVsYDiARfFCnSGhzYPdiOU4IM2fDbL+QGlyAbjgSFuqNRbSthybLbIJ915UtZBtmuLrQAT/w==} cpu: [x64] os: [darwin] '@rollup/rollup-freebsd-arm64@4.59.0': - resolution: {integrity: sha512-EsKaJ5ytAu9jI3lonzn3BgG8iRBjV4LxZexygcQbpiU0wU0ATxhNVEpXKfUa0pS05gTcSDMKpn3Sx+QB9RlTTA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.59.0.tgz} + resolution: {integrity: sha512-EsKaJ5ytAu9jI3lonzn3BgG8iRBjV4LxZexygcQbpiU0wU0ATxhNVEpXKfUa0pS05gTcSDMKpn3Sx+QB9RlTTA==} cpu: [arm64] os: [freebsd] '@rollup/rollup-freebsd-x64@4.59.0': - resolution: {integrity: sha512-d3DuZi2KzTMjImrxoHIAODUZYoUUMsuUiY4SRRcJy6NJoZ6iIqWnJu9IScV9jXysyGMVuW+KNzZvBLOcpdl3Vg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.59.0.tgz} + resolution: {integrity: sha512-d3DuZi2KzTMjImrxoHIAODUZYoUUMsuUiY4SRRcJy6NJoZ6iIqWnJu9IScV9jXysyGMVuW+KNzZvBLOcpdl3Vg==} cpu: [x64] os: [freebsd] '@rollup/rollup-linux-arm-gnueabihf@4.59.0': - resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.59.0.tgz} + resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==} cpu: [arm] os: [linux] libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.59.0': - resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.59.0.tgz} + resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==} cpu: [arm] os: [linux] libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.59.0': - resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.59.0.tgz} + resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==} cpu: [arm64] os: [linux] libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.59.0': - resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.59.0.tgz} + resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==} cpu: [arm64] os: [linux] libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.59.0': - resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.59.0.tgz} + resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==} cpu: [loong64] os: [linux] libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.59.0': - resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.59.0.tgz} + resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==} cpu: [loong64] os: [linux] libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.59.0': - resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.59.0.tgz} + resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==} cpu: [ppc64] os: [linux] libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.59.0': - resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.59.0.tgz} + resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==} cpu: [ppc64] os: [linux] libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.59.0': - resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.59.0.tgz} + resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==} cpu: [riscv64] os: [linux] libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.59.0': - resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.59.0.tgz} + resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==} cpu: [riscv64] os: [linux] libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.59.0': - resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.59.0.tgz} + resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==} cpu: [s390x] os: [linux] libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.59.0': - resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.59.0.tgz} + resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==} cpu: [x64] os: [linux] libc: [glibc] '@rollup/rollup-linux-x64-musl@4.59.0': - resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.59.0.tgz} + resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==} cpu: [x64] os: [linux] libc: [musl] '@rollup/rollup-openbsd-x64@4.59.0': - resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.59.0.tgz} + resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==} cpu: [x64] os: [openbsd] '@rollup/rollup-openharmony-arm64@4.59.0': - resolution: {integrity: sha512-tt9KBJqaqp5i5HUZzoafHZX8b5Q2Fe7UjYERADll83O4fGqJ49O1FsL6LpdzVFQcpwvnyd0i+K/VSwu/o/nWlA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.59.0.tgz} + resolution: {integrity: sha512-tt9KBJqaqp5i5HUZzoafHZX8b5Q2Fe7UjYERADll83O4fGqJ49O1FsL6LpdzVFQcpwvnyd0i+K/VSwu/o/nWlA==} cpu: [arm64] os: [openharmony] '@rollup/rollup-win32-arm64-msvc@4.59.0': - resolution: {integrity: sha512-V5B6mG7OrGTwnxaNUzZTDTjDS7F75PO1ae6MJYdiMu60sq0CqN5CVeVsbhPxalupvTX8gXVSU9gq+Rx1/hvu6A==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.59.0.tgz} + resolution: {integrity: sha512-V5B6mG7OrGTwnxaNUzZTDTjDS7F75PO1ae6MJYdiMu60sq0CqN5CVeVsbhPxalupvTX8gXVSU9gq+Rx1/hvu6A==} cpu: [arm64] os: [win32] '@rollup/rollup-win32-ia32-msvc@4.59.0': - resolution: {integrity: sha512-UKFMHPuM9R0iBegwzKF4y0C4J9u8C6MEJgFuXTBerMk7EJ92GFVFYBfOZaSGLu6COf7FxpQNqhNS4c4icUPqxA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.59.0.tgz} + resolution: {integrity: sha512-UKFMHPuM9R0iBegwzKF4y0C4J9u8C6MEJgFuXTBerMk7EJ92GFVFYBfOZaSGLu6COf7FxpQNqhNS4c4icUPqxA==} cpu: [ia32] os: [win32] '@rollup/rollup-win32-x64-gnu@4.59.0': - resolution: {integrity: sha512-laBkYlSS1n2L8fSo1thDNGrCTQMmxjYY5G0WFWjFFYZkKPjsMBsgJfGf4TLxXrF6RyhI60L8TMOjBMvXiTcxeA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.59.0.tgz} + resolution: {integrity: sha512-laBkYlSS1n2L8fSo1thDNGrCTQMmxjYY5G0WFWjFFYZkKPjsMBsgJfGf4TLxXrF6RyhI60L8TMOjBMvXiTcxeA==} cpu: [x64] os: [win32] '@rollup/rollup-win32-x64-msvc@4.59.0': - resolution: {integrity: sha512-2HRCml6OztYXyJXAvdDXPKcawukWY2GpR5/nxKp4iBgiO3wcoEGkAaqctIbZcNB6KlUQBIqt8VYkNSj2397EfA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.59.0.tgz} + resolution: {integrity: sha512-2HRCml6OztYXyJXAvdDXPKcawukWY2GpR5/nxKp4iBgiO3wcoEGkAaqctIbZcNB6KlUQBIqt8VYkNSj2397EfA==} cpu: [x64] os: [win32] @@ -1203,72 +1203,72 @@ packages: resolution: {integrity: sha512-hKL/jWf293UDSUN09rR69hrToyIXBb8CjGaWC7gfinvnQrBVvnLr08FeFi38gxtugAVyVcTa5/FD/Xnkb1siBw==} '@tauri-apps/cli-darwin-arm64@2.10.1': - resolution: {integrity: sha512-Z2OjCXiZ+fbYZy7PmP3WRnOpM9+Fy+oonKDEmUE6MwN4IGaYqgceTjwHucc/kEEYZos5GICve35f7ZiizgqEnQ==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-darwin-arm64/-/cli-darwin-arm64-2.10.1.tgz} + resolution: {integrity: sha512-Z2OjCXiZ+fbYZy7PmP3WRnOpM9+Fy+oonKDEmUE6MwN4IGaYqgceTjwHucc/kEEYZos5GICve35f7ZiizgqEnQ==} engines: {node: '>= 10'} cpu: [arm64] os: [darwin] '@tauri-apps/cli-darwin-x64@2.10.1': - resolution: {integrity: sha512-V/irQVvjPMGOTQqNj55PnQPVuH4VJP8vZCN7ajnj+ZS8Kom1tEM2hR3qbbIRoS3dBKs5mbG8yg1WC+97dq17Pw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-darwin-x64/-/cli-darwin-x64-2.10.1.tgz} + resolution: {integrity: sha512-V/irQVvjPMGOTQqNj55PnQPVuH4VJP8vZCN7ajnj+ZS8Kom1tEM2hR3qbbIRoS3dBKs5mbG8yg1WC+97dq17Pw==} engines: {node: '>= 10'} cpu: [x64] os: [darwin] '@tauri-apps/cli-linux-arm-gnueabihf@2.10.1': - resolution: {integrity: sha512-Hyzwsb4VnCWKGfTw+wSt15Z2pLw2f0JdFBfq2vHBOBhvg7oi6uhKiF87hmbXOBXUZaGkyRDkCHsdzJcIfoJC2w==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-linux-arm-gnueabihf/-/cli-linux-arm-gnueabihf-2.10.1.tgz} + resolution: {integrity: sha512-Hyzwsb4VnCWKGfTw+wSt15Z2pLw2f0JdFBfq2vHBOBhvg7oi6uhKiF87hmbXOBXUZaGkyRDkCHsdzJcIfoJC2w==} engines: {node: '>= 10'} cpu: [arm] os: [linux] '@tauri-apps/cli-linux-arm64-gnu@2.10.1': - resolution: {integrity: sha512-OyOYs2t5GkBIvyWjA1+h4CZxTcdz1OZPCWAPz5DYEfB0cnWHERTnQ/SLayQzncrT0kwRoSfSz9KxenkyJoTelA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-linux-arm64-gnu/-/cli-linux-arm64-gnu-2.10.1.tgz} + resolution: {integrity: sha512-OyOYs2t5GkBIvyWjA1+h4CZxTcdz1OZPCWAPz5DYEfB0cnWHERTnQ/SLayQzncrT0kwRoSfSz9KxenkyJoTelA==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] libc: [glibc] '@tauri-apps/cli-linux-arm64-musl@2.10.1': - resolution: {integrity: sha512-MIj78PDDGjkg3NqGptDOGgfXks7SYJwhiMh8SBoZS+vfdz7yP5jN18bNaLnDhsVIPARcAhE1TlsZe/8Yxo2zqg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.10.1.tgz} + resolution: {integrity: sha512-MIj78PDDGjkg3NqGptDOGgfXks7SYJwhiMh8SBoZS+vfdz7yP5jN18bNaLnDhsVIPARcAhE1TlsZe/8Yxo2zqg==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] libc: [musl] '@tauri-apps/cli-linux-riscv64-gnu@2.10.1': - resolution: {integrity: sha512-X0lvOVUg8PCVaoEtEAnpxmnkwlE1gcMDTqfhbefICKDnOTJ5Est3qL0SrWxizDackIOKBcvtpejrSiVpuJI1kw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-linux-riscv64-gnu/-/cli-linux-riscv64-gnu-2.10.1.tgz} + resolution: {integrity: sha512-X0lvOVUg8PCVaoEtEAnpxmnkwlE1gcMDTqfhbefICKDnOTJ5Est3qL0SrWxizDackIOKBcvtpejrSiVpuJI1kw==} engines: {node: '>= 10'} cpu: [riscv64] os: [linux] libc: [glibc] '@tauri-apps/cli-linux-x64-gnu@2.10.1': - resolution: {integrity: sha512-2/12bEzsJS9fAKybxgicCDFxYD1WEI9kO+tlDwX5znWG2GwMBaiWcmhGlZ8fi+DMe9CXlcVarMTYc0L3REIRxw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-linux-x64-gnu/-/cli-linux-x64-gnu-2.10.1.tgz} + resolution: {integrity: sha512-2/12bEzsJS9fAKybxgicCDFxYD1WEI9kO+tlDwX5znWG2GwMBaiWcmhGlZ8fi+DMe9CXlcVarMTYc0L3REIRxw==} engines: {node: '>= 10'} cpu: [x64] os: [linux] libc: [glibc] '@tauri-apps/cli-linux-x64-musl@2.10.1': - resolution: {integrity: sha512-Y8J0ZzswPz50UcGOFuXGEMrxbjwKSPgXftx5qnkuMs2rmwQB5ssvLb6tn54wDSYxe7S6vlLob9vt0VKuNOaCIQ==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-linux-x64-musl/-/cli-linux-x64-musl-2.10.1.tgz} + resolution: {integrity: sha512-Y8J0ZzswPz50UcGOFuXGEMrxbjwKSPgXftx5qnkuMs2rmwQB5ssvLb6tn54wDSYxe7S6vlLob9vt0VKuNOaCIQ==} engines: {node: '>= 10'} cpu: [x64] os: [linux] libc: [musl] '@tauri-apps/cli-win32-arm64-msvc@2.10.1': - resolution: {integrity: sha512-iSt5B86jHYAPJa/IlYw++SXtFPGnWtFJriHn7X0NFBVunF6zu9+/zOn8OgqIWSl8RgzhLGXQEEtGBdR4wzpVgg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-win32-arm64-msvc/-/cli-win32-arm64-msvc-2.10.1.tgz} + resolution: {integrity: sha512-iSt5B86jHYAPJa/IlYw++SXtFPGnWtFJriHn7X0NFBVunF6zu9+/zOn8OgqIWSl8RgzhLGXQEEtGBdR4wzpVgg==} engines: {node: '>= 10'} cpu: [arm64] os: [win32] '@tauri-apps/cli-win32-ia32-msvc@2.10.1': - resolution: {integrity: sha512-gXyxgEzsFegmnWywYU5pEBURkcFN/Oo45EAwvZrHMh+zUSEAvO5E8TXsgPADYm31d1u7OQU3O3HsYfVBf2moHw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-win32-ia32-msvc/-/cli-win32-ia32-msvc-2.10.1.tgz} + resolution: {integrity: sha512-gXyxgEzsFegmnWywYU5pEBURkcFN/Oo45EAwvZrHMh+zUSEAvO5E8TXsgPADYm31d1u7OQU3O3HsYfVBf2moHw==} engines: {node: '>= 10'} cpu: [ia32] os: [win32] '@tauri-apps/cli-win32-x64-msvc@2.10.1': - resolution: {integrity: sha512-6Cn7YpPFwzChy0ERz6djKEmUehWrYlM+xTaNzGPgZocw3BD7OfwfWHKVWxXzdjEW2KfKkHddfdxK1XXTYqBRLg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@tauri-apps/cli-win32-x64-msvc/-/cli-win32-x64-msvc-2.10.1.tgz} + resolution: {integrity: sha512-6Cn7YpPFwzChy0ERz6djKEmUehWrYlM+xTaNzGPgZocw3BD7OfwfWHKVWxXzdjEW2KfKkHddfdxK1XXTYqBRLg==} engines: {node: '>= 10'} cpu: [x64] os: [win32] @@ -1306,7 +1306,7 @@ packages: resolution: {integrity: sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==} '@types/dom-mediacapture-record@1.0.22': - resolution: {integrity: sha512-mUMZLK3NvwRLcAAT9qmcK+9p7tpU2FHdDsntR3YI4+GY88XrgG4XiE7u1Q2LAN2/FZOz/tdMDC3GQCR4T8nFuw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/@types/dom-mediacapture-record/-/dom-mediacapture-record-1.0.22.tgz} + resolution: {integrity: sha512-mUMZLK3NvwRLcAAT9qmcK+9p7tpU2FHdDsntR3YI4+GY88XrgG4XiE7u1Q2LAN2/FZOz/tdMDC3GQCR4T8nFuw==} '@types/estree-jsx@1.0.5': resolution: {integrity: sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==} @@ -1530,7 +1530,7 @@ packages: resolution: {integrity: sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==} events@3.3.0: - resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/events/-/events-3.3.0.tgz} + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==, tarball: https://registry.npmjs.org/events/-/events-3.3.0.tgz} engines: {node: '>=0.8.x'} extend@3.0.2: @@ -1560,12 +1560,12 @@ packages: resolution: {integrity: sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ==} fsevents@2.3.2: - resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/fsevents/-/fsevents-2.3.2.tgz} + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] fsevents@2.3.3: - resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/fsevents/-/fsevents-2.3.3.tgz} + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] @@ -1666,7 +1666,7 @@ packages: hasBin: true jose@6.2.2: - resolution: {integrity: sha512-d7kPDd34KO/YnzaDOlikGpOurfF0ByC2sEV4cANCtdqLlTfBlw2p14O/5d/zv40gJPbIQxfES3nSx1/oYNyuZQ==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/jose/-/jose-6.2.2.tgz} + resolution: {integrity: sha512-d7kPDd34KO/YnzaDOlikGpOurfF0ByC2sEV4cANCtdqLlTfBlw2p14O/5d/zv40gJPbIQxfES3nSx1/oYNyuZQ==, tarball: https://registry.npmjs.org/jose/-/jose-6.2.2.tgz} js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -1689,7 +1689,7 @@ packages: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} livekit-client@2.18.1: - resolution: {integrity: sha512-nGjuEEV1mVN01EcAMwGIwG3J1gpBMqwn2V4R6W/8zz9Rah1CaAohIm6AMLG7BdctQpyeh34dfAOLfDodIsWyYA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/livekit-client/-/livekit-client-2.18.1.tgz} + resolution: {integrity: sha512-nGjuEEV1mVN01EcAMwGIwG3J1gpBMqwn2V4R6W/8zz9Rah1CaAohIm6AMLG7BdctQpyeh34dfAOLfDodIsWyYA==, tarball: https://registry.npmjs.org/livekit-client/-/livekit-client-2.18.1.tgz} peerDependencies: '@types/dom-mediacapture-record': ^1 @@ -1697,7 +1697,7 @@ packages: resolution: {integrity: sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==} loglevel@1.9.2: - resolution: {integrity: sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/loglevel/-/loglevel-1.9.2.tgz} + resolution: {integrity: sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==, tarball: https://registry.npmjs.org/loglevel/-/loglevel-1.9.2.tgz} engines: {node: '>= 0.6.0'} longest-streak@3.1.0: @@ -2107,17 +2107,17 @@ packages: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} rxjs@7.8.2: - resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/rxjs/-/rxjs-7.8.2.tgz} + resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} scheduler@0.27.0: resolution: {integrity: sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==} sdp-transform@2.15.0: - resolution: {integrity: sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/sdp-transform/-/sdp-transform-2.15.0.tgz} + resolution: {integrity: sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw==, tarball: https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.15.0.tgz} hasBin: true sdp@3.2.2: - resolution: {integrity: sha512-xZocWwfyp4hkbN4hLWxMjmv2Q8aNa9MhmOZ7L9aCZPT+dZsgRr6wZRrSYE3HTdyk/2pZKPSgqI7ns7Een1xMSA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/sdp/-/sdp-3.2.2.tgz} + resolution: {integrity: sha512-xZocWwfyp4hkbN4hLWxMjmv2Q8aNa9MhmOZ7L9aCZPT+dZsgRr6wZRrSYE3HTdyk/2pZKPSgqI7ns7Een1xMSA==, tarball: https://registry.npmjs.org/sdp/-/sdp-3.2.2.tgz} semver@6.3.1: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} @@ -2214,7 +2214,7 @@ packages: resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} tslib@2.8.1: - resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/tslib/-/tslib-2.8.1.tgz} + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==, tarball: https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz} tsx@4.21.0: resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==} @@ -2222,7 +2222,7 @@ packages: hasBin: true typed-emitter@2.1.0: - resolution: {integrity: sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/typed-emitter/-/typed-emitter-2.1.0.tgz} + resolution: {integrity: sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==, tarball: https://registry.npmjs.org/typed-emitter/-/typed-emitter-2.1.0.tgz} typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} @@ -2341,7 +2341,7 @@ packages: resolution: {integrity: sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ==} webrtc-adapter@9.0.4: - resolution: {integrity: sha512-5ZZY1+lGq8LEKuDlg9M2RPJHlH3R7OVwyHqMcUsLKCgd9Wvf+QrFTCItkXXYPmrJn8H6gRLXbSgxLLdexiqHxw==, tarball: https://global.block-artifacts.com/artifactory/api/npm/square-npm/webrtc-adapter/-/webrtc-adapter-9.0.4.tgz} + resolution: {integrity: sha512-5ZZY1+lGq8LEKuDlg9M2RPJHlH3R7OVwyHqMcUsLKCgd9Wvf+QrFTCItkXXYPmrJn8H6gRLXbSgxLLdexiqHxw==, tarball: https://registry.npmjs.org/webrtc-adapter/-/webrtc-adapter-9.0.4.tgz} engines: {node: '>=6.0.0', npm: '>=3.10.0'} yallist@3.1.1: diff --git a/desktop/scripts/check-file-sizes.mjs b/desktop/scripts/check-file-sizes.mjs index 81e4036506..3e70e22ecd 100644 --- a/desktop/scripts/check-file-sizes.mjs +++ b/desktop/scripts/check-file-sizes.mjs @@ -47,7 +47,7 @@ const overrides = new Map([ ["src-tauri/src/commands/agents.rs", 880], // remote agent lifecycle routing (local + provider branches) + scope enforcement + persona pack metadata wiring + mcp_toolsets field ["src-tauri/src/managed_agents/runtime.rs", 690], // KNOWN_AGENT_BINARIES const + process_belongs_to_us FFI (macOS proc_name + Linux /proc/comm) + terminate_process + start/stop/sync lifecycle + pack persona live-read ["src-tauri/src/managed_agents/backend.rs", 530], // provider IPC, validation, discovery, binary resolution + tests - ["src/features/huddle/HuddleContext.tsx", 630], // huddle lifecycle context + joinHuddle + connectAndSetupMedia shared helper + activeSpeakers/isReconnecting state + PTT + TTS subscription + mic level analyser + agent pubkey refresh + ["src/features/huddle/HuddleContext.tsx", 650], // huddle lifecycle context + joinHuddle + connectAndSetupMedia shared helper + activeSpeakers/isReconnecting state + PTT (reusable AudioContext) + TTS subscription + mic level analyser (10fps throttle) + agent pubkey refresh ["src/features/agents/hooks.ts", 540], // agent query/mutation surface now includes built-in persona library activation + useUpdateManagedAgentMutation ["src/features/agents/ui/AgentsView.tsx", 880], // remote agent lifecycle controls + persona/team management + persona import-update dialog wiring + built-in catalog/library state orchestration ["src/features/agents/ui/ManagedAgentRow.tsx", 530], // EditAgentDialog integration + provider/local branching diff --git a/desktop/src-tauri/src/huddle/kokoro.rs b/desktop/src-tauri/src/huddle/kokoro.rs index 0c5bf7224e..9f1c138c11 100644 --- a/desktop/src-tauri/src/huddle/kokoro.rs +++ b/desktop/src-tauri/src/huddle/kokoro.rs @@ -4,7 +4,7 @@ //! //! load_text_to_speech(model_dir) → KokoroTTS //! load_voice_style(path) → VoiceStyle -//! tts.call(text, lang, &style) → Vec @ 24 kHz +//! tts.synth_chunk(text, lang, &style, steps, speed) → Vec @ 24 kHz //! //! ┌──────────┐ G2P ┌──────────┐ tokenize ┌──────────┐ //! │ raw text │ ──────→ │ IPA str │ ─────────→ │ int64[] │ @@ -26,8 +26,6 @@ use std::path::{Path, PathBuf}; use ndarray::{Array1, Array2}; use ort::{session::Session, value::Value}; -use super::preprocessing::split_sentences; - // ── Public constants ────────────────────────────────────────────────────────── pub const SAMPLE_RATE: u32 = 24_000; @@ -639,40 +637,6 @@ pub fn load_text_to_speech(model_dir: &str) -> Result { } impl KokoroTTS { - /// Synthesize `text` to 24 kHz mono PCM. - /// - /// - `_total_step` is ignored — Kokoro is not diffusion-based. - /// - `speed` controls speech rate (0.5–2.0; 1.0 = normal). - /// - `silence_secs` of silence is inserted between sentence chunks. - /// - `lang` is accepted for API compatibility but currently unused - /// (Kokoro v1.0 language is selected by voice name prefix, e.g. `af_*`). - pub fn call( - &mut self, - text: &str, - _lang: &str, - style: &VoiceStyle, - _total_step: usize, - speed: f32, - silence_secs: f32, - ) -> Result, String> { - let silence_samples = (silence_secs * SAMPLE_RATE as f32) as usize; - let silence = vec![0.0f32; silence_samples]; - - let sentences = split_sentences(text); - let mut output: Vec = Vec::new(); - - for (i, sentence) in sentences.iter().enumerate() { - let chunk_audio = self.synth_chunk(sentence, _lang, style, _total_step, speed)?; - - if i > 0 && !output.is_empty() { - output.extend_from_slice(&silence); - } - output.extend(chunk_audio); - } - - Ok(output) - } - /// Synthesize a single pre-split text chunk. Caller is responsible for sentence splitting. /// This avoids double-splitting when the TTS pipeline has already split the text. /// diff --git a/desktop/src-tauri/src/huddle/models.rs b/desktop/src-tauri/src/huddle/models.rs index 8ad45eae8a..e264493dcb 100644 --- a/desktop/src-tauri/src/huddle/models.rs +++ b/desktop/src-tauri/src/huddle/models.rs @@ -42,6 +42,7 @@ const KOKORO_FILE_HASHES: &[(&str, &str)] = &[ ("af_heart.bin", "d583ccff3cdca2f7fae535cb998ac07e9fcb90f09737b9a41fa2734ec44a8f0b"), ("us_gold.json", "dc414872a49a28ae6c141463d502fd945f3b2fde040484fdc47d00cc4612686f"), ("us_silver.json", "de8f67be911bb6c659187b4a65fd966b6a30e56350e0f790d763210b053ac475"), + ("cmudict.dict", "81917843c7f44ce2b094ac63873c2c7a4cf802040792c455ba3ca406891c3d22"), ]; // ── Model versioning ────────────────────────────────────────────────────────── diff --git a/desktop/src-tauri/src/huddle/state.rs b/desktop/src-tauri/src/huddle/state.rs index 68d105337d..b73e41a2d6 100644 --- a/desktop/src-tauri/src/huddle/state.rs +++ b/desktop/src-tauri/src/huddle/state.rs @@ -214,27 +214,11 @@ impl HuddleState { /// The frontend listens for `"huddle-state-changed"` and updates its UI /// immediately, replacing the previous 2-second polling loop. /// -/// **Call sites** — call this in `huddle/mod.rs` after every state transition -/// that the frontend needs to observe: -/// - After `phase` changes (Creating → Connecting → Connected → Active → Leaving → Idle) -/// - After `participants` is updated (join/leave) -/// - After `tts_enabled` is toggled -/// -/// **Usage in mod.rs:** -/// ```rust -/// // After mutating huddle state, while still holding the lock: -/// let snapshot = hs.clone(); -/// drop(hs); // release lock before emitting -/// if let Ok(guard) = state.app_handle.lock() { -/// if let Some(app) = guard.as_ref() { -/// emit_huddle_state(app, &snapshot); -/// } -/// } -/// ``` +/// **Call sites** — called from `AppState::emit_huddle_state_changed()` in +/// `app_state.rs` after every state transition the frontend needs to observe +/// (phase changes, participant updates, tts_enabled toggle). /// /// Best-effort — silently ignores errors (e.g., no listeners attached yet). -// Not yet called from mod.rs — suppress dead_code until that integration lands. -#[allow(dead_code)] pub fn emit_huddle_state(app: &tauri::AppHandle, state: &HuddleState) { use tauri::Emitter; let _ = app.emit("huddle-state-changed", state); diff --git a/desktop/src/features/huddle/HuddleContext.tsx b/desktop/src/features/huddle/HuddleContext.tsx index 57221df2a0..4ca093f100 100644 --- a/desktop/src/features/huddle/HuddleContext.tsx +++ b/desktop/src/features/huddle/HuddleContext.tsx @@ -110,6 +110,10 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { }); }, []); + // Persistent AudioContext for PTT audio cues — reused across all PTT presses + // to avoid exhausting the browser's ~6 concurrent AudioContext limit. + const pttAudioCtxRef = React.useRef(null); + // PTT state from Rust (Ctrl+Space). UI feedback + 50ms audio cue when mic active. // Actual audio gating is in audioWorklet.ts → worklet.js. React.useEffect(() => { @@ -120,7 +124,13 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { setPttActive(event.payload); if (micConnected) { try { - const ac = new AudioContext(); + if ( + !pttAudioCtxRef.current || + pttAudioCtxRef.current.state === "closed" + ) { + pttAudioCtxRef.current = new AudioContext(); + } + const ac = pttAudioCtxRef.current; const osc = ac.createOscillator(); const g = ac.createGain(); osc.connect(g); @@ -129,7 +139,6 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { g.gain.value = 0.05; osc.start(); osc.stop(ac.currentTime + 0.05); - osc.onended = () => void ac.close(); } catch { /* best-effort */ } @@ -141,6 +150,11 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { return () => { cancelled = true; unlisten?.(); + // Close the PTT AudioContext when the effect is cleaned up. + if (pttAudioCtxRef.current && pttAudioCtxRef.current.state !== "closed") { + void pttAudioCtxRef.current.close(); + pttAudioCtxRef.current = null; + } }; }, [micConnected]); @@ -571,13 +585,18 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { const buf = new Uint8Array(analyser.frequencyBinCount); let raf = 0; - function tick() { + let lastUpdate = 0; + function tick(now: number) { + raf = requestAnimationFrame(tick); + // Throttle state updates to ~10fps — voice meters don't need 60fps + // visual fidelity, and setMicLevel re-renders the entire HuddleBar. + if (now - lastUpdate < 100) return; + lastUpdate = now; analyser.getByteFrequencyData(buf); // RMS-ish: average of frequency bins, normalized to 0–1 let sum = 0; for (let i = 0; i < buf.length; i++) sum += buf[i]; setMicLevel(sum / (buf.length * 255)); - raf = requestAnimationFrame(tick); } raf = requestAnimationFrame(tick); From 269edaca1a93412099cca61d5b802a77153471d3 Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Tue, 14 Apr 2026 12:29:23 -0400 Subject: [PATCH 40/41] ci: add libasound2-dev for rodio/ALSA on Linux CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rodio → cpal → alsa-sys requires ALSA development headers on Linux. The desktop-tauri-check step compiles the Rust desktop crate which now depends on rodio for TTS audio playback. --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e2043786f..43e3af0dd6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,7 @@ jobs: build-essential \ curl \ file \ + libasound2-dev \ libayatana-appindicator3-dev \ libgtk-3-dev \ librsvg2-dev \ From 350335055bb12c477c86c2f7de8b0ebf20b3cdb2 Mon Sep 17 00:00:00 2001 From: Tyler Longwell Date: Tue, 14 Apr 2026 12:33:35 -0400 Subject: [PATCH 41/41] fix(agents): default GUI-launched agents to owner-interrupt mode The ACP harness defaults to multiple_event_handling=queue when the env var is absent. GUI-launched agents never set this, so owners couldn't interrupt their own agents mid-turn. Now runtime.rs explicitly sets: SPROUT_ACP_MULTIPLE_EVENT_HANDLING=owner-interrupt SPROUT_ACP_DEDUP=queue (required by owner-interrupt) This means the agent owner can @mention the agent during an in-flight turn and the harness will cancel the current turn and re-dispatch with the new prompt. Non-owner mentions still queue normally. --- desktop/src-tauri/src/managed_agents/runtime.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/desktop/src-tauri/src/managed_agents/runtime.rs b/desktop/src-tauri/src/managed_agents/runtime.rs index a13755b021..01485a5e3c 100644 --- a/desktop/src-tauri/src/managed_agents/runtime.rs +++ b/desktop/src-tauri/src/managed_agents/runtime.rs @@ -513,6 +513,8 @@ pub fn start_managed_agent_process( .unwrap_or(super::types::DEFAULT_AGENT_MAX_TURN_DURATION_SECONDS); command.env("SPROUT_ACP_MAX_TURN_DURATION", max_dur.to_string()); command.env("SPROUT_ACP_AGENTS", record.parallelism.to_string()); + command.env("SPROUT_ACP_MULTIPLE_EVENT_HANDLING", "owner-interrupt"); + command.env("SPROUT_ACP_DEDUP", "queue"); command.env( "GOOSE_MODE", std::env::var("GOOSE_MODE").unwrap_or_else(|_| "auto".to_string()),