Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
76b5403
feat(huddles): Phase 1 — Voice Call Foundation
tlongwell-block Apr 11, 2026
a2304b9
feat(huddles): Phase 2 — Speech-to-Text Pipeline
tlongwell-block Apr 11, 2026
65aefed
feat(huddles): Phase 3 — Agent Integration
tlongwell-block Apr 11, 2026
54c5cdb
feat(huddles): Phase 4 — Text-to-Speech Pipeline
tlongwell-block Apr 11, 2026
4504c12
style: format all huddle files (biome + rustfmt)
tlongwell-block Apr 11, 2026
55a725e
feat(huddles): add headphones button to channel header bar
tlongwell-block Apr 11, 2026
05d8c8a
feat(huddles): replace sherpa-onnx TTS with kokoro-tts (GPL-free)
tlongwell-block Apr 11, 2026
b8db58d
feat(huddles): wire headphones button E2E — LiveKit + AudioWorklet + …
tlongwell-block Apr 11, 2026
2bdae57
fix(relay): allow huddle lifecycle events (kind 48100-48103)
tlongwell-block Apr 11, 2026
329744f
fix(huddles): include current user and agents in participant list
tlongwell-block Apr 12, 2026
5745df8
fix(desktop): add audio-input entitlement for mic access
tlongwell-block Apr 12, 2026
cae755b
feat(huddles): auto-download voice models + mic activity indicator
tlongwell-block Apr 12, 2026
9a1ae62
fix(huddles): update model URLs and file layout for Moonshine + Kokoro
tlongwell-block Apr 12, 2026
1dfb733
feat(huddles): include parent channel ID in voice-mode guidelines
tlongwell-block Apr 12, 2026
d0c98a2
fix(huddles): switch Kokoro TTS to v1.0 int8 model from mzdk100/kokoro
tlongwell-block Apr 12, 2026
3b6cb99
feat(huddles): wire TTS — subscribe to ephemeral channel, pipe agent …
tlongwell-block Apr 12, 2026
ff9d5ad
fix(huddles): split TTS into sentences + skip historical messages
tlongwell-block Apr 12, 2026
2349047
fix(huddles): TTS lookahead synthesis + smarter sentence splitting
tlongwell-block Apr 12, 2026
fba0945
feat(huddles): replace Kokoro TTS with Supertonic (MIT, 167× RTF, no …
tlongwell-block Apr 12, 2026
609e787
fix(huddles): u8 overflow in Supertonic download progress calculation
tlongwell-block Apr 12, 2026
3961f4f
fix(huddles): Supertonic sample rate is 44100, not 24000
tlongwell-block Apr 12, 2026
210f72f
chore: update Cargo.lock for Supertonic deps (ndarray 0.17, rand, regex)
tlongwell-block Apr 12, 2026
d1f8b8b
fix(huddles): TTS smoothness — 7 fixes for less clunky speech
tlongwell-block Apr 12, 2026
261cf79
chore: remove debug console.log from livekit.ts
tlongwell-block Apr 12, 2026
a8034ff
style: rustfmt + biome format all huddle files
tlongwell-block Apr 12, 2026
f22d3d7
fix(huddles): crossfire hardening — 20+ fixes across voice pipeline
tlongwell-block Apr 12, 2026
f3798d7
fix(huddles): crossfire quality pass — safety, DRY, UX hardening
tlongwell-block Apr 12, 2026
bed4911
refactor(huddles): crossfire quality pass — DRY, safety, correctness
tlongwell-block Apr 13, 2026
f11520e
refactor(huddles): crossfire cleanup — DRY, safety, display names
tlongwell-block Apr 13, 2026
7861a29
huddles: crossfire improvements — fault boundary, DRY, feature-gate, UX
tlongwell-block Apr 13, 2026
414fcac
huddles: allow kind 48106 in relay ingest + preload voice models at a…
tlongwell-block Apr 13, 2026
908049c
huddles: live-testing fixes — TTS playback, STT tuning, barge-in disa…
tlongwell-block Apr 13, 2026
c5d05fe
feat(relay): enable huddles by default with dev credentials
tlongwell-block Apr 13, 2026
caef80f
feat(huddles): push-to-talk as default voice input mode
tlongwell-block Apr 13, 2026
0370ed5
feat(huddle): multi-human support with auto-end lifecycle
tlongwell-block Apr 13, 2026
9117715
fix(huddle): crossfire review fixes — split, harden, clean up
tlongwell-block Apr 13, 2026
20dbfc4
Merge remote-tracking branch 'origin/main' into feat/huddles-plan
tlongwell-block Apr 14, 2026
fa1491e
fix(huddle): crossfire P0/P1/P2 fixes — safety, DRY, UX, accessibility
tlongwell-block Apr 14, 2026
4062d08
feat(huddle): relay-side auto-add for huddle joiners (Option D)
tlongwell-block Apr 14, 2026
57be7e7
fix(huddle): crossfire round 4 — integrity, perf, cleanup
tlongwell-block Apr 14, 2026
269edac
ci: add libasound2-dev for rodio/ALSA on Linux CI
tlongwell-block Apr 14, 2026
3503350
fix(agents): default GUI-launched agents to owner-interrupt mode
tlongwell-block Apr 14, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ jobs:
build-essential \
curl \
file \
libasound2-dev \
libayatana-appindicator3-dev \
libgtk-3-dev \
librsvg2-dev \
Expand Down
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions crates/sprout-core/src/kind.rs
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,8 @@ pub const KIND_HUDDLE_ENDED: u32 = 48103;
pub const KIND_HUDDLE_TRACK_PUBLISHED: u32 = 48104;
/// A huddle recording became available.
pub const KIND_HUDDLE_RECORDING_AVAILABLE: u32 = 48105;
/// Huddle channel guidelines/rules document.
pub const KIND_HUDDLE_GUIDELINES: u32 = 48106;

// Media (49000–49999)
/// Internal kind for media upload audit entries. Not a relay event kind.
Expand Down
10 changes: 7 additions & 3 deletions crates/sprout-huddle/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ chrono = { workspace = true }
tracing = { workspace = true }
thiserror = { workspace = true }
jsonwebtoken = { workspace = true }
hmac = { workspace = true }
sha2 = { workspace = true }
hex = { workspace = true }
hmac = { workspace = true, optional = true }
sha2 = { workspace = true, optional = true }
hex = { workspace = true, optional = true }

[features]
default = []
webhook = ["hmac", "sha2", "hex"]
7 changes: 7 additions & 0 deletions crates/sprout-huddle/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,24 @@
/// Error types for the huddle layer.
pub mod error;
/// In-memory huddle session and participant tracking.
#[cfg(feature = "webhook")]
pub mod session;
/// LiveKit access token generation.
pub mod token;
/// LiveKit webhook signature verification and event parsing.
#[cfg(feature = "webhook")]
pub mod webhook;

pub use error::HuddleError;
#[cfg(feature = "webhook")]
pub use session::{HuddleParticipant, HuddleSession, TrackInfo, TrackKind};
pub use token::LiveKitToken;
#[cfg(feature = "webhook")]
pub use webhook::WebhookEvent;

use uuid::Uuid;

#[cfg(feature = "webhook")]
pub use sprout_core::kind::{
KIND_HUDDLE_ENDED, KIND_HUDDLE_PARTICIPANT_JOINED, KIND_HUDDLE_PARTICIPANT_LEFT,
KIND_HUDDLE_RECORDING_AVAILABLE, KIND_HUDDLE_STARTED, KIND_HUDDLE_TRACK_PUBLISHED,
Expand Down Expand Up @@ -74,6 +79,7 @@ impl HuddleService {
}

/// Verify the webhook signature and parse the LiveKit event payload.
#[cfg(feature = "webhook")]
pub fn parse_webhook(
&self,
body: &[u8],
Expand Down Expand Up @@ -146,6 +152,7 @@ mod tests {
}

#[test]
#[cfg(feature = "webhook")]
fn session_lifecycle() {
let channel_id = Uuid::new_v4();
let room_name = HuddleService::create_room_name(channel_id);
Expand Down
2 changes: 1 addition & 1 deletion crates/sprout-huddle/src/token.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ pub fn generate_token(
ttl: Option<Duration>,
) -> Result<LiveKitToken, HuddleError> {
let now = Utc::now();
let ttl = ttl.unwrap_or_else(|| Duration::hours(6));
let ttl = ttl.unwrap_or_else(|| Duration::hours(1));
let expires_at = now + ttl;

let claims = LiveKitClaims {
Expand Down
1 change: 1 addition & 0 deletions crates/sprout-relay/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ deadpool-redis = { workspace = true }
redis = { workspace = true }
sqlx = { workspace = true }
base64 = "0.22"
sprout-huddle = { workspace = true, features = ["webhook"] }
sprout-workflow = { workspace = true, features = ["reqwest"] }
sprout-media = { workspace = true }
bytes = "1"
Expand Down
128 changes: 128 additions & 0 deletions crates/sprout-relay/src/api/huddles.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
//! LiveKit huddle token endpoint.
//!
//! ## Routes
//! - `POST /api/huddles/{channel_id}/token` — generate a LiveKit access token
//! for the authenticated user to join the channel's huddle room.

use std::sync::Arc;

use axum::{
extract::{Path, Query, State},
http::{HeaderMap, StatusCode},
response::Json,
};
use uuid::Uuid;

use super::{
check_channel_membership, check_token_channel_access, extract_auth_context, internal_error,
scope_error,
};
use crate::state::AppState;

/// Query parameters for [`huddle_token`].
#[derive(serde::Deserialize)]
pub struct HuddleTokenQuery {
/// The parent (non-ephemeral) channel this huddle belongs to.
///
/// When provided and the caller is a member of the parent channel, the relay
/// will auto-add them to the private ephemeral huddle channel so they can
/// obtain a token without requiring an explicit invite.
pub parent_channel_id: Option<Uuid>,
}

/// `POST /api/huddles/{channel_id}/token` — generate a LiveKit access token.
///
/// Returns `{ "token": "<jwt>", "url": "<livekit_url>", "room": "sprout-<channel_id>" }`.
/// Returns 501 if LiveKit is not configured on this relay.
/// Returns 403 if the caller is not a member of the channel.
pub async fn huddle_token(
State(state): State<Arc<AppState>>,
headers: HeaderMap,
Path(channel_id): Path<Uuid>,
Query(query): Query<HuddleTokenQuery>,
) -> Result<Json<serde_json::Value>, (StatusCode, Json<serde_json::Value>)> {
let ctx = extract_auth_context(&headers, &state).await?;
sprout_auth::require_scope(&ctx.scopes, sprout_auth::Scope::MessagesRead)
.map_err(scope_error)?;
check_token_channel_access(&ctx, &channel_id)?;

// Require LiveKit to be configured.
let huddle_service = state.huddle_service.as_ref().ok_or_else(|| {
(
StatusCode::NOT_IMPLEMENTED,
Json(serde_json::json!({
"error": "huddles_not_configured",
"message": "LiveKit is not configured on this relay"
})),
)
})?;

// Verify the caller is a member of the channel (or it's an open channel).
// If they're not a member, attempt relay-side auto-add when:
// 1. parent_channel_id was provided
// 2. The target channel is private + ephemeral (ttl_seconds IS NOT NULL)
// 3. The caller IS a member of the parent channel
let membership_result = check_channel_membership(&state, channel_id, &ctx.pubkey_bytes).await;

if let Err(ref membership_err) = membership_result {
if let Some(parent_id) = query.parent_channel_id {
// Gate 1: caller must be a member of the parent channel.
check_channel_membership(&state, parent_id, &ctx.pubkey_bytes).await?;

// Gate 2: target channel must be private + ephemeral.
let channel = state
.db
.get_channel(channel_id)
.await
.map_err(|e| internal_error(&format!("db error: {e}")))?;

if channel.visibility == "private" && channel.ttl_seconds.is_some() {
// Auto-add: use the channel creator as invited_by (truthful attribution).
// The creator is always an active owner, satisfying add_member's invite check.
state
.db
.add_member(
channel_id,
&ctx.pubkey_bytes,
sprout_db::channel::MemberRole::Member,
Some(&channel.created_by),
)
.await
.map_err(|e| internal_error(&format!("auto-add failed: {e}")))?;

tracing::info!(
"Huddle auto-add: added {} to ephemeral channel {} (parent: {})",
ctx.pubkey.to_hex(),
channel_id,
parent_id
);
// Fall through to token generation.
} else {
// Not a private ephemeral channel — return the original 403.
return Err(membership_err.clone());
}
} else {
// No parent_channel_id provided — return the original 403.
membership_result?;
}
}

// Generate the LiveKit token.
let room = sprout_huddle::HuddleService::create_room_name(channel_id);
let identity = ctx.pubkey.to_hex();
let lk_token = huddle_service
.generate_token(&room, &identity, &identity)
.map_err(|e| internal_error(&format!("token generation failed: {e}")))?;

let livekit_url = state.livekit_url.as_deref().unwrap_or_default();

if livekit_url.is_empty() {
return Err(internal_error("livekit_url is not configured"));
}

Ok(Json(serde_json::json!({
"token": lk_token.token,
"url": livekit_url,
"room": room,
})))
}
6 changes: 6 additions & 0 deletions crates/sprout-relay/src/api/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ pub mod dms;
pub mod events;
/// Personalized home feed endpoint.
pub mod feed;
/// LiveKit huddle token endpoint.
pub mod huddles;
/// Blossom-compatible media upload, retrieval, and existence check endpoints.
pub mod media;
/// Channel membership endpoints.
Expand All @@ -44,6 +46,8 @@ pub mod search;
pub mod tokens;
/// User profile endpoints.
pub mod users;
/// LiveKit webhook handler for server-side presence tracking.
pub mod webhooks;
/// Shared helpers for workflow API handlers.
pub mod workflow_helpers;
/// Workflow CRUD, trigger, and webhook endpoints.
Expand All @@ -58,6 +62,7 @@ pub use channels_metadata::get_channel_handler;
pub use dms::{add_dm_member_handler, hide_dm_handler, list_dms_handler, open_dm_handler};
pub use events::get_event;
pub use feed::feed_handler;
pub use huddles::huddle_token;
pub use members::list_members;
pub use messages::{get_thread, list_messages, validate_imeta_tags, verify_imeta_blobs};
pub use presence::{presence_handler, set_presence_handler};
Expand All @@ -67,6 +72,7 @@ pub use users::{
get_contact_list, get_profile, get_user_notes, get_user_profile, get_users_batch,
put_channel_add_policy, search_users,
};
pub use webhooks::handle_livekit_webhook;
pub use workflows::{
create_workflow, delete_workflow, get_workflow, list_channel_workflows, list_run_approvals,
list_workflow_runs, trigger_workflow, update_workflow, workflow_webhook,
Expand Down
120 changes: 120 additions & 0 deletions crates/sprout-relay/src/api/webhooks.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
//! LiveKit webhook handler — server-side huddle presence tracking.
//!
//! Receives webhook events from LiveKit and emits corresponding Nostr
//! huddle lifecycle events (kinds 48100–48103). This provides authoritative
//! presence tracking that survives client crashes — LiveKit fires
//! `participant_left` when the WebRTC connection drops, regardless of
//! whether the client performed a graceful shutdown.
//!
//! ## Route
//! `POST /internal/livekit/webhook` — internal only, not exposed through
//! the public API gateway.

use std::sync::Arc;

use axum::{
extract::State,
http::{HeaderMap, StatusCode},
};
use sprout_huddle::WebhookEvent;

use crate::state::AppState;

/// Handle a LiveKit webhook event.
///
/// Verifies the webhook signature, parses the event, and emits the
/// corresponding Nostr huddle lifecycle event to the parent channel.
///
/// Returns 200 on success (even if the event type is unrecognized —
/// LiveKit expects 2xx for all webhook deliveries).
/// Returns 401 if signature verification fails.
/// Returns 501 if huddles are not configured.
pub async fn handle_livekit_webhook(
State(state): State<Arc<AppState>>,
headers: HeaderMap,
body: String,
) -> StatusCode {
let huddle_service = match state.huddle_service.as_ref() {
Some(svc) => svc,
None => return StatusCode::NOT_IMPLEMENTED,
};

// Verify signature and parse the event.
let auth_header = headers
.get("Authorization")
.and_then(|v| v.to_str().ok())
.unwrap_or("");

let event = match huddle_service.parse_webhook(body.as_bytes(), auth_header) {
Ok(e) => e,
Err(sprout_huddle::HuddleError::InvalidWebhookSignature) => {
tracing::warn!("LiveKit webhook: signature verification failed");
return StatusCode::UNAUTHORIZED;
}
Err(e) => {
// Signed but malformed/unknown — log and return 200 so LiveKit
// doesn't retry. Parse failures are not auth failures.
tracing::warn!("LiveKit webhook: parse error (signed OK): {e}");
return StatusCode::OK;
}
};

// Dispatch on the parsed enum variant.
// Room names follow the format "sprout-{channel_uuid}".
match event {
WebhookEvent::RoomStarted { room } => {
tracing::info!("LiveKit: room started {room}");
// TODO: Emit kind:48100 signed by relay keypair
}
WebhookEvent::RoomFinished { room } => {
tracing::info!("LiveKit: room finished {room}");
// TODO: Emit kind:48103 + archive ephemeral channel
}
WebhookEvent::ParticipantJoined { room, identity } => {
let Some(channel_id) = parse_channel_id(&room) else {
tracing::debug!("LiveKit webhook for non-sprout room or invalid UUID: {room}");
return StatusCode::OK;
};
tracing::info!(
"LiveKit: participant joined room {room} (channel {channel_id}), identity={identity}"
);
// TODO: Emit kind:48101 signed by relay keypair.
// The client also emits this event; the server-side copy provides
// crash-recovery redundancy.
}
WebhookEvent::ParticipantLeft { room, identity } => {
let Some(channel_id) = parse_channel_id(&room) else {
tracing::debug!("LiveKit webhook for non-sprout room or invalid UUID: {room}");
return StatusCode::OK;
};
tracing::info!(
"LiveKit: participant left room {room} (channel {channel_id}), identity={identity}"
);
// TODO: Emit kind:48102 signed by relay keypair.
// This is the key crash-recovery path — fires even if the client crashed.
}
WebhookEvent::TrackPublished {
room,
identity,
kind,
} => {
tracing::debug!("LiveKit: track published in {room} by {identity} (kind={kind})");
// TODO: Emit kind:48104 (track published) if/when that event kind is defined.
}
}

StatusCode::OK
}

/// Extract the channel UUID from a LiveKit room name.
///
/// Room names follow the format `sprout-{uuid}`. Returns `None` if the name
/// does not match the expected prefix or contains an invalid UUID.
fn parse_channel_id(room_name: &str) -> Option<uuid::Uuid> {
let channel_id = room_name.strip_prefix("sprout-")?;
uuid::Uuid::parse_str(channel_id)
.map_err(|_| {
tracing::warn!("LiveKit webhook: invalid channel UUID in room name: {room_name}");
})
.ok()
}
Loading