diff --git a/.gitignore b/.gitignore index 42cd6d0f7..0b3e4c155 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,28 @@ interface/node_modules/ interface/dist/ interface/public/opencode-embed/ -.opencode-build-cache/ + +# OpenCode +.opencode*/ + +# Rust +/target +.Cargo.lock + +# Python +__pycache__/ +*.pyc +*.pyo +*.pyd +.env +.venv +venv/ +.env.local + +# Node +node_modules/ +dist/ +build/ # Desktop sidecar binaries (built by scripts/bundle-sidecar.sh) desktop/src-tauri/binaries/ diff --git a/docs/content/docs/(configuration)/config.mdx b/docs/content/docs/(configuration)/config.mdx index e85ab5a7d..a5502736b 100644 --- a/docs/content/docs/(configuration)/config.mdx +++ b/docs/content/docs/(configuration)/config.mdx @@ -28,6 +28,10 @@ These environment variables control instance-level behavior and are not set in ` | `SPACEBOT_USER_TIMEZONE` | inherits cron | Default timezone for channel/worker temporal context. Overridden by config equivalents. | | `SPACEBOT_CHANNEL_MODEL` | `anthropic/claude-sonnet-4-20250514` | Default channel model (env-only mode). | | `SPACEBOT_WORKER_MODEL` | `anthropic/claude-haiku-4.5-20250514` | Default worker model (env-only mode). | +| `SPACEBOT_VOICE_MODEL` | None | Voice transcription model (e.g. `groq/whisper-large-v3-turbo`). | +| `SPACEBOT_VOICE_LANGUAGE` | None | Language hint for voice transcription (ISO 639-1, e.g. `en`, `es`). | +| `SPACEBOT_VOICE_TRANSLATE` | `false` | Set to `true` to translate audio to English instead of transcribing. | +| `SPACEBOT_STT_PROVIDER` | None | Override which provider handles speech-to-text (e.g. `groq`, `openai`). | ## Full Reference @@ -85,6 +89,10 @@ branch = "anthropic/claude-sonnet-4-20250514" worker = "anthropic/claude-haiku-4.5-20250514" compactor = "anthropic/claude-haiku-4.5-20250514" cortex = "anthropic/claude-haiku-4.5-20250514" +voice = "groq/whisper-large-v3-turbo" # STT model (provider/model) +voice_language = "en" # optional language hint +# voice_translate = false # set true to translate to English +# stt_provider = "groq" # optional provider override rate_limit_cooldown_secs = 60 # Task-type overrides for workers/branches. @@ -462,6 +470,10 @@ At least one provider (legacy key or custom provider) must be configured. | `worker` | string | `anthropic/claude-haiku-4.5-20250514` | Model for task workers | | `compactor` | string | `anthropic/claude-haiku-4.5-20250514` | Model for summarization | | `cortex` | string | `anthropic/claude-haiku-4.5-20250514` | Model for system observation | +| `voice` | string | Provider-dependent | STT model for audio transcription (e.g. `groq/whisper-large-v3-turbo`). Empty disables voice. See [Voice Transcription](/docs/voice-transcription). | +| `voice_language` | string | None | ISO 639-1 language hint for transcription accuracy (e.g. `en`, `es`, `ja`). Ignored in translation mode. | +| `voice_translate` | bool | `false` | When `true`, translates audio to English via `/v1/audio/translations` instead of transcribing in the source language. | +| `stt_provider` | string | None | Override which provider handles STT. When absent, the provider is extracted from the `voice` model prefix. | | `rate_limit_cooldown_secs` | integer | 60 | How long to deprioritize a rate-limited model | Routing selects providers by the prefix before the first `/` in the model name. diff --git a/docs/content/docs/(core)/routing.mdx b/docs/content/docs/(core)/routing.mdx index 28bc5b195..77b14cbf3 100644 --- a/docs/content/docs/(core)/routing.mdx +++ b/docs/content/docs/(core)/routing.mdx @@ -111,6 +111,10 @@ pub struct RoutingConfig { pub worker: String, pub compactor: String, pub cortex: String, + pub voice: String, + pub voice_language: Option, + pub voice_translate: bool, + pub stt_provider: Option, pub task_overrides: HashMap, pub fallbacks: HashMap>, pub rate_limit_cooldown_secs: u64, @@ -212,6 +216,42 @@ pub struct LlmManager { Rate limit state is shared across all agents (it's provider-level, not agent-level). When a 429 is received, the model is marked with the current timestamp. Future routing decisions can check `is_rate_limited()` to proactively skip models in cooldown. +## Voice Transcription Routing + +Voice transcription (speech-to-text) uses a separate routing path from the main LLM models. When a user sends an audio attachment (e.g. a Telegram voice message), Spacebot transcribes it to text using a Whisper-compatible API before the channel LLM ever sees it. + +Voice routing is independent from the main process-type routing. You can use Anthropic for chat and Groq for transcription. + +```toml +[defaults.routing] +channel = "anthropic/claude-sonnet-4-20250514" # chat +voice = "groq/whisper-large-v3-turbo" # transcription (different provider) +``` + +### How It Routes + +1. `stt_provider` override (if set) determines the provider +2. Otherwise, the provider prefix in `voice` is used (e.g. `groq/` in `groq/whisper-large-v3-turbo`) +3. The provider must support the Whisper-compatible `/v1/audio/transcriptions` endpoint + +### Supported STT Providers + +| Provider | Default Voice Model | Endpoint | +|----------|-------------------|----------| +| OpenAI | `openai/whisper-1` | `/v1/audio/transcriptions` | +| Groq | `groq/whisper-large-v3-turbo` | `/openai/v1/audio/transcriptions` | +| Gemini | `gemini/gemini-2.5-flash` | `/v1/audio/transcriptions` (OpenAI-compatible) | + +Providers without native STT (Anthropic, OpenRouter, DeepSeek, etc.) require configuring a separate STT provider: + +```toml +[defaults.routing] +channel = "openrouter/anthropic/claude-sonnet-4" # chat via OpenRouter +voice = "groq/whisper-large-v3-turbo" # STT via Groq +``` + +See [Voice Transcription](/docs/voice-transcription) for the full feature reference including language hints, translation mode, and configuration examples. + ## What We Don't Do **No prompt-level content analysis.** We know the process type and task type at spawn time. diff --git a/docs/content/docs/(features)/meta.json b/docs/content/docs/(features)/meta.json index a9903832a..068fa7858 100644 --- a/docs/content/docs/(features)/meta.json +++ b/docs/content/docs/(features)/meta.json @@ -1,4 +1,4 @@ { "title": "Features", - "pages": ["workers", "tasks", "opencode", "tools", "mcp", "browser", "cron", "skills", "ingestion"] + "pages": ["workers", "tasks", "opencode", "tools", "mcp", "browser", "cron", "skills", "ingestion", "voice-transcription"] } diff --git a/docs/content/docs/(features)/voice-transcription.mdx b/docs/content/docs/(features)/voice-transcription.mdx new file mode 100644 index 000000000..78c025bc6 --- /dev/null +++ b/docs/content/docs/(features)/voice-transcription.mdx @@ -0,0 +1,194 @@ +--- +title: Voice Transcription +description: Speech-to-text transcription for audio attachments using Whisper-compatible APIs. +--- + +# Voice Transcription + +Spacebot converts audio attachments (Telegram voice messages, Discord audio clips, etc.) to text using Whisper-compatible speech-to-text APIs. The transcript is injected into the conversation before the channel LLM processes it. + +## How It Works + +When a user sends an audio attachment, Spacebot: + +1. Downloads the audio bytes from the messaging platform +2. Resolves the STT provider and model from routing config +3. Sends a multipart `POST` to the provider's `/v1/audio/transcriptions` endpoint +4. Injects the transcript into the conversation as a structured XML tag + +The channel LLM sees the transcript, not raw audio: + +```xml + +Hello, this is what the user said in their voice message. + +``` + +When translation mode is enabled, the tag changes: + +```xml + +Hello, this is the English translation of what the user said. + +``` + +## Configuration + +All voice settings live under `[defaults.routing]` or per-agent `[agents.routing]`. + +```toml +[defaults.routing] +voice = "groq/whisper-large-v3-turbo" +voice_language = "en" # optional +voice_translate = false # optional +stt_provider = "groq" # optional +``` + +### Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `voice` | string | Provider-dependent | STT model in `provider/model` format. Empty string disables voice transcription. | +| `voice_language` | string | None | ISO 639-1 language hint for accuracy (e.g. `en`, `es`, `fr`, `ja`). Ignored in translation mode. | +| `voice_translate` | bool | `false` | When `true`, uses the translations endpoint to translate audio to English. | +| `stt_provider` | string | None | Override which provider handles STT. When absent, provider is extracted from the `voice` model prefix. | + +### Provider Defaults + +When no explicit `voice` is set, Spacebot applies a default based on the primary provider: + +| Primary Provider | Default `voice` | Notes | +|------------------|----------------|-------| +| OpenAI | `openai/whisper-1` | Native Whisper API | +| Groq | `groq/whisper-large-v3-turbo` | Fast and cheap | +| Gemini | `gemini/gemini-2.5-flash` | OpenAI-compatible endpoint | +| OpenRouter | *(empty)* | No native STT — configure `stt_provider` separately | +| Anthropic | *(empty)* | No STT — configure `stt_provider` separately | +| All others | *(empty)* | Must configure `voice` explicitly | + +### Environment Variables + +| Variable | Description | Example | +|----------|-------------|---------| +| `SPACEBOT_VOICE_MODEL` | STT model | `groq/whisper-large-v3-turbo` | +| `SPACEBOT_VOICE_LANGUAGE` | Language hint | `en` | +| `SPACEBOT_VOICE_TRANSLATE` | Translation mode | `true` | +| `SPACEBOT_STT_PROVIDER` | Provider override | `groq` | + +Resolution order: **environment variable > config file > provider default**. + +## Supported Providers + +Voice transcription requires a provider that supports the OpenAI-compatible Whisper API (`/v1/audio/transcriptions` with multipart form data). + +| Provider | Models | Transcription Endpoint | Translation Endpoint | +|----------|--------|----------------------|---------------------| +| **OpenAI** | `whisper-1`, `gpt-4o-transcribe`, `gpt-4o-mini-transcribe` | `/v1/audio/transcriptions` | `/v1/audio/translations` | +| **Groq** | `whisper-large-v3`, `whisper-large-v3-turbo` | `/openai/v1/audio/transcriptions` | `/openai/v1/audio/translations` | +| **Gemini** | `gemini-2.5-flash` (and other Gemini models) | `/v1/audio/transcriptions` | Not supported | + +Providers that do **not** have a transcription endpoint (Anthropic, OpenRouter, DeepSeek, Together, xAI, Mistral, etc.) cannot be used directly for voice. Configure a separate STT provider instead. + +### Supported Audio Formats + +The Whisper API accepts: `flac`, `m4a`, `mp3`, `mp4`, `mpeg`, `mpga`, `oga`, `ogg`, `wav`, `webm`. + +Telegram voice messages (OGG/Opus) are natively supported with no conversion needed. + +## Examples + +### Groq for chat and transcription + +```toml +[llm] +groq_key = "gsk_xxx" + +[defaults.routing] +channel = "groq/llama-3.3-70b-versatile" +voice = "groq/whisper-large-v3-turbo" +``` + +### OpenRouter for chat, Groq for transcription + +```toml +[llm] +openrouter_key = "sk-or-xxx" +groq_key = "gsk_xxx" + +[defaults.routing] +channel = "openrouter/anthropic/claude-sonnet-4" +voice = "groq/whisper-large-v3-turbo" +voice_language = "en" +``` + +### Anthropic for chat, OpenAI for transcription with translation + +```toml +[llm] +anthropic_key = "sk-ant-xxx" +openai_key = "sk-xxx" + +[defaults.routing] +channel = "anthropic/claude-sonnet-4" +voice = "openai/whisper-1" +voice_translate = true +stt_provider = "openai" +``` + +### Multilingual transcription with language hint + +```toml +[llm] +openai_key = "sk-xxx" + +[defaults.routing] +channel = "openai/gpt-4.1" +voice = "openai/whisper-1" +voice_language = "ja" +``` + +### Gemini for everything + +```toml +[llm] +gemini_key = "xxx" + +[defaults.routing] +channel = "gemini/gemini-2.5-pro" +voice = "gemini/gemini-2.5-flash" +``` + +## Error Handling + +Errors are returned as inline text in the conversation so the channel LLM can inform the user: + +| Condition | Message | +|-----------|---------| +| No voice model configured | `[Audio attachment received but no voice model is configured...]` | +| STT provider not found | `[Audio transcription failed: provider 'xxx' is not configured]` | +| Provider doesn't support Whisper | `[Audio transcription not supported by provider 'xxx'...]` | +| API error | `[Audio transcription failed for filename.ogg: Whisper API error (400): ...]` | +| Download failure | `[Failed to download audio: filename.ogg]` | + +There is no fallback to alternative transcription methods. If transcription fails, the error is returned directly. + +## API + +### Runtime Configuration + +Voice settings are included in the agent config API: + +``` +GET /api/config?agent_id=main +PATCH /api/config { "agent_id": "main", "routing": { "voice": "...", ... } } +``` + +### Model Discovery + +Filter models to transcription-capable providers: + +``` +GET /api/models?capability=voice_transcription +``` + +Returns models from providers that support the Whisper-compatible transcription endpoint (currently: OpenAI, Groq, Gemini). diff --git a/src/agent/channel_attachments.rs b/src/agent/channel_attachments.rs index 7d7d21bb5..797d70bd9 100644 --- a/src/agent/channel_attachments.rs +++ b/src/agent/channel_attachments.rs @@ -9,7 +9,8 @@ //! table for later recall. use crate::AgentDeps; -use crate::config::ApiType; +use crate::llm::transcription::supports_whisper_transcription; +use crate::llm::{TranscriptionRequest, transcribe_audio}; use rig::message::{ImageMediaType, MimeType, UserContent}; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; @@ -213,194 +214,74 @@ async fn transcribe_audio_attachment( let routing = deps.runtime_config.routing.load(); let voice_model = routing.voice.trim(); + if voice_model.is_empty() { - return UserContent::text(format!( - "[Audio attachment received but no voice model is configured in routing.voice: {}]", - attachment.filename - )); + return UserContent::text( + "[Audio attachment received but no voice model is configured. \ + Add `voice = \"provider/model\"` to [defaults.routing] in config.]", + ); } - let (provider_id, model_name) = match deps.llm_manager.resolve_model(voice_model) { - Ok(parts) => parts, - Err(error) => { - tracing::warn!(%error, model = %voice_model, "invalid voice model route"); - return UserContent::text(format!( - "[Audio transcription failed for {}: invalid voice model '{}']", - attachment.filename, voice_model - )); - } - }; + let provider_id = routing.stt_provider.as_deref().unwrap_or_else(|| { + voice_model + .split_once('/') + .map(|(p, _)| p) + .unwrap_or("anthropic") + }); + + let model_name = voice_model + .split_once('/') + .map(|(_, m)| m) + .unwrap_or(voice_model); let provider = match deps.llm_manager.get_provider(&provider_id) { - Ok(provider) => provider, + Ok(p) => p, Err(error) => { - tracing::warn!(%error, provider = %provider_id, "voice provider not configured"); + tracing::warn!(%error, provider = %provider_id, "STT provider not configured"); return UserContent::text(format!( - "[Audio transcription failed for {}: provider '{}' is not configured]", - attachment.filename, provider_id + "[Audio transcription failed: provider '{}' is not configured]", + provider_id )); } }; - if provider.api_type == ApiType::Anthropic { + if !supports_whisper_transcription(&provider) { return UserContent::text(format!( - "[Audio transcription failed for {}: provider '{}' does not support input_audio on this endpoint]", - attachment.filename, provider_id + "[Audio transcription not supported by provider '{}'. \ + Configure a Whisper-compatible STT provider (openai, groq, gemini).]", + provider_id )); } - let format = audio_format_for_attachment(attachment); - use base64::Engine as _; - let base64_audio = base64::engine::general_purpose::STANDARD.encode(&bytes); - - let endpoint = format!( - "{}/v1/chat/completions", - provider.base_url.trim_end_matches('/') - ); - let body = serde_json::json!({ - "model": model_name, - "messages": [{ - "role": "user", - "content": [ - { - "type": "text", - "text": "Transcribe this audio verbatim. Return only the transcription text." - }, - { - "type": "input_audio", - "input_audio": { - "data": base64_audio, - "format": format, - } - } - ] - }], - "temperature": 0 - }); - - let response = match deps - .llm_manager - .http_client() - .post(&endpoint) - .header("authorization", format!("Bearer {}", provider.api_key)) - .header("content-type", "application/json") - .json(&body) - .send() - .await - { - Ok(response) => response, - Err(error) => { - tracing::warn!(%error, model = %voice_model, "voice transcription request failed"); - return UserContent::text(format!( - "[Audio transcription failed for {}]", - attachment.filename - )); - } + let request = TranscriptionRequest { + audio_bytes: &bytes, + filename: &attachment.filename, + mime_type: &attachment.mime_type, + model: model_name, + language: routing.voice_language.as_deref(), + translate: routing.voice_translate, }; - let status = response.status(); - let response_body = match response.json::().await { - Ok(body) => body, + match transcribe_audio(deps.llm_manager.http_client(), &provider, request).await { + Ok(response) => { + let tag = if response.translated { + "voice_translation" + } else { + "voice_transcript" + }; + UserContent::text(format!( + "<{} name=\"{}\" mime=\"{}\">\n{}\n", + tag, attachment.filename, attachment.mime_type, response.text, tag + )) + } Err(error) => { - tracing::warn!(%error, model = %voice_model, "invalid transcription response"); - return UserContent::text(format!( - "[Audio transcription failed for {}]", - attachment.filename - )); + tracing::warn!(%error, "audio transcription failed"); + UserContent::text(format!( + "[Audio transcription failed for {}: {}]", + attachment.filename, error + )) } - }; - - if !status.is_success() { - let message = response_body["error"]["message"] - .as_str() - .unwrap_or("unknown error"); - tracing::warn!( - status = %status, - model = %voice_model, - error = %message, - "voice transcription provider returned error" - ); - return UserContent::text(format!( - "[Audio transcription failed for {}: {}]", - attachment.filename, message - )); - } - - let transcript = extract_transcript_text(&response_body); - if transcript.is_empty() { - tracing::warn!(model = %voice_model, "empty transcription returned"); - return UserContent::text(format!( - "[Audio transcription returned empty text for {}]", - attachment.filename - )); - } - - UserContent::text(format!( - "\n{}\n", - attachment.filename, attachment.mime_type, transcript - )) -} - -fn audio_format_for_attachment(attachment: &crate::Attachment) -> &'static str { - let mime = attachment.mime_type.to_lowercase(); - if mime.contains("mpeg") || mime.contains("mp3") { - return "mp3"; - } - if mime.contains("wav") { - return "wav"; - } - if mime.contains("flac") { - return "flac"; - } - if mime.contains("aac") { - return "aac"; } - if mime.contains("ogg") { - return "ogg"; - } - if mime.contains("mp4") || mime.contains("m4a") { - return "m4a"; - } - - match attachment - .filename - .rsplit('.') - .next() - .unwrap_or_default() - .to_lowercase() - .as_str() - { - "mp3" => "mp3", - "wav" => "wav", - "flac" => "flac", - "aac" => "aac", - "m4a" | "mp4" => "m4a", - "oga" | "ogg" => "ogg", - _ => "ogg", - } -} - -fn extract_transcript_text(body: &serde_json::Value) -> String { - if let Some(text) = body["choices"][0]["message"]["content"].as_str() { - return text.trim().to_string(); - } - - let Some(parts) = body["choices"][0]["message"]["content"].as_array() else { - return String::new(); - }; - - parts - .iter() - .filter_map(|part| { - if part["type"].as_str() == Some("text") { - part["text"].as_str().map(str::trim) - } else { - None - } - }) - .filter(|text| !text.is_empty()) - .collect::>() - .join("\n") } /// Download a text attachment and inline its content for the LLM. diff --git a/src/api/config.rs b/src/api/config.rs index 59a5f4101..5e66154b8 100644 --- a/src/api/config.rs +++ b/src/api/config.rs @@ -15,6 +15,9 @@ pub(super) struct RoutingSection { compactor: String, cortex: String, voice: String, + voice_language: Option, + voice_translate: bool, + stt_provider: Option, rate_limit_cooldown_secs: u64, } @@ -171,6 +174,9 @@ pub(super) struct RoutingUpdate { compactor: Option, cortex: Option, voice: Option, + voice_language: Option, + voice_translate: Option, + stt_provider: Option, rate_limit_cooldown_secs: Option, } @@ -298,6 +304,9 @@ pub(super) async fn get_agent_config( compactor: routing.compactor.clone(), cortex: routing.cortex.clone(), voice: routing.voice.clone(), + voice_language: routing.voice_language.clone(), + voice_translate: routing.voice_translate, + stt_provider: routing.stt_provider.clone(), rate_limit_cooldown_secs: routing.rate_limit_cooldown_secs, }, tuning: TuningSection { @@ -604,6 +613,15 @@ fn update_routing_table( if let Some(ref v) = routing.voice { table["voice"] = toml_edit::value(v.as_str()); } + if let Some(ref v) = routing.voice_language { + table["voice_language"] = toml_edit::value(v.as_str()); + } + if let Some(v) = routing.voice_translate { + table["voice_translate"] = toml_edit::value(v); + } + if let Some(ref v) = routing.stt_provider { + table["stt_provider"] = toml_edit::value(v.as_str()); + } if let Some(v) = routing.rate_limit_cooldown_secs { table["rate_limit_cooldown_secs"] = toml_edit::value(v as i64); } diff --git a/src/api/models.rs b/src/api/models.rs index f988bf6b6..d65df0541 100644 --- a/src/api/models.rs +++ b/src/api/models.rs @@ -78,26 +78,8 @@ static MODELS_CACHE: std::sync::LazyLock< const MODELS_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(3600); -/// Models known to work with Spacebot's current voice transcription path -/// (OpenAI-compatible `/v1/chat/completions` with `input_audio`). -const KNOWN_VOICE_TRANSCRIPTION_MODELS: &[&str] = &[ - // Native Gemini API - "gemini/gemini-2.0-flash", - "gemini/gemini-2.5-flash", - "gemini/gemini-2.5-flash-lite", - "gemini/gemini-2.5-pro", - "gemini/gemini-3-flash-preview", - "gemini/gemini-3-pro-preview", - "gemini/gemini-3.1-pro-preview", - // Via OpenRouter - "openrouter/google/gemini-2.0-flash-001", - "openrouter/google/gemini-2.5-flash", - "openrouter/google/gemini-2.5-flash-lite", - "openrouter/google/gemini-2.5-pro", - "openrouter/google/gemini-3-flash-preview", - "openrouter/google/gemini-3-pro-preview", - "openrouter/google/gemini-3.1-pro-preview", -]; +/// Providers that support Whisper-compatible `/v1/audio/transcriptions` endpoint. +const WHISPER_CAPABLE_PROVIDERS: &[&str] = &["openai", "groq", "gemini"]; /// Maps models.dev provider IDs to spacebot's internal provider IDs for /// providers with direct integrations. @@ -123,8 +105,9 @@ fn direct_provider_mapping(models_dev_id: &str) -> Option<&'static str> { } } -fn is_known_voice_transcription_model(model_id: &str) -> bool { - KNOWN_VOICE_TRANSCRIPTION_MODELS.contains(&model_id) +/// Returns true if the provider supports Whisper-compatible voice transcription. +fn supports_voice_transcription(provider: &str) -> bool { + WHISPER_CAPABLE_PROVIDERS.contains(&provider) } fn as_openai_chatgpt_model(model: &ModelInfo) -> Option { @@ -378,9 +361,7 @@ pub(super) async fn get_models( if let Some(capability) = requested_capability { match capability { "input_audio" => model.input_audio, - "voice_transcription" => { - model.input_audio && is_known_voice_transcription_model(&model.id) - } + "voice_transcription" => supports_voice_transcription(&model.provider), _ => true, } } else { @@ -423,8 +404,7 @@ pub(super) async fn get_models( if capability == "input_audio" && !model.input_audio { continue; } - if capability == "voice_transcription" - && (!model.input_audio || !is_known_voice_transcription_model(&model.id)) + if capability == "voice_transcription" && !supports_voice_transcription(&model.provider) { continue; } diff --git a/src/config/load.rs b/src/config/load.rs index 05a99aedc..c6fbcc9df 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -830,6 +830,17 @@ impl Config { if let Ok(voice_model) = std::env::var("SPACEBOT_VOICE_MODEL") { routing.voice = voice_model; } + if let Ok(voice_language) = std::env::var("SPACEBOT_VOICE_LANGUAGE") { + routing.voice_language = Some(voice_language); + } + if let Ok(voice_translate) = std::env::var("SPACEBOT_VOICE_TRANSLATE") { + if voice_translate.eq_ignore_ascii_case("true") { + routing.voice_translate = true; + } + } + if let Ok(stt_provider) = std::env::var("SPACEBOT_STT_PROVIDER") { + routing.stt_provider = Some(stt_provider); + } let agents = vec![AgentConfig { id: "main".into(), diff --git a/src/config/providers.rs b/src/config/providers.rs index 091ca71cc..12eca82ae 100644 --- a/src/config/providers.rs +++ b/src/config/providers.rs @@ -316,6 +316,9 @@ pub(super) fn resolve_routing( compactor: t.compactor.unwrap_or_else(|| base.compactor.clone()), cortex: t.cortex.unwrap_or_else(|| base.cortex.clone()), voice: t.voice.unwrap_or_else(|| base.voice.clone()), + voice_language: t.voice_language.or(base.voice_language.clone()), + voice_translate: t.voice_translate.unwrap_or(base.voice_translate), + stt_provider: t.stt_provider.or(base.stt_provider.clone()), task_overrides, fallbacks, rate_limit_cooldown_secs: t diff --git a/src/config/toml_schema.rs b/src/config/toml_schema.rs index d68cc7615..ac1060614 100644 --- a/src/config/toml_schema.rs +++ b/src/config/toml_schema.rs @@ -304,6 +304,9 @@ pub(super) struct TomlRoutingConfig { pub(super) compactor: Option, pub(super) cortex: Option, pub(super) voice: Option, + pub(super) voice_language: Option, + pub(super) voice_translate: Option, + pub(super) stt_provider: Option, pub(super) rate_limit_cooldown_secs: Option, pub(super) channel_thinking_effort: Option, pub(super) branch_thinking_effort: Option, diff --git a/src/llm.rs b/src/llm.rs index 68776ebf3..2a52d29be 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -6,7 +6,10 @@ pub mod model; pub mod pricing; pub mod providers; pub mod routing; +pub mod transcription; pub use manager::LlmManager; pub use model::SpacebotModel; pub use routing::RoutingConfig; +// Re-export types from transcription module +pub use transcription::{TranscriptionRequest, TranscriptionResponse, transcribe_audio}; diff --git a/src/llm/routing.rs b/src/llm/routing.rs index eb677a8c8..37731f0a5 100644 --- a/src/llm/routing.rs +++ b/src/llm/routing.rs @@ -32,6 +32,13 @@ pub struct RoutingConfig { pub worker_thinking_effort: String, pub compactor_thinking_effort: String, pub cortex_thinking_effort: String, + + /// Language hint for voice transcription accuracy (e.g., "en", "es"). + pub voice_language: Option, + /// If true, use translations endpoint (translates to English). + pub voice_translate: bool, + /// Optional STT provider override (defaults to voice model provider). + pub stt_provider: Option, } impl Default for RoutingConfig { @@ -58,6 +65,9 @@ impl RoutingConfig { worker_thinking_effort: "auto".into(), compactor_thinking_effort: "auto".into(), cortex_thinking_effort: "auto".into(), + voice_language: None, + voice_translate: false, + stt_provider: None, } } } @@ -205,7 +215,7 @@ pub fn defaults_for_provider(provider: &str) -> RoutingConfig { worker: worker.clone(), compactor: worker.clone(), cortex: worker.clone(), - voice: String::new(), + voice: "openai/whisper-1".into(), task_overrides: HashMap::from([("coding".into(), channel.clone())]), fallbacks: HashMap::from([(channel, vec![worker])]), rate_limit_cooldown_secs: 60, @@ -253,7 +263,7 @@ pub fn defaults_for_provider(provider: &str) -> RoutingConfig { worker: worker.clone(), compactor: worker.clone(), cortex: worker.clone(), - voice: String::new(), + voice: "groq/whisper-large-v3-turbo".into(), task_overrides: HashMap::from([("coding".into(), channel.clone())]), fallbacks: HashMap::from([(channel, vec![worker])]), rate_limit_cooldown_secs: 60, @@ -352,7 +362,7 @@ pub fn defaults_for_provider(provider: &str) -> RoutingConfig { worker: worker.clone(), compactor: worker.clone(), cortex: worker.clone(), - voice: String::new(), + voice: "gemini/gemini-2.5-flash".into(), task_overrides: HashMap::from([("coding".into(), channel.clone())]), fallbacks: HashMap::from([(channel, vec![worker.clone()]), (worker, vec![lite])]), rate_limit_cooldown_secs: 60, diff --git a/src/llm/transcription.rs b/src/llm/transcription.rs new file mode 100644 index 000000000..93bee3188 --- /dev/null +++ b/src/llm/transcription.rs @@ -0,0 +1,414 @@ +//! Whisper-compatible audio transcription. +//! +//! Provides speech-to-text transcription using OpenAI-compatible /v1/audio/transcriptions +//! endpoints (OpenAI Whisper, Groq Whisper, Gemini OpenAI-compatible). + +use crate::config::ProviderConfig; +use crate::error::{LlmError, Result}; + +/// Request for audio transcription. +pub struct TranscriptionRequest<'a> { + /// Raw audio bytes. + pub audio_bytes: &'a [u8], + /// Original filename (used for MIME detection). + pub filename: &'a str, + /// MIME type of the audio. + pub mime_type: &'a str, + /// Model name (e.g., "whisper-1", "whisper-large-v3-turbo"). + pub model: &'a str, + /// Optional language hint for accuracy (e.g., "en", "es"). + pub language: Option<&'a str>, + /// If true, use /audio/translations endpoint (translates to English). + pub translate: bool, +} + +/// Response from audio transcription. +pub struct TranscriptionResponse { + /// Transcribed text. + pub text: String, + /// Duration of the input audio in seconds (if provided by API). + pub duration_secs: Option, + /// True if translation mode was used. + pub translated: bool, +} + +/// Transcribe audio using a Whisper-compatible API. +/// +/// Supports OpenAI, Groq, and Gemini OpenAI-compatible endpoints. +/// Uses multipart form data for the request. +pub async fn transcribe_audio( + http: &reqwest::Client, + provider: &ProviderConfig, + request: TranscriptionRequest<'_>, +) -> Result { + transcribe_with_whisper_compatible(http, provider, request).await +} + +async fn transcribe_with_whisper_compatible( + http: &reqwest::Client, + provider: &ProviderConfig, + request: TranscriptionRequest<'_>, +) -> Result { + let endpoint = build_whisper_endpoint(&provider.base_url, request.translate); + let form = build_multipart_form(&request)?; + + let mut request_builder = http + .post(&endpoint) + .header("Authorization", format!("Bearer {}", provider.api_key)); + + for (key, value) in &provider.extra_headers { + request_builder = request_builder.header(key, value); + } + + let response = request_builder + .multipart(form) + .send() + .await + .map_err(|e| LlmError::ProviderRequest(format!("HTTP request failed: {}", e)))?; + parse_whisper_response(response, request.translate).await +} + +/// Build the Whisper API endpoint URL. +/// +/// Provider-specific paths: +/// - Groq: /openai/v1/audio/{transcriptions,translations} +/// - OpenAI/Gemini: /v1/audio/{transcriptions,translations} +fn build_whisper_endpoint(base_url: &str, translate: bool) -> String { + let base = base_url.trim_end_matches('/'); + let path = if translate { + "audio/translations" + } else { + "audio/transcriptions" + }; + + if base.contains("groq.com") { + format!("{}/openai/v1/{}", base, path) + } else { + format!("{}/v1/{}", base, path) + } +} + +/// Build a multipart form for the Whisper API request. +fn build_multipart_form(request: &TranscriptionRequest<'_>) -> Result { + let audio_part = reqwest::multipart::Part::bytes(request.audio_bytes.to_vec()) + .file_name(request.filename.to_string()) + .mime_str(request.mime_type) + .map_err(|e| LlmError::ProviderRequest(format!("invalid MIME type: {}", e)))?; + + let mut form = reqwest::multipart::Form::new() + .part("file", audio_part) + .text("model", request.model.to_string()) + .text("response_format", "json"); + + // Language hint only valid for transcriptions, not translations + if let Some(lang) = request.language { + if !request.translate { + form = form.text("language", lang.to_string()); + } + } + + Ok(form) +} + +/// Parse the Whisper API response. +async fn parse_whisper_response( + response: reqwest::Response, + translated: bool, +) -> Result { + let status = response.status(); + let body: serde_json::Value = response + .json() + .await + .map_err(|e| LlmError::ProviderRequest(format!("failed to parse response JSON: {}", e)))?; + + if !status.is_success() { + let message = body["error"]["message"].as_str().unwrap_or("unknown error"); + return Err(LlmError::ProviderRequest(format!( + "Whisper API error ({}): {}", + status, message + )) + .into()); + } + + let text = body["text"] + .as_str() + .ok_or_else(|| { + LlmError::ProviderRequest("missing text in transcription response".to_string()) + })? + .to_string(); + + let duration_secs = body["duration"].as_f64(); + + Ok(TranscriptionResponse { + text, + duration_secs, + translated, + }) +} + +/// Check if a provider supports Whisper-compatible transcription. +/// +/// Supports: OpenAI, Groq, Gemini (via OpenAI-compatible endpoint). +pub fn supports_whisper_transcription(provider: &ProviderConfig) -> bool { + let base = provider.base_url.to_lowercase(); + base.contains("openai.com") + || base.contains("groq.com") + || base.contains("googleapis.com") + || base.contains("generativelanguage.googleapis.com") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_transcription_request<'a>( + audio: &'a [u8], + language: Option<&'a str>, + translate: bool, + ) -> TranscriptionRequest<'a> { + TranscriptionRequest { + audio_bytes: audio, + filename: "test.mp3", + mime_type: "audio/mpeg", + model: "whisper-1", + language, + translate, + } + } + + #[test] + fn test_build_whisper_endpoint_openai_transcription() { + let url = build_whisper_endpoint("https://api.openai.com", false); + assert_eq!(url, "https://api.openai.com/v1/audio/transcriptions"); + } + + #[test] + fn test_build_whisper_endpoint_openai_translation() { + let url = build_whisper_endpoint("https://api.openai.com", true); + assert_eq!(url, "https://api.openai.com/v1/audio/translations"); + } + + #[test] + fn test_build_whisper_endpoint_groq_transcription() { + let url = build_whisper_endpoint("https://api.groq.com/openai", false); + assert_eq!( + url, + "https://api.groq.com/openai/openai/v1/audio/transcriptions" + ); + } + + #[test] + fn test_build_whisper_endpoint_groq_translation() { + let url = build_whisper_endpoint("https://api.groq.com/openai", true); + assert_eq!( + url, + "https://api.groq.com/openai/openai/v1/audio/translations" + ); + } + + #[test] + fn test_build_whisper_endpoint_gemini() { + let url = build_whisper_endpoint( + "https://generativelanguage.googleapis.com/v1beta/openai", + false, + ); + assert_eq!( + url, + "https://generativelanguage.googleapis.com/v1beta/openai/v1/audio/transcriptions" + ); + } + + #[test] + fn test_build_whisper_endpoint_gemini_translation() { + let url = build_whisper_endpoint( + "https://generativelanguage.googleapis.com/v1beta/openai", + true, + ); + assert_eq!( + url, + "https://generativelanguage.googleapis.com/v1beta/openai/v1/audio/translations" + ); + } + + #[test] + fn test_build_whisper_endpoint_trailing_slash() { + let url = build_whisper_endpoint("https://api.openai.com/", false); + assert_eq!(url, "https://api.openai.com/v1/audio/transcriptions"); + } + + #[test] + fn test_build_multipart_form_with_language() { + let audio = vec![0u8; 100]; + let request = test_transcription_request(&audio, Some("en"), false); + // Form builds successfully with language hint for transcription + assert!(build_multipart_form(&request).is_ok()); + } + + #[test] + fn test_build_multipart_form_translation_ignores_language() { + let audio = vec![0u8; 100]; + // Translation mode with language hint — should still build OK + // (language is silently ignored for translations) + let request = test_transcription_request(&audio, Some("es"), true); + assert!(build_multipart_form(&request).is_ok()); + } + + #[test] + fn test_build_multipart_form_no_language() { + let audio = vec![0u8; 100]; + let request = test_transcription_request(&audio, None, false); + assert!(build_multipart_form(&request).is_ok()); + } + + #[test] + fn test_build_multipart_form_success() { + let audio = vec![0u8; 100]; + let request = test_transcription_request(&audio, None, false); + // Verify the form builds without error — reqwest::multipart::Form + // is opaque so we can't inspect individual fields. + assert!(build_multipart_form(&request).is_ok()); + } + + #[test] + fn test_supports_whisper_transcription_openai() { + let provider = ProviderConfig { + api_type: crate::config::ApiType::OpenAiCompletions, + base_url: "https://api.openai.com".to_string(), + api_key: "test-key".to_string(), + name: Some("OpenAI".to_string()), + use_bearer_auth: false, + extra_headers: vec![], + }; + assert!(supports_whisper_transcription(&provider)); + } + + #[test] + fn test_supports_whisper_transcription_groq() { + let provider = ProviderConfig { + api_type: crate::config::ApiType::OpenAiCompletions, + base_url: "https://api.groq.com/openai".to_string(), + api_key: "test-key".to_string(), + name: Some("Groq".to_string()), + use_bearer_auth: false, + extra_headers: vec![], + }; + assert!(supports_whisper_transcription(&provider)); + } + + #[test] + fn test_supports_whisper_transcription_gemini() { + let provider = ProviderConfig { + api_type: crate::config::ApiType::Gemini, + base_url: "https://generativelanguage.googleapis.com/v1beta/openai".to_string(), + api_key: "test-key".to_string(), + name: Some("Gemini".to_string()), + use_bearer_auth: false, + extra_headers: vec![], + }; + assert!(supports_whisper_transcription(&provider)); + } + + #[test] + fn test_supports_whisper_transcription_anthropic_not_supported() { + let provider = ProviderConfig { + api_type: crate::config::ApiType::Anthropic, + base_url: "https://api.anthropic.com".to_string(), + api_key: "test-key".to_string(), + name: Some("Anthropic".to_string()), + use_bearer_auth: false, + extra_headers: vec![], + }; + assert!(!supports_whisper_transcription(&provider)); + } + + #[test] + fn test_supports_whisper_transcription_openrouter_not_supported() { + let provider = ProviderConfig { + api_type: crate::config::ApiType::OpenAiCompletions, + base_url: "https://openrouter.ai/api/v1".to_string(), + api_key: "test-key".to_string(), + name: Some("OpenRouter".to_string()), + use_bearer_auth: false, + extra_headers: vec![], + }; + assert!(!supports_whisper_transcription(&provider)); + } + + #[test] + fn test_supports_whisper_transcription_case_insensitive() { + let provider = ProviderConfig { + api_type: crate::config::ApiType::OpenAiCompletions, + base_url: "https://API.OPENAI.COM".to_string(), + api_key: "test-key".to_string(), + name: Some("OpenAI".to_string()), + use_bearer_auth: false, + extra_headers: vec![], + }; + assert!(supports_whisper_transcription(&provider)); + } + + #[test] + fn test_transcription_request_lifetimes() { + let audio = vec![1u8, 2, 3, 4, 5]; + let filename = "recording.mp3"; + let mime = "audio/mpeg"; + let model = "whisper-large-v3"; + let lang = "es"; + + let request = TranscriptionRequest { + audio_bytes: &audio, + filename, + mime_type: mime, + model, + language: Some(lang), + translate: false, + }; + + assert_eq!(request.audio_bytes.len(), 5); + assert_eq!(request.filename, "recording.mp3"); + assert_eq!(request.mime_type, "audio/mpeg"); + assert_eq!(request.model, "whisper-large-v3"); + assert_eq!(request.language, Some("es")); + assert!(!request.translate); + } + + #[test] + fn test_transcription_response_fields() { + let response = TranscriptionResponse { + text: "Hello, world!".to_string(), + duration_secs: Some(5.5), + translated: false, + }; + + assert_eq!(response.text, "Hello, world!"); + assert_eq!(response.duration_secs, Some(5.5)); + assert!(!response.translated); + } + + #[test] + fn test_transcription_response_translation_mode() { + let response = TranscriptionResponse { + text: "Hello, world!".to_string(), + duration_secs: None, + translated: true, + }; + + assert!(response.translated); + assert!(response.duration_secs.is_none()); + } + + #[test] + fn test_build_multipart_form_invalid_mime() { + let audio = vec![0u8; 100]; + let request = TranscriptionRequest { + audio_bytes: &audio, + filename: "test.mp3", + mime_type: "invalid/mime type with spaces", + model: "whisper-1", + language: None, + translate: false, + }; + let result = build_multipart_form(&request); + assert!(result.is_err()); + } +}