From 5ef5d7f50001bba13aed7d6e711044c468b67ea2 Mon Sep 17 00:00:00 2001 From: Zane Staggs Date: Tue, 26 Aug 2025 13:17:36 -0700 Subject: [PATCH 1/6] fix eleven labs and add logging --- crates/goose-server/src/routes/audio.rs | 37 ++++---- .../src/routes/config_management.rs | 5 +- .../settings/dictation/DictationSection.tsx | 30 ++++-- ui/desktop/src/hooks/useWhisper.ts | 92 +++++++++++++++---- 4 files changed, 121 insertions(+), 43 deletions(-) diff --git a/crates/goose-server/src/routes/audio.rs b/crates/goose-server/src/routes/audio.rs index e0071249b79f..62ab5970f651 100644 --- a/crates/goose-server/src/routes/audio.rs +++ b/crates/goose-server/src/routes/audio.rs @@ -209,7 +209,7 @@ async fn transcribe_elevenlabs_handler( let config = goose::config::Config::global(); // First try to get it as a secret - let api_key: String = match config.get_secret("ELEVENLABS_API_KEY") { + let api_key: String = match config.get_secret::("ELEVENLABS_API_KEY") { Ok(key) => key, Err(_) => { // Try to get it as non-secret (for backward compatibility) @@ -217,7 +217,6 @@ async fn transcribe_elevenlabs_handler( Ok(value) => { match value.as_str() { Some(key_str) => { - tracing::info!("Migrating ElevenLabs API key to secret storage"); let key = key_str.to_string(); // Migrate to secret storage if let Err(e) = config.set( @@ -228,17 +227,25 @@ async fn transcribe_elevenlabs_handler( tracing::error!("Failed to migrate ElevenLabs API key: {:?}", e); } // Delete the non-secret version - let _ = config.delete("ELEVENLABS_API_KEY"); + if let Err(e) = config.delete("ELEVENLABS_API_KEY") { + tracing::warn!( + "Failed to delete non-secret ElevenLabs API key: {:?}", + e + ); + } key } None => { - tracing::error!("ElevenLabs API key is not a string"); + tracing::error!( + "ElevenLabs API key is not a string, found: {:?}", + value + ); return Err(StatusCode::PRECONDITION_FAILED); } } } - Err(e) => { - tracing::error!("Failed to get ElevenLabs API key from config: {:?}", e); + Err(_) => { + tracing::error!("No ElevenLabs API key found in configuration"); return Err(StatusCode::PRECONDITION_FAILED); } } @@ -286,8 +293,9 @@ async fn transcribe_elevenlabs_handler( })?; if !response.status().is_success() { + let status = response.status(); let error_text = response.text().await.unwrap_or_default(); - tracing::error!("ElevenLabs API error: {}", error_text); + tracing::error!("ElevenLabs API error (status: {}): {}", status, error_text); // Check for specific error codes if error_text.contains("Unauthorized") || error_text.contains("Invalid API key") { @@ -330,16 +338,13 @@ async fn check_dictation_config( let config = goose::config::Config::global(); // Check if ElevenLabs API key is configured - let has_elevenlabs = config - .get_secret::("ELEVENLABS_API_KEY") - .map(|_| true) - .unwrap_or_else(|_| { + let has_elevenlabs = match config.get_secret::("ELEVENLABS_API_KEY") { + Ok(_) => true, + Err(_) => { // Check non-secret for backward compatibility - config - .get("ELEVENLABS_API_KEY", false) - .map(|_| true) - .unwrap_or(false) - }); + config.get("ELEVENLABS_API_KEY", false).is_ok() + } + }; Ok(Json(serde_json::json!({ "elevenlabs": has_elevenlabs diff --git a/crates/goose-server/src/routes/config_management.rs b/crates/goose-server/src/routes/config_management.rs index 0224cc79565d..c243b1511f6c 100644 --- a/crates/goose-server/src/routes/config_management.rs +++ b/crates/goose-server/src/routes/config_management.rs @@ -167,6 +167,7 @@ pub async fn read_config( } let config = Config::global(); + let response_value = match config.get(&query.key, query.is_secret) { Ok(value) => { if query.is_secret { @@ -182,7 +183,9 @@ pub async fn read_config( Value::Null } } - Err(_) => return Err(StatusCode::INTERNAL_SERVER_ERROR), + Err(_) => { + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } }; Ok(Json(response_value)) } diff --git a/ui/desktop/src/components/settings/dictation/DictationSection.tsx b/ui/desktop/src/components/settings/dictation/DictationSection.tsx index 23f8e16b5fb3..669a30873aa7 100644 --- a/ui/desktop/src/components/settings/dictation/DictationSection.tsx +++ b/ui/desktop/src/components/settings/dictation/DictationSection.tsx @@ -27,25 +27,30 @@ export default function DictationSection() { useEffect(() => { const loadSettings = async () => { const savedSettings = localStorage.getItem(DICTATION_SETTINGS_KEY); + let loadedSettings: DictationSettings; + if (savedSettings) { const parsed = JSON.parse(savedSettings); - setSettings(parsed); - setShowElevenLabsKey(parsed.provider === 'elevenlabs'); + loadedSettings = parsed; + console.log('Loaded dictation settings from localStorage:', parsed); } else { - // Default settings - const defaultSettings: DictationSettings = { - enabled: true, - provider: 'openai', + // Default settings - don't force OpenAI as default + loadedSettings = { + enabled: false, + provider: null, }; - setSettings(defaultSettings); - localStorage.setItem(DICTATION_SETTINGS_KEY, JSON.stringify(defaultSettings)); + console.log('Using default dictation settings (no saved settings found):', loadedSettings); } + setSettings(loadedSettings); + setShowElevenLabsKey(loadedSettings.provider === 'elevenlabs'); + // Load ElevenLabs API key from storage setIsLoadingKey(true); try { // Try reading as secret - will return true if exists const keyExists = await read(ELEVENLABS_API_KEY, true); + console.log('ElevenLabs API key exists in secure storage:', keyExists); if (keyExists === true) { setHasElevenLabsKey(true); // Don't set the actual key since we can't read secrets @@ -109,6 +114,7 @@ export default function DictationSection() { }; const saveSettings = (newSettings: DictationSettings) => { + console.log('Saving dictation settings to localStorage:', newSettings); setSettings(newSettings); localStorage.setItem(DICTATION_SETTINGS_KEY, JSON.stringify(newSettings)); }; @@ -130,18 +136,26 @@ export default function DictationSection() { const handleElevenLabsKeyChange = (key: string) => { setElevenLabsApiKey(key); elevenLabsApiKeyRef.current = key; + // If user starts typing, they're updating the key + if (key.length > 0) { + setHasElevenLabsKey(false); // Hide "configured" while typing + } }; const saveElevenLabsKey = async () => { // Save to secure storage try { if (elevenLabsApiKey.trim()) { + console.log('Saving ElevenLabs API key to secure storage...'); await upsert(ELEVENLABS_API_KEY, elevenLabsApiKey, true); setHasElevenLabsKey(true); + console.log('ElevenLabs API key saved successfully'); } else { // If key is empty, remove it from storage + console.log('Removing ElevenLabs API key from secure storage...'); await upsert(ELEVENLABS_API_KEY, null, true); setHasElevenLabsKey(false); + console.log('ElevenLabs API key removed successfully'); } } catch (error) { console.error('Error saving ElevenLabs API key:', error); diff --git a/ui/desktop/src/hooks/useWhisper.ts b/ui/desktop/src/hooks/useWhisper.ts index 24c9847a0a4e..cc13807b7acc 100644 --- a/ui/desktop/src/hooks/useWhisper.ts +++ b/ui/desktop/src/hooks/useWhisper.ts @@ -87,7 +87,7 @@ export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisp // Define stopRecording before startRecording to avoid circular dependency const stopRecording = useCallback(() => { - setIsRecording(false); // Always update the visual state + setIsRecording(false); if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') { mediaRecorderRef.current.stop(); @@ -166,7 +166,7 @@ export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisp }; let body: Record = { audio: base64Audio, - mime_type: 'audio/webm', + mime_type: audioBlob.type || 'audio/webm', }; // Choose endpoint based on provider @@ -234,23 +234,42 @@ export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisp try { // Request microphone permission - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + sampleRate: 44100, + }, + }); streamRef.current = stream; - // Create audio context and analyser for visualization - const context = new AudioContext(); - const source = context.createMediaStreamSource(stream); - const analyserNode = context.createAnalyser(); - analyserNode.fftSize = 2048; - source.connect(analyserNode); + // Verify we have valid audio tracks + const audioTracks = stream.getAudioTracks(); + if (audioTracks.length === 0) { + throw new Error('No audio tracks available in the microphone stream'); + } - setAudioContext(context); - setAnalyser(analyserNode); + // Note: AudioContext creation is disabled to prevent MediaRecorder conflicts + // The AudioContext was interfering with MediaRecorder, causing the stream to become inactive + setAudioContext(null); + setAnalyser(null); - // Create MediaRecorder - const mediaRecorder = new MediaRecorder(stream, { - mimeType: 'audio/webm', - }); + // Check supported MIME types and create MediaRecorder + let mimeType = 'audio/webm'; + if (!MediaRecorder.isTypeSupported('audio/webm')) { + if (MediaRecorder.isTypeSupported('audio/webm;codecs=opus')) { + mimeType = 'audio/webm;codecs=opus'; + } else if (MediaRecorder.isTypeSupported('audio/mp4')) { + mimeType = 'audio/mp4'; + } else if (MediaRecorder.isTypeSupported('audio/wav')) { + mimeType = 'audio/wav'; + } else { + mimeType = ''; + } + } + + const mediaRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : {}); mediaRecorderRef.current = mediaRecorder; audioChunksRef.current = []; @@ -297,12 +316,49 @@ export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisp }; mediaRecorder.onstop = async () => { - const audioBlob = new Blob(audioChunksRef.current, { type: 'audio/webm' }); + const audioBlob = new Blob(audioChunksRef.current, { type: mimeType || 'audio/webm' }); + + // Check if the blob is empty + if (audioBlob.size === 0) { + onError?.( + new Error( + 'No audio data was recorded. Please check your microphone permissions and try again.' + ) + ); + return; + } + await transcribeAudio(audioBlob); }; - mediaRecorder.start(1000); // Collect data every second for size monitoring - setIsRecording(true); + // Add error handler for MediaRecorder + mediaRecorder.onerror = (event) => { + console.error('MediaRecorder error:', event); + onError?.(new Error('Recording failed: Unknown error')); + }; + + if (!stream.active) { + throw new Error('Audio stream became inactive before recording could start'); + } + + // Check audio tracks again before starting recording + if (audioTracks.length === 0) { + throw new Error('No audio tracks available in the stream'); + } + + const activeAudioTracks = audioTracks.filter((track) => track.readyState === 'live'); + if (activeAudioTracks.length === 0) { + throw new Error('No live audio tracks available'); + } + + try { + mediaRecorder.start(100); + setIsRecording(true); + } catch (startError) { + console.error('Error calling mediaRecorder.start():', startError); + const errorMessage = startError instanceof Error ? startError.message : String(startError); + throw new Error(`Failed to start recording: ${errorMessage}`); + } } catch (error) { console.error('Error starting recording:', error); stopRecording(); From 16fd8921e419e5b280e55354cae80bf9050b5a05 Mon Sep 17 00:00:00 2001 From: Zane Staggs Date: Tue, 26 Aug 2025 13:39:42 -0700 Subject: [PATCH 2/6] add logging for open ai --- crates/goose-server/src/routes/audio.rs | 45 ++++++++++++++++--- .../src/routes/config_management.rs | 2 +- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/crates/goose-server/src/routes/audio.rs b/crates/goose-server/src/routes/audio.rs index 62ab5970f651..a224e9473ccc 100644 --- a/crates/goose-server/src/routes/audio.rs +++ b/crates/goose-server/src/routes/audio.rs @@ -96,9 +96,10 @@ async fn transcribe_handler( // Get the OpenAI API key from config (after input validation) let config = goose::config::Config::global(); - let api_key: String = config - .get_secret("OPENAI_API_KEY") - .map_err(|_| StatusCode::PRECONDITION_FAILED)?; + let api_key: String = config.get_secret("OPENAI_API_KEY").map_err(|e| { + tracing::error!("Failed to get OpenAI API key: {:?}", e); + StatusCode::PRECONDITION_FAILED + })?; // Get the OpenAI host from config (with default) let openai_host = match config.get("OPENAI_HOST", false) { @@ -109,19 +110,30 @@ async fn transcribe_handler( Err(_) => "https://api.openai.com".to_string(), }; - tracing::debug!("Using OpenAI host: {}", openai_host); + tracing::info!("Using OpenAI host: {}", openai_host); + tracing::info!( + "Audio file size: {} bytes, extension: {}, mime_type: {}", + audio_bytes.len(), + file_extension, + request.mime_type + ); // Create a multipart form with the audio file let part = reqwest::multipart::Part::bytes(audio_bytes) .file_name(format!("audio.{}", file_extension)) .mime_str(&request.mime_type) - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + .map_err(|e| { + tracing::error!("Failed to create multipart part: {:?}", e); + StatusCode::INTERNAL_SERVER_ERROR + })?; let form = reqwest::multipart::Form::new() .part("file", part) .text("model", "whisper-1") .text("response_format", "json"); + tracing::info!("Created multipart form for OpenAI Whisper API"); + // Make request to OpenAI Whisper API let client = Client::builder() .timeout(Duration::from_secs(OPENAI_TIMEOUT_SECONDS)) @@ -131,6 +143,11 @@ async fn transcribe_handler( StatusCode::INTERNAL_SERVER_ERROR })?; + tracing::info!( + "Sending request to OpenAI: {}/v1/audio/transcriptions", + openai_host + ); + let response = client .post(format!("{}/v1/audio/transcriptions", openai_host)) .header("Authorization", format!("Bearer {}", api_key)) @@ -150,9 +167,25 @@ async fn transcribe_handler( } })?; + tracing::info!( + "Received response from OpenAI with status: {}", + response.status() + ); + if !response.status().is_success() { + let status = response.status(); let error_text = response.text().await.unwrap_or_default(); - tracing::error!("OpenAI API error: {}", error_text); + tracing::error!("OpenAI API error (status: {}): {}", status, error_text); + + // Check for specific error codes + if status == 401 { + tracing::error!("OpenAI API key appears to be invalid or unauthorized"); + return Err(StatusCode::UNAUTHORIZED); + } else if status == 429 { + tracing::error!("OpenAI API quota or rate limit exceeded"); + return Err(StatusCode::TOO_MANY_REQUESTS); + } + return Err(StatusCode::BAD_GATEWAY); } diff --git a/crates/goose-server/src/routes/config_management.rs b/crates/goose-server/src/routes/config_management.rs index c243b1511f6c..570ae5bf349b 100644 --- a/crates/goose-server/src/routes/config_management.rs +++ b/crates/goose-server/src/routes/config_management.rs @@ -167,7 +167,7 @@ pub async fn read_config( } let config = Config::global(); - + let response_value = match config.get(&query.key, query.is_secret) { Ok(value) => { if query.is_secret { From d012a7717b586fa6429629ad0c4725f27c9dace7 Mon Sep 17 00:00:00 2001 From: Zane Staggs Date: Tue, 26 Aug 2025 13:50:29 -0700 Subject: [PATCH 3/6] cargo clippy recommendations --- crates/goose-server/src/routes/audio.rs | 88 +++++++++++++++++-------- 1 file changed, 59 insertions(+), 29 deletions(-) diff --git a/crates/goose-server/src/routes/audio.rs b/crates/goose-server/src/routes/audio.rs index a224e9473ccc..e3c70a1e8d39 100644 --- a/crates/goose-server/src/routes/audio.rs +++ b/crates/goose-server/src/routes/audio.rs @@ -42,31 +42,8 @@ struct WhisperResponse { text: String, } -/// Transcribe audio using OpenAI's Whisper API -/// -/// # Request -/// - `audio`: Base64 encoded audio data -/// - `mime_type`: MIME type of the audio (e.g., "audio/webm", "audio/wav") -/// -/// # Response -/// - `text`: Transcribed text from the audio -/// -/// # Errors -/// - 401: Unauthorized (missing or invalid X-Secret-Key header) -/// - 412: Precondition Failed (OpenAI API key not configured) -/// - 400: Bad Request (invalid base64 audio data) -/// - 413: Payload Too Large (audio file exceeds 25MB limit) -/// - 415: Unsupported Media Type (unsupported audio format) -/// - 502: Bad Gateway (OpenAI API error) -/// - 503: Service Unavailable (network error) -async fn transcribe_handler( - State(state): State>, - headers: HeaderMap, - Json(request): Json, -) -> Result, StatusCode> { - verify_secret_key(&headers, &state)?; - - // Validate input first before checking API key configuration +/// Validate audio input and return decoded bytes and file extension +fn validate_audio_input(request: &TranscribeRequest) -> Result<(Vec, &'static str), StatusCode> { // Decode the base64 audio data let audio_bytes = BASE64 .decode(&request.audio) @@ -94,14 +71,18 @@ async fn transcribe_handler( _ => return Err(StatusCode::UNSUPPORTED_MEDIA_TYPE), }; - // Get the OpenAI API key from config (after input validation) + Ok((audio_bytes, file_extension)) +} + +/// Get OpenAI configuration (API key and host) +fn get_openai_config() -> Result<(String, String), StatusCode> { let config = goose::config::Config::global(); + let api_key: String = config.get_secret("OPENAI_API_KEY").map_err(|e| { tracing::error!("Failed to get OpenAI API key: {:?}", e); StatusCode::PRECONDITION_FAILED })?; - // Get the OpenAI host from config (with default) let openai_host = match config.get("OPENAI_HOST", false) { Ok(value) => value .as_str() @@ -110,18 +91,29 @@ async fn transcribe_handler( Err(_) => "https://api.openai.com".to_string(), }; + Ok((api_key, openai_host)) +} + +/// Send transcription request to OpenAI Whisper API +async fn send_openai_request( + audio_bytes: Vec, + file_extension: &str, + mime_type: &str, + api_key: &str, + openai_host: &str, +) -> Result { tracing::info!("Using OpenAI host: {}", openai_host); tracing::info!( "Audio file size: {} bytes, extension: {}, mime_type: {}", audio_bytes.len(), file_extension, - request.mime_type + mime_type ); // Create a multipart form with the audio file let part = reqwest::multipart::Part::bytes(audio_bytes) .file_name(format!("audio.{}", file_extension)) - .mime_str(&request.mime_type) + .mime_str(mime_type) .map_err(|e| { tracing::error!("Failed to create multipart part: {:?}", e); StatusCode::INTERNAL_SERVER_ERROR @@ -194,6 +186,44 @@ async fn transcribe_handler( StatusCode::INTERNAL_SERVER_ERROR })?; + Ok(whisper_response) +} + +/// Transcribe audio using OpenAI's Whisper API +/// +/// # Request +/// - `audio`: Base64 encoded audio data +/// - `mime_type`: MIME type of the audio (e.g., "audio/webm", "audio/wav") +/// +/// # Response +/// - `text`: Transcribed text from the audio +/// +/// # Errors +/// - 401: Unauthorized (missing or invalid X-Secret-Key header) +/// - 412: Precondition Failed (OpenAI API key not configured) +/// - 400: Bad Request (invalid base64 audio data) +/// - 413: Payload Too Large (audio file exceeds 25MB limit) +/// - 415: Unsupported Media Type (unsupported audio format) +/// - 502: Bad Gateway (OpenAI API error) +/// - 503: Service Unavailable (network error) +async fn transcribe_handler( + State(state): State>, + headers: HeaderMap, + Json(request): Json, +) -> Result, StatusCode> { + verify_secret_key(&headers, &state)?; + + let (audio_bytes, file_extension) = validate_audio_input(&request)?; + let (api_key, openai_host) = get_openai_config()?; + + let whisper_response = send_openai_request( + audio_bytes, + file_extension, + &request.mime_type, + &api_key, + &openai_host, + ).await?; + Ok(Json(TranscribeResponse { text: whisper_response.text, })) From 5d50a53578a317810e0aff797bad10c64faca2e5 Mon Sep 17 00:00:00 2001 From: Zane Staggs Date: Tue, 26 Aug 2025 13:51:54 -0700 Subject: [PATCH 4/6] clean up comment --- ui/desktop/src/hooks/useWhisper.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ui/desktop/src/hooks/useWhisper.ts b/ui/desktop/src/hooks/useWhisper.ts index cc13807b7acc..f9192cda4d57 100644 --- a/ui/desktop/src/hooks/useWhisper.ts +++ b/ui/desktop/src/hooks/useWhisper.ts @@ -250,8 +250,7 @@ export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisp throw new Error('No audio tracks available in the microphone stream'); } - // Note: AudioContext creation is disabled to prevent MediaRecorder conflicts - // The AudioContext was interfering with MediaRecorder, causing the stream to become inactive + // AudioContext creation is disabled to prevent MediaRecorder conflicts setAudioContext(null); setAnalyser(null); From b95c46a79f7c730b5846d74b3867936414394ee3 Mon Sep 17 00:00:00 2001 From: Zane Staggs Date: Tue, 26 Aug 2025 14:51:49 -0700 Subject: [PATCH 5/6] cargo fmt --- crates/goose-server/src/routes/audio.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/goose-server/src/routes/audio.rs b/crates/goose-server/src/routes/audio.rs index e3c70a1e8d39..8e7bc6d1cdf7 100644 --- a/crates/goose-server/src/routes/audio.rs +++ b/crates/goose-server/src/routes/audio.rs @@ -43,7 +43,9 @@ struct WhisperResponse { } /// Validate audio input and return decoded bytes and file extension -fn validate_audio_input(request: &TranscribeRequest) -> Result<(Vec, &'static str), StatusCode> { +fn validate_audio_input( + request: &TranscribeRequest, +) -> Result<(Vec, &'static str), StatusCode> { // Decode the base64 audio data let audio_bytes = BASE64 .decode(&request.audio) @@ -77,7 +79,7 @@ fn validate_audio_input(request: &TranscribeRequest) -> Result<(Vec, &'stati /// Get OpenAI configuration (API key and host) fn get_openai_config() -> Result<(String, String), StatusCode> { let config = goose::config::Config::global(); - + let api_key: String = config.get_secret("OPENAI_API_KEY").map_err(|e| { tracing::error!("Failed to get OpenAI API key: {:?}", e); StatusCode::PRECONDITION_FAILED @@ -215,14 +217,15 @@ async fn transcribe_handler( let (audio_bytes, file_extension) = validate_audio_input(&request)?; let (api_key, openai_host) = get_openai_config()?; - + let whisper_response = send_openai_request( audio_bytes, file_extension, &request.mime_type, &api_key, &openai_host, - ).await?; + ) + .await?; Ok(Json(TranscribeResponse { text: whisper_response.text, From e440bd0d40ded945bbeedf144c245855e5c2fa81 Mon Sep 17 00:00:00 2001 From: Zane Staggs Date: Wed, 27 Aug 2025 12:12:40 -0700 Subject: [PATCH 6/6] address feedback and fix edge case where settings was not enabled when key was stored --- crates/goose-server/src/routes/audio.rs | 42 ++++--------------- ui/desktop/src/components/ChatInput.tsx | 4 +- .../settings/dictation/DictationSection.tsx | 20 ++++----- ui/desktop/src/hooks/dictationConstants.ts | 25 +++++++++++ ui/desktop/src/hooks/useDictationSettings.ts | 28 +++++-------- ui/desktop/src/hooks/useWhisper.ts | 25 +++++------ 6 files changed, 64 insertions(+), 80 deletions(-) create mode 100644 ui/desktop/src/hooks/dictationConstants.ts diff --git a/crates/goose-server/src/routes/audio.rs b/crates/goose-server/src/routes/audio.rs index 8e7bc6d1cdf7..8b2046f2af50 100644 --- a/crates/goose-server/src/routes/audio.rs +++ b/crates/goose-server/src/routes/audio.rs @@ -44,12 +44,11 @@ struct WhisperResponse { /// Validate audio input and return decoded bytes and file extension fn validate_audio_input( - request: &TranscribeRequest, + audio: &str, + mime_type: &str, ) -> Result<(Vec, &'static str), StatusCode> { // Decode the base64 audio data - let audio_bytes = BASE64 - .decode(&request.audio) - .map_err(|_| StatusCode::BAD_REQUEST)?; + let audio_bytes = BASE64.decode(audio).map_err(|_| StatusCode::BAD_REQUEST)?; // Check file size if audio_bytes.len() > MAX_AUDIO_SIZE_BYTES { @@ -62,8 +61,9 @@ fn validate_audio_input( } // Determine file extension based on MIME type - let file_extension = match request.mime_type.as_str() { + let file_extension = match mime_type { "audio/webm" => "webm", + "audio/webm;codecs=opus" => "webm", "audio/mp4" => "mp4", "audio/mpeg" => "mp3", "audio/mpga" => "mpga", @@ -215,7 +215,7 @@ async fn transcribe_handler( ) -> Result, StatusCode> { verify_secret_key(&headers, &state)?; - let (audio_bytes, file_extension) = validate_audio_input(&request)?; + let (audio_bytes, file_extension) = validate_audio_input(&request.audio, &request.mime_type)?; let (api_key, openai_host) = get_openai_config()?; let whisper_response = send_openai_request( @@ -243,33 +243,7 @@ async fn transcribe_elevenlabs_handler( ) -> Result, StatusCode> { verify_secret_key(&headers, &state)?; - // Validate input first before checking API key configuration - // Decode the base64 audio data - let audio_bytes = BASE64 - .decode(&request.audio) - .map_err(|_| StatusCode::BAD_REQUEST)?; - - // Check file size - if audio_bytes.len() > MAX_AUDIO_SIZE_BYTES { - tracing::warn!( - "Audio file too large: {} bytes (max: {} bytes)", - audio_bytes.len(), - MAX_AUDIO_SIZE_BYTES - ); - return Err(StatusCode::PAYLOAD_TOO_LARGE); - } - - // Determine file extension and content type based on MIME type - let (file_extension, content_type) = match request.mime_type.as_str() { - "audio/webm" => ("webm", "audio/webm"), - "audio/mp4" => ("mp4", "audio/mp4"), - "audio/mpeg" => ("mp3", "audio/mpeg"), - "audio/mpga" => ("mp3", "audio/mpeg"), - "audio/m4a" => ("m4a", "audio/m4a"), - "audio/wav" => ("wav", "audio/wav"), - "audio/x-wav" => ("wav", "audio/wav"), - _ => return Err(StatusCode::UNSUPPORTED_MEDIA_TYPE), - }; + let (audio_bytes, file_extension) = validate_audio_input(&request.audio, &request.mime_type)?; // Get the ElevenLabs API key from config (after input validation) let config = goose::config::Config::global(); @@ -321,7 +295,7 @@ async fn transcribe_elevenlabs_handler( // Create multipart form for ElevenLabs API let part = reqwest::multipart::Part::bytes(audio_bytes) .file_name(format!("audio.{}", file_extension)) - .mime_str(content_type) + .mime_str(&request.mime_type) .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; let form = reqwest::multipart::Form::new() diff --git a/ui/desktop/src/components/ChatInput.tsx b/ui/desktop/src/components/ChatInput.tsx index f1b69b155bf8..dba2f0080046 100644 --- a/ui/desktop/src/components/ChatInput.tsx +++ b/ui/desktop/src/components/ChatInput.tsx @@ -1259,8 +1259,8 @@ export default function ChatInput({ {/* Inline action buttons on the right */}
- {/* Microphone button - show if dictation is enabled, disable if not configured */} - {(dictationSettings?.enabled || dictationSettings?.provider === null) && ( + {/* Microphone button - show only if dictation is enabled */} + {dictationSettings?.enabled && ( <> {!canUseDictation ? ( diff --git a/ui/desktop/src/components/settings/dictation/DictationSection.tsx b/ui/desktop/src/components/settings/dictation/DictationSection.tsx index 669a30873aa7..f28baa79f3e9 100644 --- a/ui/desktop/src/components/settings/dictation/DictationSection.tsx +++ b/ui/desktop/src/components/settings/dictation/DictationSection.tsx @@ -4,9 +4,11 @@ import { ChevronDown } from 'lucide-react'; import { Input } from '../../ui/input'; import { useConfig } from '../../ConfigContext'; import { DictationProvider, DictationSettings } from '../../../hooks/useDictationSettings'; - -const DICTATION_SETTINGS_KEY = 'dictation_settings'; -const ELEVENLABS_API_KEY = 'ELEVENLABS_API_KEY'; +import { + DICTATION_SETTINGS_KEY, + ELEVENLABS_API_KEY, + getDefaultDictationSettings, +} from '../../../hooks/dictationConstants'; export default function DictationSection() { const [settings, setSettings] = useState({ @@ -27,19 +29,14 @@ export default function DictationSection() { useEffect(() => { const loadSettings = async () => { const savedSettings = localStorage.getItem(DICTATION_SETTINGS_KEY); + let loadedSettings: DictationSettings; if (savedSettings) { const parsed = JSON.parse(savedSettings); loadedSettings = parsed; - console.log('Loaded dictation settings from localStorage:', parsed); } else { - // Default settings - don't force OpenAI as default - loadedSettings = { - enabled: false, - provider: null, - }; - console.log('Using default dictation settings (no saved settings found):', loadedSettings); + loadedSettings = await getDefaultDictationSettings(getProviders); } setSettings(loadedSettings); @@ -50,7 +47,6 @@ export default function DictationSection() { try { // Try reading as secret - will return true if exists const keyExists = await read(ELEVENLABS_API_KEY, true); - console.log('ElevenLabs API key exists in secure storage:', keyExists); if (keyExists === true) { setHasElevenLabsKey(true); // Don't set the actual key since we can't read secrets @@ -63,7 +59,7 @@ export default function DictationSection() { }; loadSettings(); - }, [read]); + }, [read, getProviders]); // Save ElevenLabs key on unmount if it has changed useEffect(() => { diff --git a/ui/desktop/src/hooks/dictationConstants.ts b/ui/desktop/src/hooks/dictationConstants.ts new file mode 100644 index 000000000000..972ba08b6cfa --- /dev/null +++ b/ui/desktop/src/hooks/dictationConstants.ts @@ -0,0 +1,25 @@ +import { DictationSettings, DictationProvider } from './useDictationSettings'; + +export const DICTATION_SETTINGS_KEY = 'dictation_settings'; +export const ELEVENLABS_API_KEY = 'ELEVENLABS_API_KEY'; + +export const getDefaultDictationSettings = async ( + getProviders: (refresh: boolean) => Promise> +): Promise => { + const providers = await getProviders(false); + + // Check if we have an OpenAI API key as primary default + const openAIProvider = providers.find((p) => p.name === 'openai'); + + if (openAIProvider && openAIProvider.is_configured) { + return { + enabled: true, + provider: 'openai' as DictationProvider, + }; + } else { + return { + enabled: false, + provider: null as DictationProvider, + }; + } +}; diff --git a/ui/desktop/src/hooks/useDictationSettings.ts b/ui/desktop/src/hooks/useDictationSettings.ts index 57ab0abb8906..d6fe046b5a4a 100644 --- a/ui/desktop/src/hooks/useDictationSettings.ts +++ b/ui/desktop/src/hooks/useDictationSettings.ts @@ -1,5 +1,10 @@ import { useState, useEffect } from 'react'; import { useConfig } from '../components/ConfigContext'; +import { + DICTATION_SETTINGS_KEY, + ELEVENLABS_API_KEY, + getDefaultDictationSettings, +} from './dictationConstants'; export type DictationProvider = 'openai' | 'elevenlabs' | null; @@ -8,9 +13,6 @@ export interface DictationSettings { provider: DictationProvider; } -const DICTATION_SETTINGS_KEY = 'dictation_settings'; -const ELEVENLABS_API_KEY = 'ELEVENLABS_API_KEY'; - export const useDictationSettings = () => { const [settings, setSettings] = useState(null); const [hasElevenLabsKey, setHasElevenLabsKey] = useState(false); @@ -20,23 +22,13 @@ export const useDictationSettings = () => { const loadSettings = async () => { // Load settings from localStorage const saved = localStorage.getItem(DICTATION_SETTINGS_KEY); + if (saved) { - setSettings(JSON.parse(saved)); + const parsedSettings = JSON.parse(saved); + setSettings(parsedSettings); } else { - const providers = await getProviders(false); - // Check if we have an OpenAI API key as primary default - const openAIProvider = providers.find((p) => p.name === 'openai'); - if (openAIProvider && openAIProvider.is_configured) { - setSettings({ - enabled: true, - provider: 'openai', - }); - } else { - setSettings({ - enabled: false, - provider: null, - }); - } + const defaultSettings = await getDefaultDictationSettings(getProviders); + setSettings(defaultSettings); } // Load ElevenLabs API key from storage (non-secret for frontend access) diff --git a/ui/desktop/src/hooks/useWhisper.ts b/ui/desktop/src/hooks/useWhisper.ts index f9192cda4d57..fe0ddb62d8c2 100644 --- a/ui/desktop/src/hooks/useWhisper.ts +++ b/ui/desktop/src/hooks/useWhisper.ts @@ -159,14 +159,20 @@ export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisp reader.readAsDataURL(audioBlob); }); + const mimeType = audioBlob.type; + if (!mimeType) { + throw new Error('Unable to determine audio format. Please try again.'); + } + let endpoint = ''; let headers: Record = { 'Content-Type': 'application/json', 'X-Secret-Key': await window.electron.getSecretKey(), }; + let body: Record = { audio: base64Audio, - mime_type: audioBlob.type || 'audio/webm', + mime_type: mimeType, }; // Choose endpoint based on provider @@ -254,19 +260,10 @@ export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisp setAudioContext(null); setAnalyser(null); - // Check supported MIME types and create MediaRecorder - let mimeType = 'audio/webm'; - if (!MediaRecorder.isTypeSupported('audio/webm')) { - if (MediaRecorder.isTypeSupported('audio/webm;codecs=opus')) { - mimeType = 'audio/webm;codecs=opus'; - } else if (MediaRecorder.isTypeSupported('audio/mp4')) { - mimeType = 'audio/mp4'; - } else if (MediaRecorder.isTypeSupported('audio/wav')) { - mimeType = 'audio/wav'; - } else { - mimeType = ''; - } - } + // Determine best supported MIME type + const supportedTypes = ['audio/webm;codecs=opus', 'audio/webm', 'audio/mp4', 'audio/wav']; + + const mimeType = supportedTypes.find((type) => MediaRecorder.isTypeSupported(type)) || ''; const mediaRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : {});