diff --git a/assistant/src/daemon/conversation-process.ts b/assistant/src/daemon/conversation-process.ts index 9d6854b2d07..19c6a6fb754 100644 --- a/assistant/src/daemon/conversation-process.ts +++ b/assistant/src/daemon/conversation-process.ts @@ -54,7 +54,8 @@ export function formatCompactResult(result: ContextWindowResult): string { if (!result.compacted) { return `Context compaction skipped — ${result.reason ?? "nothing to compact"}.`; } - const saved = result.previousEstimatedInputTokens - result.estimatedInputTokens; + const saved = + result.previousEstimatedInputTokens - result.estimatedInputTokens; return [ "Context Compacted\n", `Tokens: ${fmt(result.previousEstimatedInputTokens)} → ${fmt(result.estimatedInputTokens)} (${fmt(saved)} saved)`, @@ -295,13 +296,13 @@ export async function drainQueue( if (next.isInteractive === false) { conversation.clearProxyAvailability(); } else { - // Restore proxy availability only for desktop-originating turns (macos/ios) + // Restore proxy availability only for desktop-originating turns (macos) // in case a prior non-interactive drain disabled it. Non-desktop interactive // interfaces (CLI, Vellum) should not re-enable desktop host proxies. const interfaceCtx = queuedInterfaceCtx ?? conversation.getTurnInterfaceContext(); const sourceInterface = interfaceCtx?.userMessageInterface; - if (sourceInterface === "macos" || sourceInterface === "ios") { + if (sourceInterface === "macos") { conversation.restoreProxyAvailability(); conversation.addPreactivatedSkillId("computer-use"); } @@ -310,7 +311,8 @@ export async function drainQueue( // Snapshot persona context at turn start so later tool turns can't pick up // a different actor's context if a concurrent request mutates the live fields. conversation.currentTurnTrustContext = conversation.trustContext; - conversation.currentTurnChannelCapabilities = conversation.channelCapabilities; + conversation.currentTurnChannelCapabilities = + conversation.channelCapabilities; // Resolve slash commands for queued messages const slashResult = await resolveSlash( @@ -671,7 +673,8 @@ export async function processMessage( // Snapshot persona context at turn start so later tool turns can't pick up // a different actor's context if a concurrent request mutates the live fields. conversation.currentTurnTrustContext = conversation.trustContext; - conversation.currentTurnChannelCapabilities = conversation.channelCapabilities; + conversation.currentTurnChannelCapabilities = + conversation.channelCapabilities; conversation.currentActiveSurfaceId = activeSurfaceId; conversation.currentPage = currentPage; const trimmedContent = content.trim(); @@ -880,7 +883,9 @@ export async function processMessage( try { const pmTurnCtx = conversation.getTurnChannelContext(); const pmInterfaceCtx = conversation.getTurnInterfaceContext(); - const pmProvenance = provenanceFromTrustContext(conversation.trustContext); + const pmProvenance = provenanceFromTrustContext( + conversation.trustContext, + ); const pmChannelMeta = { ...pmProvenance, ...(pmTurnCtx @@ -892,7 +897,8 @@ export async function processMessage( ...(pmInterfaceCtx ? { userMessageInterface: pmInterfaceCtx.userMessageInterface, - assistantMessageInterface: pmInterfaceCtx.assistantMessageInterface, + assistantMessageInterface: + pmInterfaceCtx.assistantMessageInterface, } : {}), }; diff --git a/assistant/src/daemon/handlers/conversations.ts b/assistant/src/daemon/handlers/conversations.ts index d732ea4f27b..39e23c637f2 100644 --- a/assistant/src/daemon/handlers/conversations.ts +++ b/assistant/src/daemon/handlers/conversations.ts @@ -103,13 +103,10 @@ export function makeEventSender(params: { guardianPrincipalId: trustContext?.guardianPrincipalId ?? undefined, toolName: event.toolName, commandPreview: - redactSecrets( - summarizeToolInput(event.toolName, inputRecord), - ) || undefined, + redactSecrets(summarizeToolInput(event.toolName, inputRecord)) || + undefined, riskLevel: event.riskLevel, - activityText: activityRaw - ? redactSecrets(activityRaw) - : undefined, + activityText: activityRaw ? redactSecrets(activityRaw) : undefined, executionTarget: event.executionTarget, status: "pending", requestCode: generateCanonicalRequestCode(), @@ -304,7 +301,7 @@ export async function handleConversationCreate( // Only create the host bash proxy for desktop client interfaces that can // execute commands on the user's machine. Set before updateClient so // updateClient's call to hostBashProxy.updateSender targets the new proxy. - if (transportInterface === "macos" || transportInterface === "ios") { + if (transportInterface === "macos") { const proxy = new HostBashProxy(sendEvent, (requestId) => { pendingInteractions.resolve(requestId); }); diff --git a/assistant/src/daemon/message-types/conversations.ts b/assistant/src/daemon/message-types/conversations.ts index 78ac08f3d62..7f6319743ee 100644 --- a/assistant/src/daemon/message-types/conversations.ts +++ b/assistant/src/daemon/message-types/conversations.ts @@ -14,12 +14,10 @@ export interface ConversationListRequest { limit?: number; } -/** Lightweight conversation transport metadata for channel identity and natural-language guidance. */ -export interface ConversationTransportMetadata { +/** Shared fields for all transport metadata variants. */ +interface BaseTransportMetadata { /** Logical channel identifier (e.g. "desktop", "telegram", "mobile"). */ channelId: ChannelId; - /** Interface identifier for this transport (e.g. "macos", "ios", "cli"). */ - interfaceId?: InterfaceId; /** Optional natural-language hints for channel-specific UX behavior. */ hints?: string[]; /** Optional concise UX brief for this channel. */ @@ -28,6 +26,27 @@ export interface ConversationTransportMetadata { chatType?: string; } +/** Transport metadata for macOS desktop clients, including host environment fields. */ +export interface MacosTransportMetadata extends BaseTransportMetadata { + /** Interface identifier for macOS transport. */ + interfaceId: "macos"; + /** Home directory of the host macOS user. */ + hostHomeDir?: string; + /** Username of the host macOS user. */ + hostUsername?: string; +} + +/** Transport metadata for non-macOS transports. */ +export interface NonMacosTransportMetadata extends BaseTransportMetadata { + /** Interface identifier for this transport (e.g. "ios", "cli"). */ + interfaceId?: Exclude; +} + +/** Lightweight conversation transport metadata for channel identity and natural-language guidance. */ +export type ConversationTransportMetadata = + | MacosTransportMetadata + | NonMacosTransportMetadata; + export interface ConversationCreateRequest { type: "conversation_create"; title?: string; diff --git a/assistant/src/daemon/server.ts b/assistant/src/daemon/server.ts index 2a6a81bd2dd..38a63f14e04 100644 --- a/assistant/src/daemon/server.ts +++ b/assistant/src/daemon/server.ts @@ -373,7 +373,28 @@ export class DaemonServer { { channelId: transport.channelId }, "Transport metadata received", ); - conversation.setTransportHints(transport.hints); + + // Build enriched hints: interface ID first, then host environment (macOS + // only), then any client-provided hints. + const enrichedHints: string[] = []; + + const interfaceLabel = parseInterfaceId(transport.interfaceId) ?? "vellum"; + enrichedHints.push(`User is messaging from interface: ${interfaceLabel}`); + + if (transport.interfaceId === "macos") { + if (transport.hostHomeDir) { + enrichedHints.push(`Host home directory: ${transport.hostHomeDir}`); + } + if (transport.hostUsername) { + enrichedHints.push(`Host username: ${transport.hostUsername}`); + } + } + + if (transport.hints) { + enrichedHints.push(...transport.hints); + } + + conversation.setTransportHints(enrichedHints); } constructor() { @@ -1070,7 +1091,7 @@ export class DaemonServer { // Guard: don't replace an active proxy during concurrent turn races — // another request may have started processing between the isProcessing() // check above and the await on ensureActorScopedHistory(). - if (resolvedInterface === "macos" || resolvedInterface === "ios") { + if (resolvedInterface === "macos") { if (!conversation.isProcessing() || !conversation.hostBashProxy) { conversation.setHostBashProxy( new HostBashProxy(conversation.getCurrentSender(), (requestId) => { diff --git a/assistant/src/runtime/routes/conversation-routes.ts b/assistant/src/runtime/routes/conversation-routes.ts index 23ee47a142a..5e9e81a9c7d 100644 --- a/assistant/src/runtime/routes/conversation-routes.ts +++ b/assistant/src/runtime/routes/conversation-routes.ts @@ -643,7 +643,9 @@ function isToolResultType(type: string): boolean { function isSystemNoticeText(block: Record): boolean { if (block.type !== "text") return false; const text = typeof block.text === "string" ? block.text : ""; - return text.startsWith("") && text.endsWith(""); + return ( + text.startsWith("") && text.endsWith("") + ); } /** @@ -1115,7 +1117,7 @@ export async function handleSendMessage( // channels, headless) fall back to local execution. // Set the proxy BEFORE updateClient so updateClient's call to // hostBashProxy.updateSender targets the correct (new) proxy. - if (sourceInterface === "macos" || sourceInterface === "ios") { + if (sourceInterface === "macos") { // Reuse the existing proxy if the conversation is actively processing a // host bash request to avoid orphaning in-flight requests. if (!conversation.isProcessing() || !conversation.hostBashProxy) { @@ -1152,9 +1154,7 @@ export async function handleSendMessage( // When proxies are preserved during an active turn (non-desktop request while // processing), skip updating proxy senders to avoid degrading them. const preservingProxies = - conversation.isProcessing() && - sourceInterface !== "macos" && - sourceInterface !== "ios"; + conversation.isProcessing() && sourceInterface !== "macos"; conversation.updateClient(onEvent, !isInteractive, { skipProxySenderUpdate: preservingProxies, }); diff --git a/cli/src/commands/teleport.ts b/cli/src/commands/teleport.ts index bb29370dbdf..ce2c438fbe3 100644 --- a/cli/src/commands/teleport.ts +++ b/cli/src/commands/teleport.ts @@ -422,7 +422,7 @@ async function importViaHttp( "Content-Type": "application/octet-stream", }, body: new Blob([bundleData]), - signal: AbortSignal.timeout(120_000), + signal: AbortSignal.timeout(300_000), }); // Retry once with a fresh token on 401 @@ -446,13 +446,13 @@ async function importViaHttp( "Content-Type": "application/octet-stream", }, body: new Blob([bundleData]), - signal: AbortSignal.timeout(120_000), + signal: AbortSignal.timeout(300_000), }); } } } catch (err) { if (err instanceof Error && err.name === "TimeoutError") { - console.error("Error: Import request timed out after 2 minutes."); + console.error("Error: Import request timed out after 5 minutes."); process.exit(1); } const msg = err instanceof Error ? err.message : String(err); @@ -706,7 +706,7 @@ async function importToAssistant( : await platformImportBundle(bundleData, token, entry.runtimeUrl); } catch (err) { if (err instanceof Error && err.name === "TimeoutError") { - console.error("Error: Import request timed out after 2 minutes."); + console.error("Error: Import request timed out after 5 minutes."); process.exit(1); } throw err; diff --git a/cli/src/lib/platform-client.ts b/cli/src/lib/platform-client.ts index 17cf65687ca..c22453eaa4f 100644 --- a/cli/src/lib/platform-client.ts +++ b/cli/src/lib/platform-client.ts @@ -421,7 +421,7 @@ export async function platformImportBundle( "Content-Type": "application/octet-stream", }, body: new Blob([bundleData]), - signal: AbortSignal.timeout(120_000), + signal: AbortSignal.timeout(300_000), }); const body = (await response.json().catch(() => ({}))) as Record< @@ -529,7 +529,7 @@ export async function platformImportBundleFromGcs( method: "POST", headers: await authHeaders(token, platformUrl), body: JSON.stringify({ bundle_key: bundleKey }), - signal: AbortSignal.timeout(120_000), + signal: AbortSignal.timeout(300_000), }, ); diff --git a/clients/ios/Views/InputBarView.swift b/clients/ios/Views/InputBarView.swift index 33fde65bc44..122cdc95b3e 100644 --- a/clients/ios/Views/InputBarView.swift +++ b/clients/ios/Views/InputBarView.swift @@ -187,7 +187,7 @@ struct InputBarView: View { VButton( label: "Stop generation", iconOnly: VIcon.square.rawValue, - style: .contrast, + style: .primary, action: onStop ) } else { diff --git a/clients/macos/vellum-assistant/Features/Chat/ComposerView.swift b/clients/macos/vellum-assistant/Features/Chat/ComposerView.swift index 050d0af43f8..5a11e059d6a 100644 --- a/clients/macos/vellum-assistant/Features/Chat/ComposerView.swift +++ b/clients/macos/vellum-assistant/Features/Chat/ComposerView.swift @@ -427,7 +427,7 @@ struct ComposerView: View { VButton( label: "Stop generation", iconOnly: VIcon.square.rawValue, - style: .contrast, + style: .primary, iconSize: composerActionButtonSize, action: onStop ) @@ -612,7 +612,7 @@ VStreamingWaveform( VButton( label: manager.state == .listening ? "Mute" : "Unmute", iconOnly: manager.state == .listening ? VIcon.mic.rawValue : VIcon.micOff.rawValue, - style: .contrast, + style: .primary, iconSize: composerActionButtonSize, action: { manager.toggleListening() } ) diff --git a/clients/macos/vellum-assistant/Features/Chat/ThinkingBlockView.swift b/clients/macos/vellum-assistant/Features/Chat/ThinkingBlockView.swift index 84e0c626ce9..71f2fca2df7 100644 --- a/clients/macos/vellum-assistant/Features/Chat/ThinkingBlockView.swift +++ b/clients/macos/vellum-assistant/Features/Chat/ThinkingBlockView.swift @@ -11,14 +11,7 @@ struct ThinkingBlockView: View { let content: String let isStreaming: Bool - @State private var isExpanded: Bool - @State private var userHasToggled: Bool = false - - init(content: String, isStreaming: Bool) { - self.content = content - self.isStreaming = isStreaming - _isExpanded = State(initialValue: isStreaming) - } + @State private var isExpanded: Bool = false var body: some View { VStack(alignment: .leading, spacing: 0) { @@ -52,20 +45,6 @@ struct ThinkingBlockView: View { #endif } } - .onChange(of: isStreaming) { _, newValue in - Task { @MainActor in - if newValue { - userHasToggled = false - withAnimation(VAnimation.fast) { - isExpanded = true - } - } else if !userHasToggled { - withAnimation(VAnimation.fast) { - isExpanded = false - } - } - } - } .background(VColor.surfaceOverlay) .clipShape(RoundedRectangle(cornerRadius: VRadius.md)) } @@ -74,7 +53,6 @@ struct ThinkingBlockView: View { private var headerRow: some View { Button(action: { - userHasToggled = true withAnimation(VAnimation.fast) { isExpanded.toggle() } diff --git a/clients/macos/vellum-assistant/Features/Settings/SettingsBillingReferralCard.swift b/clients/macos/vellum-assistant/Features/Settings/SettingsBillingReferralCard.swift index 3b3c00453a9..08fb20d7dcb 100644 --- a/clients/macos/vellum-assistant/Features/Settings/SettingsBillingReferralCard.swift +++ b/clients/macos/vellum-assistant/Features/Settings/SettingsBillingReferralCard.swift @@ -46,7 +46,7 @@ struct SettingsBillingReferralCard: View { // MARK: - Has Code State private func hasCodeState(_ code: ReferralCodeResponse) -> some View { - SettingsCard(title: "Referrals", subtitle: "Share your link and earn credits") { + SettingsCard(title: "Referrals", subtitle: "Share your referral link to earn up to 100 free credits") { VStack(alignment: .leading, spacing: VSpacing.lg) { // Referral URL row HStack(spacing: VSpacing.sm) { @@ -104,10 +104,6 @@ struct SettingsBillingReferralCard: View { } } - // Earning cap note - Text("Earn up to \(code.earning_cap.replacingOccurrences(of: ".00", with: "")) referral credits") - .font(VFont.bodySmallDefault) - .foregroundStyle(VColor.contentTertiary) } } } diff --git a/clients/macos/vellum-assistant/Features/Settings/SettingsBillingTab.swift b/clients/macos/vellum-assistant/Features/Settings/SettingsBillingTab.swift index b4a8c65ad92..c31c0835d8e 100644 --- a/clients/macos/vellum-assistant/Features/Settings/SettingsBillingTab.swift +++ b/clients/macos/vellum-assistant/Features/Settings/SettingsBillingTab.swift @@ -174,11 +174,25 @@ struct SettingsBillingTab: View { // MARK: - Add Credits Card private var addFundsCard: some View { - SettingsCard(title: "Add Credits") { + SettingsCard(title: "Add Credits", subtitle: addCreditsSubtitle) { topUpContent } } + private var addCreditsSubtitle: String? { + guard let summary else { return nil } + let maxFormatted: String = { + let value = Int(Double(summary.maximum_balance) ?? 0) + if value > 0 { + let formatter = NumberFormatter() + formatter.numberStyle = .decimal + return formatter.string(from: NSNumber(value: value)) ?? summary.maximum_balance + } + return summary.maximum_balance + }() + return "Credits cost $1 each, with a maximum balance of \(maxFormatted). Unused credits expire 12 months after purchase." + } + @ViewBuilder private var topUpContent: some View { VStack(alignment: .leading, spacing: VSpacing.md) { @@ -196,20 +210,6 @@ struct SettingsBillingTab: View { } ) .frame(maxWidth: 200) - if let summary { - let maxFormatted: String = { - let value = Int(Double(summary.maximum_balance) ?? 0) - if value > 0 { - let formatter = NumberFormatter() - formatter.numberStyle = .decimal - return formatter.string(from: NSNumber(value: value)) ?? summary.maximum_balance - } - return summary.maximum_balance - }() - Text("1 credit = $1 USD. \(maxFormatted) max credit balance. Credits expire 12 months after purchase.") - .font(VFont.bodySmallDefault) - .foregroundStyle(VColor.contentTertiary) - } } VButton( diff --git a/clients/macos/vellum-assistant/Features/Voice/AudioEngineController.swift b/clients/macos/vellum-assistant/Features/Voice/AudioEngineController.swift index 8ebdeaa3d55..cfe6ed34f3b 100644 --- a/clients/macos/vellum-assistant/Features/Voice/AudioEngineController.swift +++ b/clients/macos/vellum-assistant/Features/Voice/AudioEngineController.swift @@ -10,13 +10,11 @@ private let log = Logger(subsystem: Bundle.appBundleIdentifier, category: "Audio /// audio-subsystem queue. When that queue is contended (hardware state changes, /// Bluetooth negotiation, coreaudiod latency), the wait can exceed 2 seconds. /// -/// Fire-and-forget operations (`installTap`, `removeTap`, `stop`, `reset`) -/// use `queue.async` so the caller never blocks. Methods that return a value -/// (`inputNodeFormat`, `prepareAndStart`) or that require ordering guarantees -/// (`tearDown`, `stopAndRemoveTap` — callers call `endAudio()` immediately -/// after) use `queue.sync`. Callers should ensure `prewarm()` has run first -/// so `inputNode` is already initialized and sync calls complete in -/// sub-milliseconds. +/// Fire-and-forget operations (`stop`, `reset`) use `queue.async` so the caller +/// never blocks. Methods that require ordering guarantees (`tearDown`, +/// `stopAndRemoveTap`, `installTapAndStart`) use `queue.sync`. Callers should +/// ensure `prewarm()` has run first so `inputNode` is already initialized and +/// sync calls complete in sub-milliseconds. /// /// See: https://developer.apple.com/documentation/avfaudio/avaudionode/1387122-installtap final class AudioEngineController: @unchecked Sendable { @@ -28,18 +26,6 @@ final class AudioEngineController: @unchecked Sendable { self.queue = DispatchQueue(label: label, qos: .userInitiated) } - // MARK: - Input Node Format - - /// Returns the input node's output format for bus 0. - /// Returns `nil` if the format has zero channels or zero sample rate. - func inputNodeFormat() -> AVAudioFormat? { - queue.sync { [self] in - let format = audioEngine.inputNode.outputFormat(forBus: 0) - guard format.channelCount > 0, format.sampleRate > 0 else { return nil } - return format - } - } - // MARK: - Pre-warm /// Touch `inputNode` to force lazy initialization of the audio subsystem. @@ -51,46 +37,8 @@ final class AudioEngineController: @unchecked Sendable { } } - // MARK: - Tap Management - - /// Remove any existing tap on bus 0, then install a new one. - /// Uses `async` — the next `queue.sync` call (e.g. `prepareAndStart`) will - /// wait for this to complete thanks to serial queue ordering. - func installTap( - bufferSize: AVAudioFrameCount, - format: AVAudioFormat?, - block: @escaping AVAudioNodeTapBlock - ) { - queue.async { [weak self] in - guard let self else { return } - let inputNode = self.audioEngine.inputNode - inputNode.removeTap(onBus: 0) - inputNode.installTap(onBus: 0, bufferSize: bufferSize, format: format, block: block) - } - } - - /// Remove the tap on bus 0 from the input node. - func removeTap() { - queue.async { [weak self] in - guard let self else { return } - self.audioEngine.inputNode.removeTap(onBus: 0) - } - } - // MARK: - Engine Lifecycle - func prepare() { - queue.async { [weak self] in - self?.audioEngine.prepare() - } - } - - func start() throws { - try queue.sync { [self] in - try audioEngine.start() - } - } - func stop() { queue.async { [weak self] in guard let self else { return } @@ -122,18 +70,25 @@ final class AudioEngineController: @unchecked Sendable { // MARK: - Combined Operations - /// Atomically reads the input format, installs a tap, and starts the engine - /// in a single synchronous dispatch to the audio queue. + /// Atomically validates audio input, installs a tap with `nil` format, and + /// starts the engine in a single synchronous dispatch to the audio queue. + /// + /// Passing `nil` for `installTap`'s format parameter lets AVAudioEngine use + /// its own internal hardware format, which is always self-consistent. This + /// prevents `NSInternalInconsistencyException` crashes caused by + /// `format.sampleRate != hwFormat.sampleRate` — the cached format from + /// `outputFormat(forBus:)` can diverge from the engine's internal hardware + /// format after audio route changes (Bluetooth, USB mic, AirPods mode + /// switch), even within a single synchronous block. /// - /// Eliminates the TOCTOU race where the format read by `inputNodeFormat()` - /// becomes stale before the separate `installTap()` async block executes — - /// which crashes with `NSInternalInconsistencyException` when the hardware - /// format changes between calls (common on first use after permission grant). + /// The format validation (channels > 0, sampleRate > 0) is kept as a + /// pre-check to detect "no audio input available" — but the validated format + /// is **not** forwarded to `installTap`. /// - /// Returns `true` on success, or `false` if the format is invalid or the - /// engine fails to start. + /// Returns `true` on success, or `false` if no audio input is available or + /// the engine fails to start. /// - /// See: https://developer.apple.com/documentation/avfaudio/avaudionode/1387122-installtap + /// See: https://developer.apple.com/documentation/avfaudio/avaudionode/installtap(onbus:buffersize:format:block:) func installTapAndStart( bufferSize: AVAudioFrameCount, block: @escaping AVAudioNodeTapBlock @@ -147,7 +102,7 @@ final class AudioEngineController: @unchecked Sendable { } inputNode.removeTap(onBus: 0) - inputNode.installTap(onBus: 0, bufferSize: bufferSize, format: format, block: block) + inputNode.installTap(onBus: 0, bufferSize: bufferSize, format: nil, block: block) audioEngine.prepare() do { @@ -161,23 +116,6 @@ final class AudioEngineController: @unchecked Sendable { } } - /// Prepare and start the engine. Returns `true` on success. - /// On failure, removes tap and returns `false`. - @discardableResult - func prepareAndStart() -> Bool { - queue.sync { [self] in - audioEngine.prepare() - do { - try audioEngine.start() - return true - } catch { - log.error("Failed to start audio engine: \(error.localizedDescription)") - audioEngine.inputNode.removeTap(onBus: 0) - return false - } - } - } - /// Stop the engine and remove the input tap (if running). /// Uses `sync` because callers depend on the tap being removed before /// they call `recognitionRequest?.endAudio()` — appending audio after diff --git a/clients/macos/vellum-assistant/Features/Voice/OpenAIVoiceService.swift b/clients/macos/vellum-assistant/Features/Voice/OpenAIVoiceService.swift index 2628a2a197a..3bb94bf899c 100644 --- a/clients/macos/vellum-assistant/Features/Voice/OpenAIVoiceService.swift +++ b/clients/macos/vellum-assistant/Features/Voice/OpenAIVoiceService.swift @@ -152,11 +152,6 @@ final class OpenAIVoiceService: VoiceServiceProtocol { latestTranscription = "" livePartialText = "" - guard let format = engineController.inputNodeFormat() else { - log.error("No audio input channels") - return false - } - // Reuse existing SFSpeechRecognizer across turns to avoid OS resource // release delays that make isAvailable return false on the second turn. if speechRecognizer == nil { @@ -225,8 +220,10 @@ final class OpenAIVoiceService: VoiceServiceProtocol { } } - // Install audio tap — feeds buffers to SFSpeechRecognizer + computes RMS for amplitude - engineController.installTap(bufferSize: 4096, format: format) { [weak self] buffer, _ in + // Atomically validate format, install tap, and start engine. + // Passes nil for format so AVAudioEngine uses its internal hardware + // format, preventing sampleRate mismatch crashes. + guard engineController.installTapAndStart(bufferSize: 4096, block: { [weak self] buffer, _ in guard let floatData = buffer.floatChannelData else { return } let frameCount = Int(buffer.frameLength) guard frameCount > 0 else { return } @@ -269,24 +266,17 @@ final class OpenAIVoiceService: VoiceServiceProtocol { self.onSilenceDetected?() } } - } - - // prepare() is async, start() is sync — serial queue guarantees - // prepare() completes before start() executes. - engineController.prepare() - do { - try engineController.start() - isRecording = true - lastSpeechTime = Date() - recordingStartTime = Date() - log.info("Recording started (SFSpeechRecognizer, onDevice: \(recognizer.supportsOnDeviceRecognition, privacy: .public))") - return true - } catch { - log.error("Failed to start audio engine: \(error.localizedDescription)") - engineController.removeTap() + }) else { + log.error("Failed to start audio engine for recording") tearDownRecognition() return false } + + isRecording = true + lastSpeechTime = Date() + recordingStartTime = Date() + log.info("Recording started (SFSpeechRecognizer, onDevice: \(recognizer.supportsOnDeviceRecognition, privacy: .public))") + return true } /// Stop recording and return the transcription from SFSpeechRecognizer. @@ -453,12 +443,10 @@ final class OpenAIVoiceService: VoiceServiceProtocol { guard !bargeInMonitorActive else { return } bargeInMonitorActive = true - guard let format = engineController.inputNodeFormat() else { - bargeInMonitorActive = false - return - } - - engineController.installTap(bufferSize: 4096, format: format) { [weak self] buffer, _ in + // Atomically validate format, install tap, and start engine. + // Passes nil for format so AVAudioEngine uses its internal hardware + // format, preventing sampleRate mismatch crashes. + if engineController.installTapAndStart(bufferSize: 4096, block: { [weak self] buffer, _ in guard let floatData = buffer.floatChannelData else { return } let frameCount = Int(buffer.frameLength) guard frameCount > 0 else { return } @@ -479,9 +467,7 @@ final class OpenAIVoiceService: VoiceServiceProtocol { self.onBargeInDetected?() } } - } - - if engineController.prepareAndStart() { + }) { log.info("Barge-in monitor started") } else { log.error("Failed to start barge-in monitor") diff --git a/clients/shared/DesignSystem/Gallery/Sections/ButtonsGallerySection.swift b/clients/shared/DesignSystem/Gallery/Sections/ButtonsGallerySection.swift index 0b286805b26..bdc916e0773 100644 --- a/clients/shared/DesignSystem/Gallery/Sections/ButtonsGallerySection.swift +++ b/clients/shared/DesignSystem/Gallery/Sections/ButtonsGallerySection.swift @@ -30,7 +30,6 @@ struct ButtonsGallerySection: View { (label: "Danger", tag: VButton.Style.danger), (label: "Danger Outline", tag: VButton.Style.dangerOutline), (label: "Ghost", tag: VButton.Style.ghost), - (label: "Contrast", tag: VButton.Style.contrast), ], selection: $selectedStyle ) @@ -76,7 +75,7 @@ struct ButtonsGallerySection: View { VCard { HStack(spacing: VSpacing.xl) { - ForEach([VButton.Style.primary, .outlined, .danger, .dangerOutline, .ghost, .contrast], id: \.self) { style in + ForEach([VButton.Style.primary, .outlined, .danger, .dangerOutline, .ghost], id: \.self) { style in VStack(spacing: VSpacing.md) { VButton(label: styleName(style), style: style) {} VButton(label: "Disabled", style: style, isDisabled: true) {} @@ -155,10 +154,6 @@ struct ButtonsGallerySection: View { Text("Danger").font(VFont.labelDefault).foregroundStyle(VColor.contentTertiary) VButton(label: "Delete", iconOnly: VIcon.trash.rawValue, style: .danger) {} } - VStack(alignment: .leading, spacing: VSpacing.md) { - Text("Contrast").font(VFont.labelDefault).foregroundStyle(VColor.contentTertiary) - VButton(label: "Stop", iconOnly: VIcon.square.rawValue, style: .contrast) {} - } } } diff --git a/clients/shared/Network/Generated/GeneratedAPITypes.swift b/clients/shared/Network/Generated/GeneratedAPITypes.swift index 114dd2f440c..4557ca5af61 100644 --- a/clients/shared/Network/Generated/GeneratedAPITypes.swift +++ b/clients/shared/Network/Generated/GeneratedAPITypes.swift @@ -3588,12 +3588,18 @@ public struct ConversationTransportMetadata: Codable, Sendable { public let hints: [String]? /// Optional concise UX brief for this channel. public let uxBrief: String? + /// Home directory of the host macOS user. Only populated when interfaceId == "macos". + public let hostHomeDir: String? + /// Username of the host macOS user. Only populated when interfaceId == "macos". + public let hostUsername: String? - public init(channelId: String, interfaceId: String? = nil, hints: [String]? = nil, uxBrief: String? = nil) { + public init(channelId: String, interfaceId: String? = nil, hints: [String]? = nil, uxBrief: String? = nil, hostHomeDir: String? = nil, hostUsername: String? = nil) { self.channelId = channelId self.interfaceId = interfaceId self.hints = hints self.uxBrief = uxBrief + self.hostHomeDir = hostHomeDir + self.hostUsername = hostUsername } } diff --git a/clients/shared/Network/MessageTypes.swift b/clients/shared/Network/MessageTypes.swift index fa79323bff8..30121432191 100644 --- a/clients/shared/Network/MessageTypes.swift +++ b/clients/shared/Network/MessageTypes.swift @@ -168,7 +168,9 @@ private func buildConversationTransportMetadata( channelId: String?, interfaceId: String?, hints: [String]?, - uxBrief: String? + uxBrief: String?, + hostHomeDir: String? = nil, + hostUsername: String? = nil ) -> ConversationTransportMetadata? { guard let channelId, !channelId.isEmpty else { return nil } @@ -182,6 +184,12 @@ private func buildConversationTransportMetadata( if let uxBrief { payload["uxBrief"] = uxBrief } + if let hostHomeDir { + payload["hostHomeDir"] = hostHomeDir + } + if let hostUsername { + payload["hostUsername"] = hostUsername + } guard JSONSerialization.isValidJSONObject(payload) else { return nil } do { @@ -207,6 +215,24 @@ extension ConversationCreateRequest { self.init(type: "conversation_create", title: title, systemPromptOverride: systemPromptOverride, maxResponseTokens: maxResponseTokens, correlationId: correlationId, transport: transport, conversationType: conversationType, preactivatedSkillIds: preactivatedSkillIds, initialMessage: initialMessage) } + /// The host home directory, populated automatically on macOS. + private static var defaultHostHomeDir: String? { + #if os(macOS) + return NSHomeDirectory() + #else + return nil + #endif + } + + /// The host username, populated automatically on macOS. + private static var defaultHostUsername: String? { + #if os(macOS) + return NSUserName() + #else + return nil + #endif + } + public init( title: String?, systemPromptOverride: String? = nil, @@ -215,8 +241,15 @@ extension ConversationCreateRequest { transportChannelId: String?, transportInterfaceId: String? = nil, transportHints: [String]? = nil, - transportUxBrief: String? = nil + transportUxBrief: String? = nil, + transportHostHomeDir: String? = nil, + transportHostUsername: String? = nil ) { + let effectiveInterface = transportInterfaceId ?? Self.defaultTransportInterface + // Auto-populate host environment on macOS when using the default transport interface. + let effectiveHostHomeDir = transportHostHomeDir ?? (effectiveInterface == "macos" ? Self.defaultHostHomeDir : nil) + let effectiveHostUsername = transportHostUsername ?? (effectiveInterface == "macos" ? Self.defaultHostUsername : nil) + self.init( type: "conversation_create", title: title, @@ -225,9 +258,11 @@ extension ConversationCreateRequest { correlationId: correlationId, transport: buildConversationTransportMetadata( channelId: transportChannelId, - interfaceId: transportInterfaceId ?? Self.defaultTransportInterface, + interfaceId: effectiveInterface, hints: transportHints, - uxBrief: transportUxBrief + uxBrief: transportUxBrief, + hostHomeDir: effectiveHostHomeDir, + hostUsername: effectiveHostUsername ), conversationType: nil, preactivatedSkillIds: nil,