diff --git a/.github/workflows/build-desktop.yml b/.github/workflows/build-desktop.yml index c811334bcaa..1ba50b02dc1 100644 --- a/.github/workflows/build-desktop.yml +++ b/.github/workflows/build-desktop.yml @@ -81,6 +81,11 @@ jobs: working-directory: apps/desktop run: bun run clean:dev + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Compile app with electron-vite working-directory: apps/desktop env: diff --git a/apps/api/package.json b/apps/api/package.json index 9ac03eee334..593f85990a9 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -38,6 +38,7 @@ "lodash.chunk": "^4.2.0", "mcp-handler": "^1.0.7", "next": "^16.0.10", + "openai": "^6.17.0", "react": "19.1.0", "react-dom": "19.1.0", "require-in-the-middle": "8.0.1", diff --git a/apps/api/src/app/api/voice/route.ts b/apps/api/src/app/api/voice/route.ts new file mode 100644 index 00000000000..1e63fc3c88e --- /dev/null +++ b/apps/api/src/app/api/voice/route.ts @@ -0,0 +1,82 @@ +import { auth } from "@superset/auth/server"; +import { runVoicePipeline } from "./voice-service"; + +export async function POST(request: Request) { + const session = await auth.api.getSession({ headers: request.headers }); + if (!session?.user) { + return Response.json({ error: "Unauthorized" }, { status: 401 }); + } + + const organizationId = session.session.activeOrganizationId; + if (!organizationId) { + return Response.json({ error: "No active organization" }, { status: 400 }); + } + + let formData: FormData; + try { + formData = await request.formData(); + } catch { + return Response.json( + { error: "Expected multipart form data with audio file" }, + { status: 400 }, + ); + } + + const audioFile = formData.get("audio"); + if (!audioFile || !(audioFile instanceof File)) { + return Response.json( + { error: "Missing 'audio' file in form data" }, + { status: 400 }, + ); + } + + const MAX_AUDIO_SIZE = 5 * 1024 * 1024; // 5 MB + if (audioFile.size > MAX_AUDIO_SIZE) { + return Response.json( + { error: "Audio file too large (max 5 MB)" }, + { status: 413 }, + ); + } + + const audioBuffer = new Uint8Array(await audioFile.arrayBuffer()); + + const encoder = new TextEncoder(); + + const stream = new ReadableStream({ + async start(controller) { + const sse = { + write(event: string, data: unknown) { + const payload = `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`; + controller.enqueue(encoder.encode(payload)); + }, + }; + + try { + await runVoicePipeline({ + audioBuffer, + ctx: { userId: session.user.id, organizationId }, + sse, + signal: request.signal, + }); + } catch (error) { + if (!request.signal.aborted) { + console.error("[voice/route] Pipeline error:", error); + sse.write("error", { + message: + error instanceof Error ? error.message : "Voice pipeline failed", + }); + } + } finally { + controller.close(); + } + }, + }); + + return new Response(stream, { + headers: { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }, + }); +} diff --git a/apps/api/src/app/api/voice/voice-service.ts b/apps/api/src/app/api/voice/voice-service.ts new file mode 100644 index 00000000000..9dc823965c6 --- /dev/null +++ b/apps/api/src/app/api/voice/voice-service.ts @@ -0,0 +1,198 @@ +import Anthropic from "@anthropic-ai/sdk"; +import type { McpContext } from "@superset/mcp/auth"; +import { createInMemoryMcpClient } from "@superset/mcp/in-memory"; +import { OpenAI } from "openai"; +import { env } from "@/env"; + +const SYSTEM_PROMPT = `You are a helpful voice assistant for Superset, a project management tool. You have access to tools for creating and managing tasks, workspaces, and other organizational resources. Keep responses concise and conversational — the user is speaking to you, so respond in 1-3 sentences unless the question requires more detail. When you use tools, briefly confirm what you did.`; + +// Desktop-only tools that don't make sense in voice context +const DENIED_TOOLS = new Set([ + "navigate_to_workspace", + "switch_workspace", + "get_app_context", +]); + +interface SSEWriter { + write(event: string, data: unknown): void; +} + +async function transcribeAudio({ + audioBuffer, + signal, +}: { + audioBuffer: Uint8Array; + signal?: AbortSignal; +}): Promise { + const openai = new OpenAI({ apiKey: env.OPENAI_API_KEY }); + + const blob = new Blob([audioBuffer as BlobPart], { type: "audio/wav" }); + const file = new File([blob], "audio.wav", { type: "audio/wav" }); + + const result = await openai.audio.transcriptions.create( + { + model: "whisper-1", + file, + }, + { signal }, + ); + + // Strip wake word from transcription + let text = result.text.trim(); + text = text.replace(/^hey\s*jarvis[,.\s!?]*/i, "").trim(); + return text; +} + +/** + * Runs the full voice pipeline: transcription → Claude with MCP tools → streaming SSE. + */ +export async function runVoicePipeline({ + audioBuffer, + ctx, + sse, + signal, +}: { + audioBuffer: Uint8Array; + ctx: McpContext; + sse: SSEWriter; + signal?: AbortSignal; +}): Promise { + // 1. Transcribe + const transcription = await transcribeAudio({ audioBuffer, signal }); + sse.write("transcription", { text: transcription }); + + if (!transcription) { + sse.write("done", { fullResponse: "" }); + return; + } + + // 2. Create in-memory MCP client for tool access + const { client: mcpClient, cleanup } = await createInMemoryMcpClient({ + userId: ctx.userId, + organizationId: ctx.organizationId, + }); + + try { + const { tools: mcpTools } = await mcpClient.listTools(); + + const anthropicTools: Anthropic.Tool[] = mcpTools + .filter((t) => !DENIED_TOOLS.has(t.name)) + .map((t) => ({ + name: t.name, + description: t.description ?? "", + input_schema: t.inputSchema as Anthropic.Tool.InputSchema, + })); + + // 3. Stream Claude response with tool use loop + const anthropic = new Anthropic({ apiKey: env.ANTHROPIC_API_KEY }); + + const messages: Anthropic.MessageParam[] = [ + { role: "user", content: transcription }, + ]; + + let fullResponse = ""; + + try { + const MAX_TOOL_ROUNDS = 5; + for (let round = 0; round < MAX_TOOL_ROUNDS; round++) { + if (signal?.aborted) return; + + const stream = anthropic.messages.stream( + { + model: "claude-sonnet-4-20250514", + max_tokens: 1024, + system: SYSTEM_PROMPT, + messages, + tools: anthropicTools.length > 0 ? anthropicTools : undefined, + }, + { signal }, + ); + + for await (const event of stream) { + if (event.type === "content_block_delta") { + if (event.delta.type === "text_delta") { + fullResponse += event.delta.text; + sse.write("text_delta", { delta: event.delta.text }); + } + } + } + + const finalMessage = await stream.finalMessage(); + const contentBlocks = finalMessage.content; + + const toolUseBlocks = contentBlocks.filter( + (block): block is Anthropic.ToolUseBlock => block.type === "tool_use", + ); + + if (toolUseBlocks.length === 0) { + break; + } + + const toolResults: Anthropic.ToolResultBlockParam[] = []; + + for (const toolBlock of toolUseBlocks) { + if (signal?.aborted) return; + + sse.write("tool_use", { + toolName: toolBlock.name, + toolInput: toolBlock.input, + }); + + try { + const result = await mcpClient.callTool({ + name: toolBlock.name, + arguments: toolBlock.input as Record, + }); + + const resultText = JSON.stringify(result.content); + + sse.write("tool_result", { + toolName: toolBlock.name, + result: resultText, + }); + + toolResults.push({ + type: "tool_result", + tool_use_id: toolBlock.id, + content: resultText, + }); + } catch (error) { + if (signal?.aborted) return; + console.error( + `[voice/tool] Error executing ${toolBlock.name}:`, + error, + ); + const errorText = JSON.stringify({ + error: + error instanceof Error + ? error.message + : "Tool execution failed", + }); + + sse.write("tool_result", { + toolName: toolBlock.name, + result: errorText, + }); + + toolResults.push({ + type: "tool_result", + tool_use_id: toolBlock.id, + content: errorText, + is_error: true, + }); + } + } + + messages.push({ role: "assistant", content: contentBlocks }); + messages.push({ role: "user", content: toolResults }); + } + } catch (error) { + if (signal?.aborted) return; + throw error; + } + + sse.write("done", { fullResponse }); + } finally { + await cleanup().catch(() => {}); + } +} diff --git a/apps/api/src/env.ts b/apps/api/src/env.ts index a5e2bbc5b0f..17a0e09cdb1 100644 --- a/apps/api/src/env.ts +++ b/apps/api/src/env.ts @@ -40,6 +40,7 @@ export const env = createEnv({ STRIPE_PRO_MONTHLY_PRICE_ID: z.string(), STRIPE_PRO_YEARLY_PRICE_ID: z.string(), SENTRY_AUTH_TOKEN: z.string().optional(), + OPENAI_API_KEY: z.string().min(1), }, client: { NEXT_PUBLIC_API_URL: z.string().url(), diff --git a/apps/desktop/electron-builder.canary.ts b/apps/desktop/electron-builder.canary.ts index fe20ad0d2c1..edfded17095 100644 --- a/apps/desktop/electron-builder.canary.ts +++ b/apps/desktop/electron-builder.canary.ts @@ -31,6 +31,7 @@ const config: Configuration = { icon: join(pkg.resources, "build/icons/icon-canary.icns"), artifactName: `Superset-Canary-\${version}-\${arch}.\${ext}`, extendInfo: { + ...baseConfig.mac?.extendInfo, CFBundleName: productName, CFBundleDisplayName: productName, }, diff --git a/apps/desktop/electron-builder.ts b/apps/desktop/electron-builder.ts index 9fb84f56634..55c80133476 100644 --- a/apps/desktop/electron-builder.ts +++ b/apps/desktop/electron-builder.ts @@ -56,6 +56,12 @@ const config: Configuration = { to: "resources/migrations", filter: ["**/*"], }, + // Voice sidecar binary (built by PyInstaller via scripts/build-voice-sidecar.sh) + { + from: "dist/voice-sidecar/voice-sidecar", + to: "voice-sidecar", + filter: ["**/*"], + }, ], files: [ @@ -117,6 +123,8 @@ const config: Configuration = { hardenedRuntime: true, gatekeeperAssess: false, notarize: true, + entitlements: join(pkg.resources, "build/entitlements.mac.plist"), + entitlementsInherit: join(pkg.resources, "build/entitlements.mac.plist"), extendInfo: { CFBundleName: productName, CFBundleDisplayName: productName, @@ -125,6 +133,9 @@ const config: Configuration = { "Superset needs access to your local network to discover and connect to development servers running on your network.", // Bonjour service types to browse for (triggers the permission prompt) NSBonjourServices: ["_http._tcp", "_https._tcp"], + // Required for microphone access (voice commands) + NSMicrophoneUsageDescription: + "Superset uses the microphone for voice commands to interact with your development environment.", }, }, diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 288a116f80f..973cacd5bce 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -22,7 +22,7 @@ "copy:native-modules": "bun run scripts/copy-native-modules.ts", "prebuild": "bun run clean:dev && bun run compile:app && bun run copy:native-modules", "build": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false electron-builder --publish never", - "prepackage": "bun run copy:native-modules", + "prepackage": "bun run copy:native-modules && bash scripts/build-voice-sidecar.sh", "package": "electron-builder --config electron-builder.ts", "install:deps": "electron-builder install-app-deps", "release": "electron-builder --publish always", diff --git a/apps/desktop/scripts/build-voice-sidecar.sh b/apps/desktop/scripts/build-voice-sidecar.sh new file mode 100755 index 00000000000..648e6559a52 --- /dev/null +++ b/apps/desktop/scripts/build-voice-sidecar.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Builds the voice sidecar Python script into a standalone binary using PyInstaller. +# The output binary is placed in dist/voice-sidecar/ and gets bundled into +# the Electron app's extraResources by electron-builder. +# +# This script is self-contained: it creates the venv and installs all +# dependencies automatically if they are missing. The only prerequisite +# is that `python3` is available on the PATH. +# +# Usage: +# ./scripts/build-voice-sidecar.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +DESKTOP_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +PYTHON_DIR="$DESKTOP_DIR/src/main/lib/voice/python" +VENV_DIR="$PYTHON_DIR/.venv" +OUTPUT_DIR="$DESKTOP_DIR/dist/voice-sidecar" + +PYTHON="$VENV_DIR/bin/python3" +PIP="$VENV_DIR/bin/pip" + +# Create venv if it doesn't exist +if [ ! -d "$VENV_DIR" ]; then + echo "[voice-sidecar] Creating Python venv..." + python3 -m venv "$VENV_DIR" +fi + +# Install runtime dependencies +if ! "$PYTHON" -c "import openwakeword" 2>/dev/null; then + echo "[voice-sidecar] Installing dependencies..." + "$PIP" install --quiet openwakeword sounddevice numpy +fi + +# openwakeword >=0.6.0 no longer ships pre-trained models in the pip package. +# Download the required models into the package's resources directory. +OWW_PKG_DIR=$("$PYTHON" -c "import openwakeword, os; print(os.path.dirname(openwakeword.__file__))") +OWW_MODELS_DIR="$OWW_PKG_DIR/resources/models" +mkdir -p "$OWW_MODELS_DIR" + +OWW_BASE_URL="https://github.com/dscripka/openWakeWord/releases/download/v0.5.1" +for model in hey_jarvis_v0.1.onnx melspectrogram.onnx embedding_model.onnx; do + if [ ! -f "$OWW_MODELS_DIR/$model" ]; then + echo "[voice-sidecar] Downloading model: $model" + curl -sL "$OWW_BASE_URL/$model" -o "$OWW_MODELS_DIR/$model" + fi +done + +# Install PyInstaller +if ! "$PYTHON" -c "import PyInstaller" 2>/dev/null; then + echo "[voice-sidecar] Installing PyInstaller..." + "$PIP" install --quiet pyinstaller +fi + +echo "[voice-sidecar] Building binary..." + +"$PYTHON" -m PyInstaller \ + --name voice-sidecar \ + --onedir \ + --noconfirm \ + --clean \ + --distpath "$OUTPUT_DIR" \ + --workpath "$DESKTOP_DIR/dist/voice-sidecar-build" \ + --specpath "$DESKTOP_DIR/dist" \ + --collect-data openwakeword \ + "$PYTHON_DIR/main.py" + +BUNDLE_DIR="$OUTPUT_DIR/voice-sidecar" +INTERNAL_DIR="$BUNDLE_DIR/_internal" + +echo "[voice-sidecar] Built at: $BUNDLE_DIR/" +ls -la "$BUNDLE_DIR/" + +# PyInstaller's --collect-data may miss openwakeword's data files. +# Copy them manually as a guaranteed fallback. +if [ ! -f "$INTERNAL_DIR/openwakeword/resources/models/hey_jarvis_v0.1.onnx" ]; then + echo "[voice-sidecar] Model not found in bundle, copying openwakeword data manually..." + OWW_PKG_DIR=$("$PYTHON" -c "import openwakeword, os; print(os.path.dirname(openwakeword.__file__))") + echo "[voice-sidecar] Copying openwakeword package from: $OWW_PKG_DIR" + rm -rf "$INTERNAL_DIR/openwakeword" + cp -R "$OWW_PKG_DIR" "$INTERNAL_DIR/openwakeword" +fi + +# Final verification +if [ ! -f "$INTERNAL_DIR/openwakeword/resources/models/hey_jarvis_v0.1.onnx" ]; then + echo "[voice-sidecar] ERROR: hey_jarvis model not found in bundle!" + exit 1 +fi +echo "[voice-sidecar] Verified hey_jarvis model is bundled." diff --git a/apps/desktop/src/lib/trpc/routers/index.ts b/apps/desktop/src/lib/trpc/routers/index.ts index 2545a0f404b..1c216be4355 100644 --- a/apps/desktop/src/lib/trpc/routers/index.ts +++ b/apps/desktop/src/lib/trpc/routers/index.ts @@ -16,6 +16,7 @@ import { createRingtoneRouter } from "./ringtone"; import { createSettingsRouter } from "./settings"; import { createTerminalRouter } from "./terminal"; import { createUiStateRouter } from "./ui-state"; +import { createVoiceRouter } from "./voice"; import { createWindowRouter } from "./window"; import { createWorkspacesRouter } from "./workspaces"; @@ -39,6 +40,7 @@ export const createAppRouter = (getWindow: () => BrowserWindow | null) => { config: createConfigRouter(), uiState: createUiStateRouter(), ringtone: createRingtoneRouter(), + voice: createVoiceRouter(), }); }; diff --git a/apps/desktop/src/lib/trpc/routers/settings/index.ts b/apps/desktop/src/lib/trpc/routers/settings/index.ts index ea2166ba14e..3cdf84d394e 100644 --- a/apps/desktop/src/lib/trpc/routers/settings/index.ts +++ b/apps/desktop/src/lib/trpc/routers/settings/index.ts @@ -405,5 +405,25 @@ export const createSettingsRouter = () => { return { success: true }; }), + + getVoiceCommandsEnabled: publicProcedure.query(() => { + const row = getSettings(); + return row.voiceCommandsEnabled ?? false; + }), + + setVoiceCommandsEnabled: publicProcedure + .input(z.object({ enabled: z.boolean() })) + .mutation(({ input }) => { + localDb + .insert(settings) + .values({ id: 1, voiceCommandsEnabled: input.enabled }) + .onConflictDoUpdate({ + target: settings.id, + set: { voiceCommandsEnabled: input.enabled }, + }) + .run(); + + return { success: true }; + }), }); }; diff --git a/apps/desktop/src/lib/trpc/routers/voice/index.ts b/apps/desktop/src/lib/trpc/routers/voice/index.ts new file mode 100644 index 00000000000..d999d96004f --- /dev/null +++ b/apps/desktop/src/lib/trpc/routers/voice/index.ts @@ -0,0 +1,83 @@ +import { observable } from "@trpc/server/observable"; +import { systemPreferences } from "electron"; +import { + getCurrentVoiceState, + startVoiceProcess, + stopVoiceProcess, + type VoiceSidecarEvent, + voiceProcessEmitter, +} from "main/lib/voice/voice-process"; +import { publicProcedure, router } from "../.."; + +type MicPermissionStatus = + | "not-determined" + | "granted" + | "denied" + | "restricted"; + +function getMicStatus(): MicPermissionStatus { + if (process.platform !== "darwin") { + return "granted"; + } + return systemPreferences.getMediaAccessStatus( + "microphone", + ) as MicPermissionStatus; +} + +export const createVoiceRouter = () => { + let subscriberCount = 0; + + return router({ + subscribe: publicProcedure.subscription(() => { + return observable((emit) => { + subscriberCount++; + + // Auto-start the voice process when first subscriber connects + if (subscriberCount === 1) { + startVoiceProcess(); + } + + emit.next(getCurrentVoiceState()); + + const onVoiceEvent = (event: VoiceSidecarEvent) => { + emit.next(event); + }; + + voiceProcessEmitter.on("voice-event", onVoiceEvent); + + return () => { + voiceProcessEmitter.off("voice-event", onVoiceEvent); + subscriberCount--; + + // Auto-stop when last subscriber disconnects + if (subscriberCount === 0) { + stopVoiceProcess(); + } + }; + }); + }), + + getMicPermission: publicProcedure.query((): MicPermissionStatus => { + return getMicStatus(); + }), + + requestMicPermission: publicProcedure.mutation( + async (): Promise<{ granted: boolean; status: MicPermissionStatus }> => { + const current = getMicStatus(); + + if (current === "granted") { + return { granted: true, status: "granted" }; + } + + if (current !== "not-determined") { + return { granted: false, status: current }; + } + + const granted = await systemPreferences.askForMediaAccess("microphone"); + const status = getMicStatus(); + + return { granted, status }; + }, + ), + }); +}; diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index 2a46c20f4e5..842202fdf33 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -17,6 +17,7 @@ import { shutdownOrphanedDaemon, } from "./lib/terminal"; import { disposeTray, initTray } from "./lib/tray"; +import { stopVoiceProcess } from "./lib/voice/voice-process"; import { MainWindow } from "./windows/main"; // Initialize local SQLite database (runs migrations + legacy data migration on import) @@ -158,8 +159,8 @@ app.on("before-quit", async (event) => { } // Quit confirmed or no confirmation needed - exit immediately - // Let OS clean up child processes, tray, etc. isQuitting = true; + stopVoiceProcess(); disposeTray(); app.exit(0); }); diff --git a/apps/desktop/src/main/lib/voice/python/.gitignore b/apps/desktop/src/main/lib/voice/python/.gitignore new file mode 100644 index 00000000000..77ac75498fb --- /dev/null +++ b/apps/desktop/src/main/lib/voice/python/.gitignore @@ -0,0 +1,3 @@ +.venv/ +__pycache__/ +*.pyc diff --git a/apps/desktop/src/main/lib/voice/python/audio.py b/apps/desktop/src/main/lib/voice/python/audio.py new file mode 100644 index 00000000000..5d8d96409fb --- /dev/null +++ b/apps/desktop/src/main/lib/voice/python/audio.py @@ -0,0 +1,60 @@ +import queue +from typing import Optional + +import numpy as np +import sounddevice as sd + +from config import Config + + +class AudioStream: + """Context manager for capturing audio from the microphone.""" + + def __init__(self, config: Config) -> None: + self._config = config + self._queue: queue.Queue[np.ndarray] = queue.Queue() + self._stream: Optional[sd.InputStream] = None + + def _callback( + self, + indata: np.ndarray, + frames: int, + time_info: object, + status: sd.CallbackFlags, + ) -> None: + if status: + _emit_error(f"audio callback: {status}") + self._queue.put(indata.copy()) + + def read_chunk(self, timeout: float = 2.0) -> Optional[np.ndarray]: + """Read the next audio chunk from the queue. Returns None on timeout.""" + try: + return self._queue.get(timeout=timeout) + except queue.Empty: + return None + + def __enter__(self) -> "AudioStream": + self._stream = sd.InputStream( + samplerate=self._config.sample_rate, + channels=self._config.channels, + dtype=self._config.dtype, + blocksize=self._config.chunk_size, + callback=self._callback, + ) + self._stream.start() + return self + + def __exit__(self, *exc: object) -> None: + if self._stream is not None: + self._stream.stop() + self._stream.close() + self._stream = None + + +def _emit_error(message: str) -> None: + """Helper to emit error via stdout JSON (imported lazily to avoid circular imports).""" + import json + import sys + + sys.stdout.write(json.dumps({"event": "error", "message": message}) + "\n") + sys.stdout.flush() diff --git a/apps/desktop/src/main/lib/voice/python/config.py b/apps/desktop/src/main/lib/voice/python/config.py new file mode 100644 index 00000000000..6192fce0cfc --- /dev/null +++ b/apps/desktop/src/main/lib/voice/python/config.py @@ -0,0 +1,26 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Config: + # Audio + sample_rate: int = 16000 + channels: int = 1 + dtype: str = "int16" + chunk_duration_ms: int = 80 + chunk_size: int = 0 # computed in __post_init__ + + # Wake word + wake_word_model: str = "hey_jarvis" + wake_word_threshold: float = 0.5 + + # Speech capture + pre_buffer_chunks: int = 63 # ~5s of audio to carry over into speech capture + min_capture_s: float = 1.5 # don't end capture until this much live time has passed + silence_threshold_rms: float = 200.0 + silence_duration_s: float = 1.5 + max_speech_duration_s: float = 30.0 + + def __post_init__(self) -> None: + computed = int(self.sample_rate * self.chunk_duration_ms / 1000) + object.__setattr__(self, "chunk_size", computed) diff --git a/apps/desktop/src/main/lib/voice/python/main.py b/apps/desktop/src/main/lib/voice/python/main.py new file mode 100644 index 00000000000..e02f7842bae --- /dev/null +++ b/apps/desktop/src/main/lib/voice/python/main.py @@ -0,0 +1,146 @@ +""" +Voice sidecar process — wake word detection + audio capture. + +Communicates with the parent Node.js process via stdio JSON lines. + +Stdout events: + {"event": "ready"} + {"event": "recording"} + {"event": "audio_captured", "audio_b64": "", "duration_s": 3.2} + {"event": "error", "message": "..."} + {"event": "idle"} + +Stdin commands: + {"cmd": "start"} + {"cmd": "stop"} +""" + +import base64 +import collections +import io +import json +import sys +import threading +import time +import wave +from typing import Any + +import numpy as np + +from audio import AudioStream +from config import Config +from speech_capture import CaptureStatus, SpeechCapture +from wake_word import WakeWordDetector + + +def emit(event: str, **kwargs: Any) -> None: + """Write a JSON event to stdout.""" + msg = {"event": event, **kwargs} + sys.stdout.write(json.dumps(msg) + "\n") + sys.stdout.flush() + + +def to_wav_b64(audio: np.ndarray, config: Config) -> str: + """Convert int16 numpy array to base64-encoded WAV.""" + buf = io.BytesIO() + with wave.open(buf, "wb") as wf: + wf.setnchannels(config.channels) + wf.setsampwidth(2) # 16-bit = 2 bytes + wf.setframerate(config.sample_rate) + wf.writeframes(audio.tobytes()) + return base64.b64encode(buf.getvalue()).decode("ascii") + + +def stdin_reader(stop_event: threading.Event) -> None: + """Read stdin commands in a background thread.""" + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + cmd = json.loads(line) + if cmd.get("cmd") == "stop": + stop_event.set() + except json.JSONDecodeError as e: + print(f"[stdin] Invalid JSON: {e}", file=sys.stderr) + + +def main() -> None: + config = Config() + + # Load wake word model + detector = WakeWordDetector(config) + try: + detector.load() + except Exception as e: + emit("error", message=f"Failed to load wake word model: {e}") + sys.exit(1) + + capturer = SpeechCapture(config) + + # Listen for stop commands from parent process + stop_event = threading.Event() + stdin_thread = threading.Thread(target=stdin_reader, args=(stop_event,), daemon=True) + stdin_thread.start() + + emit("ready") + + pre_buffer: collections.deque[Any] = collections.deque(maxlen=config.pre_buffer_chunks) + + try: + with AudioStream(config) as stream: + emit("idle") + + while not stop_event.is_set(): + chunk = stream.read_chunk() + if chunk is None: + continue + + pre_buffer.append(chunk.copy()) + + # Wake word detection + result = detector.process_chunk(chunk) + if not result.detected: + continue + + # Speech capture + emit("recording") + capturer.start() + for buffered_chunk in pre_buffer: + capturer.add_prebuffer(buffered_chunk) + pre_buffer.clear() + + while not stop_event.is_set(): + audio_chunk = stream.read_chunk() + if audio_chunk is None: + continue + status = capturer.add_chunk(audio_chunk) + if status != CaptureStatus.CAPTURING: + break + + speech_audio = capturer.get_audio() + + if speech_audio.size == 0: + emit("idle") + detector.reset() + continue + + # Convert to WAV and emit + audio_b64 = to_wav_b64(speech_audio, config) + emit( + "audio_captured", + audio_b64=audio_b64, + duration_s=round(capturer.duration_s, 2), + ) + + # Reset for next cycle + detector.reset() + time.sleep(0.5) + emit("idle") + + except Exception as e: + emit("error", message=str(e)) + + +if __name__ == "__main__": + main() diff --git a/apps/desktop/src/main/lib/voice/python/pyproject.toml b/apps/desktop/src/main/lib/voice/python/pyproject.toml new file mode 100644 index 00000000000..9766a581781 --- /dev/null +++ b/apps/desktop/src/main/lib/voice/python/pyproject.toml @@ -0,0 +1,11 @@ +[project] +name = "superset-voice-sidecar" +version = "0.1.0" +description = "Wake word detection + audio capture sidecar for Superset desktop" +requires-python = ">=3.10" + +dependencies = [ + "openwakeword>=0.6.0", + "sounddevice>=0.5.0", + "numpy>=1.24.0", +] diff --git a/apps/desktop/src/main/lib/voice/python/speech_capture.py b/apps/desktop/src/main/lib/voice/python/speech_capture.py new file mode 100644 index 00000000000..a5fbc04fb35 --- /dev/null +++ b/apps/desktop/src/main/lib/voice/python/speech_capture.py @@ -0,0 +1,76 @@ +import enum +import time + +import numpy as np + +from config import Config + + +class CaptureStatus(enum.Enum): + CAPTURING = "capturing" + SPEECH_ENDED = "speech_ended" + MAX_DURATION = "max_duration" + + +class SpeechCapture: + """Accumulates audio after wake word trigger, detects silence to end capture.""" + + def __init__(self, config: Config) -> None: + self._config = config + self._buffers: list[np.ndarray] = [] + self._start_time: float = 0.0 + self._last_speech_time: float = 0.0 + self._active = False + + def start(self) -> None: + """Begin a new speech capture session.""" + self._buffers = [] + self._start_time = time.perf_counter() + self._last_speech_time = self._start_time + self._active = True + + def add_prebuffer(self, chunk: np.ndarray) -> None: + """Add a pre-buffered chunk (audio only, no silence detection).""" + if not self._active: + raise RuntimeError("Capture not started. Call start() first.") + self._buffers.append(chunk.copy()) + + def add_chunk(self, chunk: np.ndarray) -> CaptureStatus: + """Add a live chunk and return the current capture status.""" + if not self._active: + raise RuntimeError("Capture not started. Call start() first.") + + self._buffers.append(chunk.copy()) + now = time.perf_counter() + elapsed = now - self._start_time + + if elapsed >= self._config.max_speech_duration_s: + self._active = False + return CaptureStatus.MAX_DURATION + + rms = np.sqrt(np.mean(chunk.astype(np.float64) ** 2)) + if rms > self._config.silence_threshold_rms: + self._last_speech_time = now + + if elapsed < self._config.min_capture_s: + return CaptureStatus.CAPTURING + + silence_duration = now - self._last_speech_time + if silence_duration >= self._config.silence_duration_s: + self._active = False + return CaptureStatus.SPEECH_ENDED + + return CaptureStatus.CAPTURING + + def get_audio(self) -> np.ndarray: + """Return all captured audio as a single array.""" + if not self._buffers: + return np.array([], dtype=np.int16) + return np.concatenate(self._buffers).flatten() + + @property + def duration_s(self) -> float: + if not self._buffers: + return 0.0 + total_samples = sum(b.size for b in self._buffers) + return total_samples / self._config.sample_rate diff --git a/apps/desktop/src/main/lib/voice/python/wake_word.py b/apps/desktop/src/main/lib/voice/python/wake_word.py new file mode 100644 index 00000000000..c66013ef64c --- /dev/null +++ b/apps/desktop/src/main/lib/voice/python/wake_word.py @@ -0,0 +1,46 @@ +from dataclasses import dataclass +from typing import Optional + +import numpy as np +from openwakeword.model import Model + +from config import Config + + +@dataclass +class WakeWordResult: + detected: bool + confidence: float + + +class WakeWordDetector: + """Wraps openwakeword for wake word detection.""" + + def __init__(self, config: Config) -> None: + self._config = config + self._model: Optional[Model] = None + + def load(self) -> None: + self._model = Model( + wakeword_models=[self._config.wake_word_model], + inference_framework="onnx", + ) + + def process_chunk(self, chunk: np.ndarray) -> WakeWordResult: + """Process an audio chunk and return detection result.""" + if self._model is None: + raise RuntimeError("Model not loaded. Call load() first.") + + audio = chunk.flatten().astype(np.int16) + self._model.predict(audio) + + scores = self._model.prediction_buffer.get(self._config.wake_word_model, []) + confidence = scores[-1] if scores else 0.0 + detected = confidence >= self._config.wake_word_threshold + + return WakeWordResult(detected=detected, confidence=confidence) + + def reset(self) -> None: + """Reset the model's prediction buffer for a new detection cycle.""" + if self._model is not None: + self._model.reset() diff --git a/apps/desktop/src/main/lib/voice/voice-process-paths.ts b/apps/desktop/src/main/lib/voice/voice-process-paths.ts new file mode 100644 index 00000000000..a65c37c2b9d --- /dev/null +++ b/apps/desktop/src/main/lib/voice/voice-process-paths.ts @@ -0,0 +1,86 @@ +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { app } from "electron"; +import { env } from "main/env.main"; + +export interface VoiceSpawnConfig { + /** The command to execute (python path or PyInstaller binary). */ + command: string; + /** Arguments to pass (e.g. ["main.py"] for dev, [] for binary). */ + args: string[]; + /** Working directory for the spawned process. */ + cwd: string; +} + +/** + * Returns the spawn configuration for the voice sidecar process. + * + * Production (packaged): PyInstaller binary at process.resourcesPath/voice-sidecar/voice-sidecar + * Development: .venv/bin/python3 main.py in the source directory + * Preview: Similar to dev, resolves relative to dist/ + */ +export function getVoiceSpawnConfig(): VoiceSpawnConfig { + if (app.isPackaged) { + return getPackagedConfig(); + } + + const isDev = env.NODE_ENV === "development"; + if (isDev) { + return getDevConfig(); + } + + return getPreviewConfig(); +} + +function getPackagedConfig(): VoiceSpawnConfig { + const binaryDir = join(process.resourcesPath, "voice-sidecar"); + const binaryName = + process.platform === "win32" ? "voice-sidecar.exe" : "voice-sidecar"; + const binaryPath = join(binaryDir, binaryName); + + if (existsSync(binaryPath)) { + return { command: binaryPath, args: [], cwd: binaryDir }; + } + + // Fallback: try system python3 with unpacked script + console.warn( + "[voice-paths] PyInstaller binary not found, falling back to system python3", + ); + const scriptDir = join( + process.resourcesPath, + "app.asar.unpacked/src/main/lib/voice/python", + ); + return { + command: "python3", + args: [join(scriptDir, "main.py")], + cwd: scriptDir, + }; +} + +function getDevConfig(): VoiceSpawnConfig { + const scriptDir = join(app.getAppPath(), "src/main/lib/voice/python"); + const venvPython = join(scriptDir, ".venv/bin/python3"); + + if (existsSync(venvPython)) { + return { command: venvPython, args: ["main.py"], cwd: scriptDir }; + } + + console.warn( + "[voice-paths] Dev venv not found, falling back to system python3", + ); + return { command: "python3", args: ["main.py"], cwd: scriptDir }; +} + +function getPreviewConfig(): VoiceSpawnConfig { + const previewDir = join(__dirname, "../lib/voice/python"); + const srcDir = join(app.getAppPath(), "src/main/lib/voice/python"); + + const scriptDir = existsSync(previewDir) ? previewDir : srcDir; + const venvPython = join(srcDir, ".venv/bin/python3"); + + if (existsSync(venvPython)) { + return { command: venvPython, args: ["main.py"], cwd: scriptDir }; + } + + return { command: "python3", args: ["main.py"], cwd: scriptDir }; +} diff --git a/apps/desktop/src/main/lib/voice/voice-process.ts b/apps/desktop/src/main/lib/voice/voice-process.ts new file mode 100644 index 00000000000..41e779e5c04 --- /dev/null +++ b/apps/desktop/src/main/lib/voice/voice-process.ts @@ -0,0 +1,158 @@ +import type { ChildProcess } from "node:child_process"; +import { spawn } from "node:child_process"; +import { EventEmitter } from "node:events"; +import { createInterface } from "node:readline"; +import { getVoiceSpawnConfig } from "./voice-process-paths"; + +export type VoiceSidecarEvent = + | { type: "ready" } + | { type: "recording" } + | { type: "audio_captured"; audioB64: string; durationS: number } + | { type: "error"; message: string } + | { type: "idle" }; + +interface PythonVoiceEvent { + event: "ready" | "recording" | "audio_captured" | "error" | "idle"; + audio_b64?: string; + duration_s?: number; + message?: string; +} + +export const voiceProcessEmitter = new EventEmitter(); + +let childProcess: ChildProcess | null = null; +let lastEvent: VoiceSidecarEvent = { type: "idle" }; + +function parsePythonEvent(raw: PythonVoiceEvent): VoiceSidecarEvent | null { + switch (raw.event) { + case "ready": + return { type: "ready" }; + case "recording": + return { type: "recording" }; + case "audio_captured": + if (raw.audio_b64 && raw.duration_s !== undefined) { + return { + type: "audio_captured", + audioB64: raw.audio_b64, + durationS: raw.duration_s, + }; + } + return null; + case "error": + return { type: "error", message: raw.message ?? "Unknown error" }; + case "idle": + return { type: "idle" }; + default: + return null; + } +} + +export function startVoiceProcess(): void { + if (childProcess) { + console.warn("[voice-process] Already running"); + return; + } + + const config = getVoiceSpawnConfig(); + + console.log( + `[voice-process] Starting: ${config.command} ${config.args.join(" ")}`, + ); + + const proc = spawn(config.command, config.args, { + cwd: config.cwd, + stdio: ["pipe", "pipe", "pipe"], + env: { ...process.env }, + }); + + childProcess = proc; + + // Parse stdout JSON lines + if (proc.stdout) { + const rl = createInterface({ input: proc.stdout }); + rl.on("line", (line) => { + try { + const raw = JSON.parse(line) as PythonVoiceEvent; + const event = parsePythonEvent(raw); + if (event) { + lastEvent = event; + voiceProcessEmitter.emit("voice-event", event); + } + } catch { + console.warn("[voice-process] Non-JSON stdout:", line); + } + }); + } + + // Log stderr + if (proc.stderr) { + const rl = createInterface({ input: proc.stderr }); + rl.on("line", (line) => { + console.error("[voice-process/stderr]", line); + }); + } + + // Only run cleanup if this process is still the active one. + // A newer process may have been spawned after stopVoiceProcess() + // cleared the reference. + proc.on("error", (err) => { + console.error("[voice-process] Spawn error:", err.message); + voiceProcessEmitter.emit("voice-event", { + type: "error", + message: `Process error: ${err.message}`, + } satisfies VoiceSidecarEvent); + if (childProcess === proc) { + cleanup(); + } + }); + + proc.on("exit", (code, signal) => { + console.log(`[voice-process] Exited with code=${code} signal=${signal}`); + if (childProcess === proc) { + cleanup(); + } + }); +} + +export function stopVoiceProcess(): void { + if (!childProcess) { + return; + } + + // Capture reference and clear immediately so startVoiceProcess() + // can proceed if called while this process is still shutting down. + const proc = childProcess; + cleanup(); + + // Send stop command via stdin + if (proc.stdin && !proc.stdin.destroyed) { + try { + proc.stdin.write(`${JSON.stringify({ cmd: "stop" })}\n`); + } catch { + // stdin may be closed already + } + } + + // Give it a moment to exit gracefully, then force kill + const timeout = setTimeout(() => { + if (!proc.killed) { + proc.kill("SIGKILL"); + } + }, 3000); + + proc.once("exit", () => { + clearTimeout(timeout); + }); + + proc.kill("SIGTERM"); +} + +export function getCurrentVoiceState(): VoiceSidecarEvent { + return lastEvent; +} + +function cleanup(): void { + childProcess = null; + lastEvent = { type: "idle" }; + voiceProcessEmitter.emit("voice-event", lastEvent); +} diff --git a/apps/desktop/src/renderer/components/Voice/VoiceListener.tsx b/apps/desktop/src/renderer/components/Voice/VoiceListener.tsx new file mode 100644 index 00000000000..4105461f3af --- /dev/null +++ b/apps/desktop/src/renderer/components/Voice/VoiceListener.tsx @@ -0,0 +1,116 @@ +import { toast } from "@superset/ui/sonner"; +import { useEffect, useRef } from "react"; +import { electronTrpc } from "renderer/lib/electron-trpc"; +import { RecordingIndicator } from "./components/RecordingIndicator"; +import { ResponsePanel } from "./components/ResponsePanel"; + +/** + * Single component that queries the voiceCommandsEnabled setting and + * passes it as the `enabled` flag to `useSubscription`. This avoids + * conditional rendering / mount-unmount cycles — the subscription hook + * is always called (React rules-of-hooks) but only connects when the + * setting is true. + */ +export function VoiceListener() { + const { data: voiceEnabled } = + electronTrpc.settings.getVoiceCommandsEnabled.useQuery(); + + const { data: micPermission } = electronTrpc.voice.getMicPermission.useQuery( + undefined, + { + refetchOnWindowFocus: true, + }, + ); + + const canListen = !!voiceEnabled && micPermission === "granted"; + + const indicatorToastRef = useRef(null); + const responseToastRef = useRef(null); + + // Dismiss any lingering toasts when voice is disabled or permission revoked + useEffect(() => { + if (!canListen) { + dismissAll(indicatorToastRef, responseToastRef); + } + }, [canListen]); + + electronTrpc.voice.subscribe.useSubscription(undefined, { + enabled: canListen, + onData: (event) => { + switch (event.type) { + case "recording": { + dismissAll(indicatorToastRef, responseToastRef); + + const toastId = toast.custom( + (id) => , + { + duration: Number.POSITIVE_INFINITY, + position: "top-center", + unstyled: true, + style: { + left: 0, + right: 0, + display: "flex", + justifyContent: "center", + }, + }, + ); + indicatorToastRef.current = toastId; + break; + } + + case "audio_captured": { + if (indicatorToastRef.current !== null) { + toast.dismiss(indicatorToastRef.current); + indicatorToastRef.current = null; + } + + const toastId = toast.custom( + (id) => , + { + duration: Number.POSITIVE_INFINITY, + position: "top-center", + unstyled: true, + style: { + left: 0, + right: 0, + display: "flex", + justifyContent: "center", + }, + }, + ); + responseToastRef.current = toastId; + break; + } + + case "idle": { + if (indicatorToastRef.current !== null) { + toast.dismiss(indicatorToastRef.current); + indicatorToastRef.current = null; + } + break; + } + + case "error": { + dismissAll(indicatorToastRef, responseToastRef); + console.error("[voice-listener] Sidecar error:", event.message); + break; + } + } + }, + onError: (error) => { + console.error("[voice-listener] Subscription error:", error); + }, + }); + + return null; +} + +function dismissAll(...refs: React.RefObject[]): void { + for (const ref of refs) { + if (ref.current !== null) { + toast.dismiss(ref.current); + ref.current = null; + } + } +} diff --git a/apps/desktop/src/renderer/components/Voice/components/RecordingIndicator/RecordingIndicator.tsx b/apps/desktop/src/renderer/components/Voice/components/RecordingIndicator/RecordingIndicator.tsx new file mode 100644 index 00000000000..1d91a5a469d --- /dev/null +++ b/apps/desktop/src/renderer/components/Voice/components/RecordingIndicator/RecordingIndicator.tsx @@ -0,0 +1,26 @@ +import { toast } from "@superset/ui/sonner"; +import { HiMiniMicrophone } from "react-icons/hi2"; + +interface RecordingIndicatorProps { + toastId: string | number; +} + +export function RecordingIndicator({ toastId }: RecordingIndicatorProps) { + return ( +
+ + + + + + Listening... + +
+ ); +} diff --git a/apps/desktop/src/renderer/components/Voice/components/RecordingIndicator/index.ts b/apps/desktop/src/renderer/components/Voice/components/RecordingIndicator/index.ts new file mode 100644 index 00000000000..a7487ae9461 --- /dev/null +++ b/apps/desktop/src/renderer/components/Voice/components/RecordingIndicator/index.ts @@ -0,0 +1 @@ +export { RecordingIndicator } from "./RecordingIndicator"; diff --git a/apps/desktop/src/renderer/components/Voice/components/ResponsePanel/ResponsePanel.tsx b/apps/desktop/src/renderer/components/Voice/components/ResponsePanel/ResponsePanel.tsx new file mode 100644 index 00000000000..d039365c2bf --- /dev/null +++ b/apps/desktop/src/renderer/components/Voice/components/ResponsePanel/ResponsePanel.tsx @@ -0,0 +1,129 @@ +import { Button } from "@superset/ui/button"; +import { toast } from "@superset/ui/sonner"; +import { useEffect } from "react"; +import { HiMiniMicrophone } from "react-icons/hi2"; +import { useVoicePipeline } from "./hooks/useVoicePipeline"; + +interface ResponsePanelProps { + toastId: string | number; + audioB64: string; +} + +export function ResponsePanel({ toastId, audioB64 }: ResponsePanelProps) { + const { + status, + transcription, + toolCalls, + responseText, + error, + processAudio, + abort, + } = useVoicePipeline(); + + // Start processing when mounted + useEffect(() => { + processAudio(audioB64); + }, [audioB64, processAudio]); + + // Auto-dismiss after done or error + useEffect(() => { + if (status === "done" || status === "error") { + const timer = setTimeout(() => { + toast.dismiss(toastId); + }, 8000); + return () => clearTimeout(timer); + } + }, [status, toastId]); + + const handleStop = () => { + abort(); + }; + + const isActive = + status === "transcribing" || + status === "processing" || + status === "streaming"; + + return ( +
+
+ {/* Header */} +
+ + Voice Command +
+ + {/* Status indicator */} + {status === "transcribing" && ( +
+ + + + + Transcribing... +
+ )} + + {/* Transcription */} + {transcription && ( +
+ “{transcription}” +
+ )} + + {/* Tool calls */} + {toolCalls.length > 0 && ( +
+ {toolCalls.map((tc, i) => ( +
+ + {tc.toolName} + {tc.result && done} +
+ ))} +
+ )} + + {/* Streaming response */} + {(status === "streaming" || status === "done") && responseText && ( +
+ {responseText} + {status === "streaming" && ( + + )} +
+ )} + + {/* Processing indicator */} + {status === "processing" && !responseText && ( +
+ + + + + Thinking... +
+ )} + + {/* Error */} + {status === "error" && ( +
+ {error || "Something went wrong"} +
+ )} +
+ + {/* Stop button footer */} + {isActive && ( +
+ +
+ )} +
+ ); +} diff --git a/apps/desktop/src/renderer/components/Voice/components/ResponsePanel/hooks/useVoicePipeline/index.ts b/apps/desktop/src/renderer/components/Voice/components/ResponsePanel/hooks/useVoicePipeline/index.ts new file mode 100644 index 00000000000..e0ce15fb143 --- /dev/null +++ b/apps/desktop/src/renderer/components/Voice/components/ResponsePanel/hooks/useVoicePipeline/index.ts @@ -0,0 +1 @@ +export { useVoicePipeline } from "./useVoicePipeline"; diff --git a/apps/desktop/src/renderer/components/Voice/components/ResponsePanel/hooks/useVoicePipeline/useVoicePipeline.ts b/apps/desktop/src/renderer/components/Voice/components/ResponsePanel/hooks/useVoicePipeline/useVoicePipeline.ts new file mode 100644 index 00000000000..8250ba7d630 --- /dev/null +++ b/apps/desktop/src/renderer/components/Voice/components/ResponsePanel/hooks/useVoicePipeline/useVoicePipeline.ts @@ -0,0 +1,194 @@ +import { useCallback, useRef, useState } from "react"; +import { getAuthToken } from "renderer/lib/auth-client"; +import { env } from "renderer/env.renderer"; + +type PipelineStatus = + | "idle" + | "transcribing" + | "processing" + | "streaming" + | "done" + | "error"; + +interface ToolCall { + toolName: string; + toolInput?: unknown; + result?: string; +} + +interface VoicePipelineState { + status: PipelineStatus; + transcription: string | null; + toolCalls: ToolCall[]; + responseText: string; + error: string | null; +} + +const INITIAL_STATE: VoicePipelineState = { + status: "idle", + transcription: null, + toolCalls: [], + responseText: "", + error: null, +}; + +export function useVoicePipeline() { + const [state, setState] = useState(INITIAL_STATE); + const abortRef = useRef(null); + + const processAudio = useCallback(async (audioB64: string) => { + abortRef.current?.abort(); + setState({ ...INITIAL_STATE, status: "transcribing" }); + + const binaryStr = atob(audioB64); + const bytes = new Uint8Array(binaryStr.length); + for (let i = 0; i < binaryStr.length; i++) { + bytes[i] = binaryStr.charCodeAt(i); + } + + const formData = new FormData(); + formData.append( + "audio", + new Blob([bytes], { type: "audio/wav" }), + "audio.wav", + ); + + const abortController = new AbortController(); + abortRef.current = abortController; + + try { + const headers: HeadersInit = {}; + const token = getAuthToken(); + if (token) { + headers.Authorization = `Bearer ${token}`; + } + + const response = await fetch(`${env.NEXT_PUBLIC_API_URL}/api/voice`, { + method: "POST", + body: formData, + credentials: "include", + headers, + signal: abortController.signal, + }); + + if (!response.ok) { + const text = await response.text(); + setState((prev) => ({ + ...prev, + status: "error", + error: `API error: ${response.status} ${text}`, + })); + return; + } + + if (!response.body) { + setState((prev) => ({ + ...prev, + status: "error", + error: "No response body", + })); + return; + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() ?? ""; + + let eventType = ""; + for (const line of lines) { + if (line.startsWith("event: ")) { + eventType = line.slice(7).trim(); + } else if (line.startsWith("data: ") && eventType) { + try { + handleSSEEvent(eventType, JSON.parse(line.slice(6))); + } catch { + // Skip malformed data + } + eventType = ""; + } + } + } + + setState((prev) => + prev.status !== "error" ? { ...prev, status: "done" } : prev, + ); + } catch (error) { + if (abortController.signal.aborted) return; + setState((prev) => ({ + ...prev, + status: "error", + error: error instanceof Error ? error.message : "Request failed", + })); + } + }, []); + + const abort = useCallback(() => { + abortRef.current?.abort(); + setState((prev) => + prev.status !== "error" && + prev.status !== "done" && + prev.status !== "idle" + ? { ...prev, status: "done" } + : prev, + ); + }, []); + + function handleSSEEvent(event: string, data: Record) { + switch (event) { + case "transcription": + setState((prev) => ({ + ...prev, + status: "processing", + transcription: data.text as string, + })); + break; + case "tool_use": + setState((prev) => ({ + ...prev, + status: "processing", + toolCalls: [ + ...prev.toolCalls, + { toolName: data.toolName as string, toolInput: data.toolInput }, + ], + })); + break; + case "tool_result": + setState((prev) => ({ + ...prev, + toolCalls: prev.toolCalls.map((tc) => + tc.toolName === data.toolName && !tc.result + ? { ...tc, result: data.result as string } + : tc, + ), + })); + break; + case "text_delta": + setState((prev) => ({ + ...prev, + status: "streaming", + responseText: prev.responseText + (data.delta as string), + })); + break; + case "done": + setState((prev) => ({ ...prev, status: "done" })); + break; + case "error": + setState((prev) => ({ + ...prev, + status: "error", + error: data.message as string, + })); + break; + } + } + + return { ...state, processAudio, abort }; +} diff --git a/apps/desktop/src/renderer/components/Voice/components/ResponsePanel/index.ts b/apps/desktop/src/renderer/components/Voice/components/ResponsePanel/index.ts new file mode 100644 index 00000000000..d34650661d5 --- /dev/null +++ b/apps/desktop/src/renderer/components/Voice/components/ResponsePanel/index.ts @@ -0,0 +1 @@ +export { ResponsePanel } from "./ResponsePanel"; diff --git a/apps/desktop/src/renderer/components/Voice/index.ts b/apps/desktop/src/renderer/components/Voice/index.ts new file mode 100644 index 00000000000..7ecc6a6c63a --- /dev/null +++ b/apps/desktop/src/renderer/components/Voice/index.ts @@ -0,0 +1 @@ +export { VoiceListener } from "./VoiceListener"; diff --git a/apps/desktop/src/renderer/routes/_authenticated/layout.tsx b/apps/desktop/src/renderer/routes/_authenticated/layout.tsx index 146b5b6de52..3100b3a070f 100644 --- a/apps/desktop/src/renderer/routes/_authenticated/layout.tsx +++ b/apps/desktop/src/renderer/routes/_authenticated/layout.tsx @@ -8,6 +8,7 @@ import { DndProvider } from "react-dnd"; import { NewWorkspaceModal } from "renderer/components/NewWorkspaceModal"; import { Paywall } from "renderer/components/Paywall"; import { useUpdateListener } from "renderer/components/UpdateToast"; +import { VoiceListener } from "renderer/components/Voice"; import { env } from "renderer/env.renderer"; import { authClient } from "renderer/lib/auth-client"; import { dragDropManager } from "renderer/lib/dnd"; @@ -78,6 +79,7 @@ function AuthenticatedLayout() { + diff --git a/apps/desktop/src/renderer/routes/_authenticated/settings/behavior/components/BehaviorSettings/BehaviorSettings.tsx b/apps/desktop/src/renderer/routes/_authenticated/settings/behavior/components/BehaviorSettings/BehaviorSettings.tsx index 25a17e454d7..7c0255ce992 100644 --- a/apps/desktop/src/renderer/routes/_authenticated/settings/behavior/components/BehaviorSettings/BehaviorSettings.tsx +++ b/apps/desktop/src/renderer/routes/_authenticated/settings/behavior/components/BehaviorSettings/BehaviorSettings.tsx @@ -32,6 +32,10 @@ export function BehaviorSettings({ visibleItems }: BehaviorSettingsProps) { SETTING_ITEM_ID.BEHAVIOR_BRANCH_PREFIX, visibleItems, ); + const showVoiceCommands = isItemVisible( + SETTING_ITEM_ID.BEHAVIOR_VOICE_COMMANDS, + visibleItems, + ); const utils = electronTrpc.useUtils(); @@ -58,6 +62,68 @@ export function BehaviorSettings({ visibleItems }: BehaviorSettingsProps) { setConfirmOnQuit.mutate({ enabled }); }; + const { data: voiceCommandsEnabled, isLoading: isVoiceLoading } = + electronTrpc.settings.getVoiceCommandsEnabled.useQuery(); + const setVoiceCommandsEnabled = + electronTrpc.settings.setVoiceCommandsEnabled.useMutation({ + onMutate: async ({ enabled }) => { + await utils.settings.getVoiceCommandsEnabled.cancel(); + const previous = utils.settings.getVoiceCommandsEnabled.getData(); + utils.settings.getVoiceCommandsEnabled.setData(undefined, enabled); + return { previous }; + }, + onError: (_err, _vars, context) => { + if (context?.previous !== undefined) { + utils.settings.getVoiceCommandsEnabled.setData( + undefined, + context.previous, + ); + } + }, + onSettled: () => { + utils.settings.getVoiceCommandsEnabled.invalidate(); + }, + }); + + const { data: micPermission } = electronTrpc.voice.getMicPermission.useQuery( + undefined, + { + refetchOnWindowFocus: true, + }, + ); + + const requestMicPermission = + electronTrpc.voice.requestMicPermission.useMutation({ + onSuccess: ({ granted }) => { + utils.voice.getMicPermission.invalidate(); + if (granted) { + setVoiceCommandsEnabled.mutate({ enabled: true }); + } + }, + }); + + const openUrl = electronTrpc.external.openUrl.useMutation(); + + const micDenied = + micPermission === "denied" || micPermission === "restricted"; + + const handleVoiceToggle = (enabled: boolean) => { + if (!enabled) { + setVoiceCommandsEnabled.mutate({ enabled: false }); + return; + } + + if (micPermission === "granted") { + setVoiceCommandsEnabled.mutate({ enabled: true }); + return; + } + + if (micPermission === "not-determined") { + requestMicPermission.mutate(); + return; + } + }; + const { data: branchPrefix, isLoading: isBranchPrefixLoading } = electronTrpc.settings.getBranchPrefix.useQuery(); const { data: gitInfo } = electronTrpc.settings.getGitInfo.useQuery(); @@ -137,6 +203,49 @@ export function BehaviorSettings({ visibleItems }: BehaviorSettingsProps) { )} + {showVoiceCommands && ( +
+
+
+ +

+ Say "Hey Jarvis" to control Superset with your voice +

+
+ +
+ {micDenied && ( +

+ Microphone access was denied.{" "} + {" "} + to grant access, then return to this window. +

+ )} +
+ )} + {showBranchPrefix && (
diff --git a/apps/desktop/src/renderer/routes/_authenticated/settings/utils/settings-search/settings-search.ts b/apps/desktop/src/renderer/routes/_authenticated/settings/utils/settings-search/settings-search.ts index 568eeaa9f60..242ebfe7d8f 100644 --- a/apps/desktop/src/renderer/routes/_authenticated/settings/utils/settings-search/settings-search.ts +++ b/apps/desktop/src/renderer/routes/_authenticated/settings/utils/settings-search/settings-search.ts @@ -21,6 +21,7 @@ export const SETTING_ITEM_ID = { KEYBOARD_SHORTCUTS: "keyboard-shortcuts", BEHAVIOR_CONFIRM_QUIT: "behavior-confirm-quit", + BEHAVIOR_VOICE_COMMANDS: "behavior-voice-commands", BEHAVIOR_BRANCH_PREFIX: "behavior-branch-prefix", TERMINAL_PRESETS: "terminal-presets", @@ -308,6 +309,23 @@ export const SETTINGS_ITEMS: SettingsItem[] = [ "unsaved", ], }, + { + id: SETTING_ITEM_ID.BEHAVIOR_VOICE_COMMANDS, + section: "behavior", + title: "Voice Commands", + description: + 'Say "Hey Jarvis" to create tasks, ask questions, and manage your work by voice', + keywords: [ + "features", + "voice", + "commands", + "wake word", + "microphone", + "speech", + "audio", + "jarvis", + ], + }, { id: SETTING_ITEM_ID.BEHAVIOR_BRANCH_PREFIX, section: "behavior", diff --git a/apps/desktop/src/resources/build/entitlements.mac.plist b/apps/desktop/src/resources/build/entitlements.mac.plist new file mode 100644 index 00000000000..f7d1e352274 --- /dev/null +++ b/apps/desktop/src/resources/build/entitlements.mac.plist @@ -0,0 +1,14 @@ + + + + + com.apple.security.cs.allow-jit + + com.apple.security.cs.allow-unsigned-executable-memory + + com.apple.security.cs.disable-library-validation + + com.apple.security.device.audio-input + + + diff --git a/bun.lock b/bun.lock index 03d8b44a23f..b75f168dfe2 100644 --- a/bun.lock +++ b/bun.lock @@ -85,6 +85,7 @@ "lodash.chunk": "^4.2.0", "mcp-handler": "^1.0.7", "next": "^16.0.10", + "openai": "^6.17.0", "react": "19.1.0", "react-dom": "19.1.0", "require-in-the-middle": "8.0.1", @@ -3948,6 +3949,8 @@ "open": ["open@7.4.2", "", { "dependencies": { "is-docker": "^2.0.0", "is-wsl": "^2.1.1" } }, "sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q=="], + "openai": ["openai@6.17.0", "", { "peerDependencies": { "ws": "^8.18.0", "zod": "^3.25 || ^4.0" }, "optionalPeers": ["ws", "zod"], "bin": { "openai": "bin/cli" } }, "sha512-NHRpPEUPzAvFOAFs9+9pC6+HCw/iWsYsKCMPXH5Kw7BpMxqd8g/A07/1o7Gx2TWtCnzevVRyKMRFqyiHyAlqcA=="], + "opentype.js": ["opentype.js@0.8.0", "", { "dependencies": { "tiny-inflate": "^1.0.2" }, "bin": { "ot": "./bin/ot" } }, "sha512-FQHR4oGP+a0m/f6yHoRpBOIbn/5ZWxKd4D/djHVJu8+KpBTYrJda0b7mLcgDEMWXE9xBCJm+qb0yv6FcvPjukg=="], "ora": ["ora@8.2.0", "", { "dependencies": { "chalk": "^5.3.0", "cli-cursor": "^5.0.0", "cli-spinners": "^2.9.2", "is-interactive": "^2.0.0", "is-unicode-supported": "^2.0.0", "log-symbols": "^6.0.0", "stdin-discarder": "^0.2.2", "string-width": "^7.2.0", "strip-ansi": "^7.1.0" } }, "sha512-weP+BZ8MVNnlCm8c0Qdc1WSWq4Qn7I+9CJGm7Qali6g44e/PUzbjNqJX5NJ9ljlNMosfJvg1fKEGILklK9cwnw=="], diff --git a/packages/local-db/drizzle/0016_add_voice_commands_enabled.sql b/packages/local-db/drizzle/0016_add_voice_commands_enabled.sql new file mode 100644 index 00000000000..76838600137 --- /dev/null +++ b/packages/local-db/drizzle/0016_add_voice_commands_enabled.sql @@ -0,0 +1 @@ +ALTER TABLE `settings` ADD `voice_commands_enabled` integer DEFAULT false; \ No newline at end of file diff --git a/packages/local-db/drizzle/meta/0016_snapshot.json b/packages/local-db/drizzle/meta/0016_snapshot.json new file mode 100644 index 00000000000..de165c0f6e9 --- /dev/null +++ b/packages/local-db/drizzle/meta/0016_snapshot.json @@ -0,0 +1,1057 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "c5371ab6-1178-4cb5-b635-ab01e82cb6a0", + "prevId": "2c6f4b00-72ca-4cc3-bc0a-f25a40163119", + "tables": { + "organization_members": { + "name": "organization_members", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "organization_members_organization_id_idx": { + "name": "organization_members_organization_id_idx", + "columns": [ + "organization_id" + ], + "isUnique": false + }, + "organization_members_user_id_idx": { + "name": "organization_members_user_id_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "organization_members_organization_id_organizations_id_fk": { + "name": "organization_members_organization_id_organizations_id_fk", + "tableFrom": "organization_members", + "tableTo": "organizations", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "organization_members_user_id_users_id_fk": { + "name": "organization_members_user_id_users_id_fk", + "tableFrom": "organization_members", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "organizations": { + "name": "organizations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "clerk_org_id": { + "name": "clerk_org_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "github_org": { + "name": "github_org", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "organizations_clerk_org_id_unique": { + "name": "organizations_clerk_org_id_unique", + "columns": [ + "clerk_org_id" + ], + "isUnique": true + }, + "organizations_slug_unique": { + "name": "organizations_slug_unique", + "columns": [ + "slug" + ], + "isUnique": true + }, + "organizations_slug_idx": { + "name": "organizations_slug_idx", + "columns": [ + "slug" + ], + "isUnique": false + }, + "organizations_clerk_org_id_idx": { + "name": "organizations_clerk_org_id_idx", + "columns": [ + "clerk_org_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "projects": { + "name": "projects", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "main_repo_path": { + "name": "main_repo_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tab_order": { + "name": "tab_order", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_opened_at": { + "name": "last_opened_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "config_toast_dismissed": { + "name": "config_toast_dismissed", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_branch": { + "name": "default_branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_owner": { + "name": "github_owner", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "branch_prefix_mode": { + "name": "branch_prefix_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "branch_prefix_custom": { + "name": "branch_prefix_custom", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "projects_main_repo_path_idx": { + "name": "projects_main_repo_path_idx", + "columns": [ + "main_repo_path" + ], + "isUnique": false + }, + "projects_last_opened_at_idx": { + "name": "projects_last_opened_at_idx", + "columns": [ + "last_opened_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "last_active_workspace_id": { + "name": "last_active_workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_used_app": { + "name": "last_used_app", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "terminal_presets": { + "name": "terminal_presets", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "terminal_presets_initialized": { + "name": "terminal_presets_initialized", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "selected_ringtone_id": { + "name": "selected_ringtone_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "confirm_on_quit": { + "name": "confirm_on_quit", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "terminal_link_behavior": { + "name": "terminal_link_behavior", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "persist_terminal": { + "name": "persist_terminal", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "auto_apply_default_preset": { + "name": "auto_apply_default_preset", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "branch_prefix_mode": { + "name": "branch_prefix_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "branch_prefix_custom": { + "name": "branch_prefix_custom", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "notification_sounds_muted": { + "name": "notification_sounds_muted", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "voice_commands_enabled": { + "name": "voice_commands_enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tasks": { + "name": "tasks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status_color": { + "name": "status_color", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status_type": { + "name": "status_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status_position": { + "name": "status_position", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "priority": { + "name": "priority", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "assignee_id": { + "name": "assignee_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "creator_id": { + "name": "creator_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "estimate": { + "name": "estimate", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "due_date": { + "name": "due_date", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "labels": { + "name": "labels", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pr_url": { + "name": "pr_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "external_provider": { + "name": "external_provider", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "external_key": { + "name": "external_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "external_url": { + "name": "external_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_synced_at": { + "name": "last_synced_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sync_error": { + "name": "sync_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "tasks_slug_unique": { + "name": "tasks_slug_unique", + "columns": [ + "slug" + ], + "isUnique": true + }, + "tasks_slug_idx": { + "name": "tasks_slug_idx", + "columns": [ + "slug" + ], + "isUnique": false + }, + "tasks_organization_id_idx": { + "name": "tasks_organization_id_idx", + "columns": [ + "organization_id" + ], + "isUnique": false + }, + "tasks_assignee_id_idx": { + "name": "tasks_assignee_id_idx", + "columns": [ + "assignee_id" + ], + "isUnique": false + }, + "tasks_status_idx": { + "name": "tasks_status_idx", + "columns": [ + "status" + ], + "isUnique": false + }, + "tasks_created_at_idx": { + "name": "tasks_created_at_idx", + "columns": [ + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "tasks_organization_id_organizations_id_fk": { + "name": "tasks_organization_id_organizations_id_fk", + "tableFrom": "tasks", + "tableTo": "organizations", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tasks_assignee_id_users_id_fk": { + "name": "tasks_assignee_id_users_id_fk", + "tableFrom": "tasks", + "tableTo": "users", + "columnsFrom": [ + "assignee_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "tasks_creator_id_users_id_fk": { + "name": "tasks_creator_id_users_id_fk", + "tableFrom": "tasks", + "tableTo": "users", + "columnsFrom": [ + "creator_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "clerk_id": { + "name": "clerk_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "users_clerk_id_unique": { + "name": "users_clerk_id_unique", + "columns": [ + "clerk_id" + ], + "isUnique": true + }, + "users_email_unique": { + "name": "users_email_unique", + "columns": [ + "email" + ], + "isUnique": true + }, + "users_email_idx": { + "name": "users_email_idx", + "columns": [ + "email" + ], + "isUnique": false + }, + "users_clerk_id_idx": { + "name": "users_clerk_id_idx", + "columns": [ + "clerk_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspaces": { + "name": "workspaces", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "worktree_id": { + "name": "worktree_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tab_order": { + "name": "tab_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_opened_at": { + "name": "last_opened_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_unread": { + "name": "is_unread", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "deleting_at": { + "name": "deleting_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "workspaces_project_id_idx": { + "name": "workspaces_project_id_idx", + "columns": [ + "project_id" + ], + "isUnique": false + }, + "workspaces_worktree_id_idx": { + "name": "workspaces_worktree_id_idx", + "columns": [ + "worktree_id" + ], + "isUnique": false + }, + "workspaces_last_opened_at_idx": { + "name": "workspaces_last_opened_at_idx", + "columns": [ + "last_opened_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "workspaces_project_id_projects_id_fk": { + "name": "workspaces_project_id_projects_id_fk", + "tableFrom": "workspaces", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workspaces_worktree_id_worktrees_id_fk": { + "name": "workspaces_worktree_id_worktrees_id_fk", + "tableFrom": "workspaces", + "tableTo": "worktrees", + "columnsFrom": [ + "worktree_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "worktrees": { + "name": "worktrees", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_branch": { + "name": "base_branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "git_status": { + "name": "git_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_status": { + "name": "github_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "worktrees_project_id_idx": { + "name": "worktrees_project_id_idx", + "columns": [ + "project_id" + ], + "isUnique": false + }, + "worktrees_branch_idx": { + "name": "worktrees_branch_idx", + "columns": [ + "branch" + ], + "isUnique": false + } + }, + "foreignKeys": { + "worktrees_project_id_projects_id_fk": { + "name": "worktrees_project_id_projects_id_fk", + "tableFrom": "worktrees", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/local-db/drizzle/meta/_journal.json b/packages/local-db/drizzle/meta/_journal.json index 84c19fde780..a72ac8c1217 100644 --- a/packages/local-db/drizzle/meta/_journal.json +++ b/packages/local-db/drizzle/meta/_journal.json @@ -113,6 +113,13 @@ "when": 1769649140217, "tag": "0015_add_notification_sounds_muted", "breakpoints": true + }, + { + "idx": 16, + "version": "6", + "when": 1769708198787, + "tag": "0016_add_voice_commands_enabled", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/local-db/src/schema/schema.ts b/packages/local-db/src/schema/schema.ts index 7b0ff0a1ef7..e7ec1208827 100644 --- a/packages/local-db/src/schema/schema.ts +++ b/packages/local-db/src/schema/schema.ts @@ -153,6 +153,9 @@ export const settings = sqliteTable("settings", { notificationSoundsMuted: integer("notification_sounds_muted", { mode: "boolean", }), + voiceCommandsEnabled: integer("voice_commands_enabled", { + mode: "boolean", + }).default(false), }); export type InsertSettings = typeof settings.$inferInsert;