From b51b16e3544e998f4eaad45b470a0b9b8f8eef89 Mon Sep 17 00:00:00 2001 From: Marius Date: Fri, 26 Jun 2026 20:40:29 +0200 Subject: [PATCH 1/2] fix(vscode): wait for microphone capture readiness --- .changeset/warm-speech-capture.md | 5 ++ .../kilo-vscode/src/services/input-tools.ts | 6 ++ .../kilo-vscode/src/speech-to-text/capture.ts | 24 +++++- .../tests/unit/speech-to-text-prewarm.test.ts | 73 +++++++++++++++++++ .../tests/unit/use-speech-to-text.test.ts | 56 ++++++++++++++ .../agent-manager/AgentManagerApp.tsx | 2 + .../webview-ui/diff-viewer/DiffViewerApp.tsx | 2 + packages/kilo-vscode/webview-ui/src/App.tsx | 2 + .../speech-to-text/SpeechToTextButton.tsx | 19 +++-- .../speech-to-text/SpeechToTextPrewarm.tsx | 20 +++++ .../speech-to-text/useSpeechToText.ts | 7 +- .../kilo-vscode/webview-ui/src/i18n/ar.ts | 1 + .../kilo-vscode/webview-ui/src/i18n/br.ts | 1 + .../kilo-vscode/webview-ui/src/i18n/bs.ts | 1 + .../kilo-vscode/webview-ui/src/i18n/da.ts | 1 + .../kilo-vscode/webview-ui/src/i18n/de.ts | 1 + .../kilo-vscode/webview-ui/src/i18n/en.ts | 3 +- .../kilo-vscode/webview-ui/src/i18n/es.ts | 1 + .../kilo-vscode/webview-ui/src/i18n/fr.ts | 1 + .../kilo-vscode/webview-ui/src/i18n/it.ts | 1 + .../kilo-vscode/webview-ui/src/i18n/ja.ts | 1 + .../kilo-vscode/webview-ui/src/i18n/ko.ts | 1 + .../kilo-vscode/webview-ui/src/i18n/nl.ts | 1 + .../kilo-vscode/webview-ui/src/i18n/no.ts | 1 + .../kilo-vscode/webview-ui/src/i18n/pl.ts | 1 + .../kilo-vscode/webview-ui/src/i18n/ru.ts | 1 + .../kilo-vscode/webview-ui/src/i18n/th.ts | 1 + .../kilo-vscode/webview-ui/src/i18n/tr.ts | 1 + .../kilo-vscode/webview-ui/src/i18n/uk.ts | 1 + .../kilo-vscode/webview-ui/src/i18n/zh.ts | 1 + .../kilo-vscode/webview-ui/src/i18n/zht.ts | 1 + .../src/types/messages/webview-messages.ts | 5 ++ 32 files changed, 233 insertions(+), 10 deletions(-) create mode 100644 .changeset/warm-speech-capture.md create mode 100644 packages/kilo-vscode/tests/unit/speech-to-text-prewarm.test.ts create mode 100644 packages/kilo-vscode/webview-ui/src/components/speech-to-text/SpeechToTextPrewarm.tsx diff --git a/.changeset/warm-speech-capture.md b/.changeset/warm-speech-capture.md new file mode 100644 index 000000000000..64d9be64cc6e --- /dev/null +++ b/.changeset/warm-speech-capture.md @@ -0,0 +1,5 @@ +--- +"kilo-code": patch +--- + +Wait for microphone capture to start before showing voice input as recording. diff --git a/packages/kilo-vscode/src/services/input-tools.ts b/packages/kilo-vscode/src/services/input-tools.ts index 7d722b684edc..23479bbf2087 100644 --- a/packages/kilo-vscode/src/services/input-tools.ts +++ b/packages/kilo-vscode/src/services/input-tools.ts @@ -1,6 +1,7 @@ import type { KiloConnectionService } from "./cli-backend/connection-service" import { routeAutocompleteMessage } from "./autocomplete/settings" import { handleSpeechToTextCancel, handleSpeechToTextStart, handleSpeechToTextStop } from "../speech-to-text/handler" +import { prewarmSpeechCapture } from "../speech-to-text/capture" type Msg = { type: string @@ -18,6 +19,11 @@ type Ctx = { export async function routeInputToolMessage(message: Msg, ctx: Ctx): Promise { if (await routeAutocompleteMessage(message, ctx.post)) return true + if (message.type === "speechToTextPrewarm") { + void prewarmSpeechCapture().catch((err) => console.warn("[Kilo New] Speech capture prewarm failed:", err)) + return true + } + if (message.type === "speechToTextStart") { if (!message.requestId) return true handleSpeechToTextStart( diff --git a/packages/kilo-vscode/src/speech-to-text/capture.ts b/packages/kilo-vscode/src/speech-to-text/capture.ts index 03efa67aaffc..a2d61885bc72 100644 --- a/packages/kilo-vscode/src/speech-to-text/capture.ts +++ b/packages/kilo-vscode/src/speech-to-text/capture.ts @@ -36,13 +36,18 @@ type Args = { let active: Recording | undefined let starting: string | undefined +let ffmpeg: Promise | undefined + +export async function prewarmSpeechCapture(): Promise { + await resolveFFmpeg() +} export async function startSpeechCapture(input: Input): Promise { if (active || starting) throw new Error("Speech recording is already in progress") starting = input.requestId try { - const bin = await findFFmpeg() + const bin = await resolveFFmpeg() const file = path.join(os.tmpdir(), `kilo-stt-${process.pid}-${Date.now()}.wav`) const state = await startWithArgs(bin, file, input, await inputArgSets(bin)) return !state.stopped @@ -229,6 +234,23 @@ function requireActive(requestId: string): Recording { return active } +async function resolveFFmpeg(): Promise { + if (ffmpeg) { + const bin = await ffmpeg + if (!path.isAbsolute(bin) || existsSync(bin)) return bin + ffmpeg = undefined + } + + const task = findFFmpeg() + ffmpeg = task + try { + return await task + } catch (err) { + if (ffmpeg === task) ffmpeg = undefined + throw err + } +} + async function findFFmpeg(): Promise { const paths = [ process.env.KILO_FFMPEG_PATH, diff --git a/packages/kilo-vscode/tests/unit/speech-to-text-prewarm.test.ts b/packages/kilo-vscode/tests/unit/speech-to-text-prewarm.test.ts new file mode 100644 index 000000000000..b8ce811bdc41 --- /dev/null +++ b/packages/kilo-vscode/tests/unit/speech-to-text-prewarm.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "bun:test" +import path from "node:path" + +const ROOT = path.resolve(import.meta.dir, "../..") + +describe("speech-to-text prewarm", () => { + it("starts only after Kilo speech access becomes available", () => { + const result = Bun.spawnSync( + [ + "bun", + "--conditions=browser", + "-e", + ` + import { Window } from "happy-dom" + + const window = new Window() + globalThis.window = window + globalThis.document = window.document + globalThis.Node = window.Node + + const sent = [] + globalThis.acquireVsCodeApi = () => ({ + postMessage: (message) => sent.push(message), + getState: () => undefined, + setState: () => {}, + }) + + const { createComponent, createSignal } = await import("solid-js") + const { render } = await import("solid-js/web") + const { ConfigContext } = await import("./webview-ui/src/context/config.tsx") + const { ProviderContext } = await import("./webview-ui/src/context/provider.tsx") + const { SpeechToTextPrewarm } = await import( + "./webview-ui/src/components/speech-to-text/SpeechToTextPrewarm.tsx" + ) + + const [config, setConfig] = createSignal({ disabled_providers: ["kilo"] }) + const [auth, setAuth] = createSignal({}) + const root = document.createElement("div") + const dispose = render( + () => + createComponent(ProviderContext.Provider, { + value: { authStates: auth }, + get children() { + return createComponent(ConfigContext.Provider, { + value: { config }, + get children() { + return createComponent(SpeechToTextPrewarm, {}) + }, + }) + }, + }), + root, + ) + + if (sent.length !== 0) throw new Error("prewarmed without Kilo access") + setAuth({ kilo: "api" }) + if (sent.length !== 0) throw new Error("prewarmed while Kilo was disabled") + setConfig({}) + if (sent.length !== 1 || sent[0]?.type !== "speechToTextPrewarm") { + throw new Error("did not prewarm after Kilo access became available") + } + setAuth({ kilo: "oauth" }) + if (sent.length !== 1) throw new Error("prewarmed more than once") + dispose() + `, + ], + { cwd: ROOT, stdout: "pipe", stderr: "pipe" }, + ) + + const output = result.stdout.toString() + result.stderr.toString() + expect(result.exitCode, output).toBe(0) + }) +}) diff --git a/packages/kilo-vscode/tests/unit/use-speech-to-text.test.ts b/packages/kilo-vscode/tests/unit/use-speech-to-text.test.ts index ae28f0b41718..3b406e9cdd9f 100644 --- a/packages/kilo-vscode/tests/unit/use-speech-to-text.test.ts +++ b/packages/kilo-vscode/tests/unit/use-speech-to-text.test.ts @@ -41,6 +41,59 @@ function setup() { } describe("useSpeechToText", () => { + it("waits for microphone readiness before reporting recording", () => { + const ctx = setup() + + ctx.speech.start({ model: "scribe", insert: () => {} }) + const start = ctx.sent[0] + if (start?.type !== "speechToTextStart") throw new Error("speech start message missing") + + expect(ctx.speech.state()).toBe("starting") + expect(ctx.speech.active()).toBe(true) + + ctx.fire({ type: "speechToTextStarted", requestId: "another-request" }) + expect(ctx.speech.state()).toBe("starting") + + ctx.fire({ type: "speechToTextStarted", requestId: start.requestId }) + expect(ctx.speech.state()).toBe("recording") + ctx.dispose() + }) + + it("does not stop or start another recording while the microphone is starting", () => { + const ctx = setup() + + ctx.speech.start({ model: "scribe", insert: () => {} }) + const start = ctx.sent[0] + if (start?.type !== "speechToTextStart") throw new Error("speech start message missing") + + ctx.speech.stop() + ctx.speech.start({ model: "other", insert: () => {} }) + expect(ctx.speech.state()).toBe("starting") + expect(ctx.sent).toEqual([start]) + + ctx.fire({ type: "speechToTextStarted", requestId: start.requestId }) + ctx.speech.stop() + expect(ctx.speech.state()).toBe("transcribing") + expect(ctx.sent[1]).toEqual({ type: "speechToTextStop", requestId: start.requestId }) + ctx.dispose() + }) + + it("cancels a pending microphone startup and ignores its late acknowledgement", () => { + const ctx = setup() + + ctx.speech.start({ model: "scribe", insert: () => {} }) + const start = ctx.sent[0] + if (start?.type !== "speechToTextStart") throw new Error("speech start message missing") + + ctx.speech.cancel() + expect(ctx.sent[1]).toEqual({ type: "speechToTextCancel", requestId: start.requestId }) + expect(ctx.speech.state()).toBe("idle") + + ctx.fire({ type: "speechToTextStarted", requestId: start.requestId }) + expect(ctx.speech.state()).toBe("idle") + ctx.dispose() + }) + it("offers sign-in when stored credentials stop authenticating", () => { const ctx = setup() @@ -70,6 +123,7 @@ describe("useSpeechToText", () => { ctx.speech.start({ model: "scribe", insert: (value) => text.push(value) }) const start = ctx.sent[0] if (start?.type !== "speechToTextStart") throw new Error("speech start message missing") + ctx.fire({ type: "speechToTextStarted", requestId: start.requestId }) ctx.speech.stop({ done: () => done++ }) ctx.fire({ type: "speechToTextResult", requestId: start.requestId, text: "Recorded prompt" }) @@ -87,6 +141,7 @@ describe("useSpeechToText", () => { ctx.speech.start({ model: "scribe", insert: () => {} }) const start = ctx.sent[0] if (start?.type !== "speechToTextStart") throw new Error("speech start message missing") + ctx.fire({ type: "speechToTextStarted", requestId: start.requestId }) ctx.speech.stop({ done: () => done++ }) ctx.speech.cancel() @@ -105,6 +160,7 @@ describe("useSpeechToText", () => { ctx.speech.start({ model: "scribe", insert: (value) => text.push(value) }) const start = ctx.sent[0] if (start?.type !== "speechToTextStart") throw new Error("speech start message missing") + ctx.fire({ type: "speechToTextStarted", requestId: start.requestId }) ctx.speech.stop({ done: () => done++, ready: () => false }) ctx.fire({ type: "speechToTextResult", requestId: start.requestId, text: "Keep as draft" }) diff --git a/packages/kilo-vscode/webview-ui/agent-manager/AgentManagerApp.tsx b/packages/kilo-vscode/webview-ui/agent-manager/AgentManagerApp.tsx index a092686fcb55..1363d949cf6b 100644 --- a/packages/kilo-vscode/webview-ui/agent-manager/AgentManagerApp.tsx +++ b/packages/kilo-vscode/webview-ui/agent-manager/AgentManagerApp.tsx @@ -83,6 +83,7 @@ import { FeedbackProvider } from "../src/context/feedback" import { SessionProvider, useSession } from "../src/context/session" import { WorktreeModeProvider } from "../src/context/worktree-mode" import { ChatView } from "../src/components/chat" +import { SpeechToTextPrewarm } from "../src/components/speech-to-text/SpeechToTextPrewarm" import HistoryView from "../src/components/history/HistoryView" import { NewWorktreeDialog } from "./NewWorktreeDialog" import { DataBridge, MermaidDownloadBridge } from "../src/App" @@ -3131,6 +3132,7 @@ export const AgentManagerApp: Component = () => { + diff --git a/packages/kilo-vscode/webview-ui/diff-viewer/DiffViewerApp.tsx b/packages/kilo-vscode/webview-ui/diff-viewer/DiffViewerApp.tsx index a60338c62c3a..5ab642292e15 100644 --- a/packages/kilo-vscode/webview-ui/diff-viewer/DiffViewerApp.tsx +++ b/packages/kilo-vscode/webview-ui/diff-viewer/DiffViewerApp.tsx @@ -23,6 +23,7 @@ import type { DiffSourceCapabilities, DiffSourceDescriptor } from "../../src/dif import type { DiffViewerNotice } from "../src/types/messages/extension-messages" import { DiffPickerHeader } from "./DiffPickerHeader" import { BaseBranchPicker } from "./BaseBranchPicker" +import { SpeechToTextPrewarm } from "../src/components/speech-to-text/SpeechToTextPrewarm" const NOTICE_KEYS: Record = { "snapshots-disabled": "diffViewer.notice.snapshotsDisabled", @@ -305,6 +306,7 @@ export const DiffViewerApp: Component = () => { + diff --git a/packages/kilo-vscode/webview-ui/src/App.tsx b/packages/kilo-vscode/webview-ui/src/App.tsx index de57030df631..d47cf5d952f6 100644 --- a/packages/kilo-vscode/webview-ui/src/App.tsx +++ b/packages/kilo-vscode/webview-ui/src/App.tsx @@ -25,6 +25,7 @@ import { ChatView } from "./components/chat" import { SidebarEmptyState } from "./components/chat/SidebarEmptyState" import { registerExpandedTaskTool } from "./components/chat/TaskToolExpanded" import { registerVscodeToolOverrides } from "./components/chat/VscodeToolOverrides" +import { SpeechToTextPrewarm } from "./components/speech-to-text/SpeechToTextPrewarm" // Override the upstream "task" tool renderer with the fully-expanded version // that shows child session parts inline in the VS Code sidebar. @@ -378,6 +379,7 @@ const App: Component = () => { + diff --git a/packages/kilo-vscode/webview-ui/src/components/speech-to-text/SpeechToTextButton.tsx b/packages/kilo-vscode/webview-ui/src/components/speech-to-text/SpeechToTextButton.tsx index 3da53666ec70..ade6048443c5 100644 --- a/packages/kilo-vscode/webview-ui/src/components/speech-to-text/SpeechToTextButton.tsx +++ b/packages/kilo-vscode/webview-ui/src/components/speech-to-text/SpeechToTextButton.tsx @@ -1,7 +1,7 @@ import { Button } from "@kilocode/kilo-ui/button" import { Tooltip } from "@kilocode/kilo-ui/tooltip" import { Spinner } from "@kilocode/kilo-ui/spinner" -import type { Component } from "solid-js" +import { onCleanup, type Component } from "solid-js" import type { SpeechToText } from "./useSpeechToText" type Props = { @@ -12,8 +12,10 @@ type Props = { } export const SpeechToTextButton: Component = (props) => { - const disabled = () => !!props.disabled + const unavailable = () => !!props.disabled && props.speech.state() === "idle" + const busy = () => props.speech.state() === "starting" || props.speech.state() === "transcribing" const label = () => { + if (props.speech.state() === "starting") return props.label("speechToText.tooltip.starting") if (props.speech.state() === "recording") return props.label("speechToText.tooltip.stop") if (props.speech.state() === "transcribing") return props.label("speechToText.tooltip.transcribing") if (props.speech.state() === "error") return props.speech.error() || props.label("speechToText.tooltip.error") @@ -21,6 +23,7 @@ export const SpeechToTextButton: Component = (props) => { } const click = () => { + if (props.speech.state() === "starting") return if (props.speech.state() === "recording") { props.speech.stop() return @@ -33,22 +36,28 @@ export const SpeechToTextButton: Component = (props) => { props.speech.clear() return } - if (disabled()) return + if (unavailable()) return props.start() } + onCleanup(() => { + if (props.speech.active()) props.speech.cancel() + }) + return (