diff --git a/.github/workflows/live-host-release.yml b/.github/workflows/live-host-release.yml index 665f5a361e7..365bdb4b6e3 100644 --- a/.github/workflows/live-host-release.yml +++ b/.github/workflows/live-host-release.yml @@ -346,6 +346,45 @@ jobs: gh release create "$FEED_TAG" "${stable_assets[@]}" --title 'Qwen Live Host latest' --notes 'Stable Qwen Live Host installer feed.' --latest=false fi + - name: 'Checkout source for qwen-live npm publish' + uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' + + - name: 'Set up Node.js' + uses: 'actions/setup-node@49933f5360751b6f8e5e4b6c6f3a8c3c5c5c5c5c' + with: + node-version-file: '.nvmrc' + registry-url: 'https://registry.npmjs.org' + + - name: 'Install dependencies' + run: 'npm install --ignore-scripts' + + - name: 'Build qwen-live' + run: 'npm run build --workspace @qwen-code/qwen-live' + + - name: 'Publish @qwen-code/qwen-live' + working-directory: 'packages/qwen-live' + env: + NODE_AUTH_TOKEN: '${{ secrets.NPM_TOKEN }}' + RELEASE_VERSION: '${{ needs.prepare.outputs.version }}' + NPM_TAG: "${{ inputs.prerelease == true && 'preview' || 'latest' }}" + run: | + set -euo pipefail + PACKAGE_NAME="$(node -p "require('./package.json').name")" + # Align the package version with the Host release version. + npm version "$RELEASE_VERSION" --no-git-tag-version --allow-same-version + if npm view "${PACKAGE_NAME}@${RELEASE_VERSION}" version >/dev/null 2>&1; then + echo "::notice::${PACKAGE_NAME}@${RELEASE_VERSION} already published; skipping" + exit 0 + fi + # Check if the package exists at all — provenance requires an + # existing package, so the first publish omits it. + if npm view "${PACKAGE_NAME}" version >/dev/null 2>&1; then + npm publish --access public --provenance --tag "$NPM_TAG" + else + echo "::notice::First publish of ${PACKAGE_NAME}; omitting --provenance" + npm publish --access public --tag "$NPM_TAG" + fi + - name: 'Publish release summary' env: RELEASE_URL: '${{ steps.release.outputs.url }}' diff --git a/.gitignore b/.gitignore index 6db3e0646c8..b1fb3aa1d5a 100644 --- a/.gitignore +++ b/.gitignore @@ -145,3 +145,4 @@ tmp/ # Brand build workspaces (created by the desktop-brand-builder skill) brand-builds/ +workspace/ diff --git a/integration-tests/qwen-live-harness.ts b/integration-tests/qwen-live-harness.ts index 4c7c93f2c0c..a079955128b 100644 --- a/integration-tests/qwen-live-harness.ts +++ b/integration-tests/qwen-live-harness.ts @@ -72,7 +72,7 @@ const SERVE_TOKEN = 'qwen-live-e2e-token'; const LIVE_LISTENING_RE = /qwen-live listening on http:\/\/127\.0\.0\.1:(\d+)/; const DISPOSE_GRACE_MS = 10_000; const LIVE_HOST_BUNDLE_ID = 'com.alibaba.qwen-code.live-host'; -const LIVE_HOST_PROTOCOL_VERSION = 6; +const LIVE_HOST_PROTOCOL_VERSION = 7; const LIVE_INPUT_AUDIO_EPOCH_BYTES = 8; // -- small async utilities ---------------------------------------------------- diff --git a/integration-tests/qwen-live-m1-call.test.ts b/integration-tests/qwen-live-m1-call.test.ts index 8fc8c2c1c86..2456af610fd 100644 --- a/integration-tests/qwen-live-m1-call.test.ts +++ b/integration-tests/qwen-live-m1-call.test.ts @@ -88,7 +88,7 @@ describeE2E('qwen-live M1 — end-to-end voice call', () => { expect(record['url']).toBe(stack.live.url); expect(typeof record['token']).toBe('string'); expect(String(record['token']).length).toBeGreaterThan(0); - expect(record['protocolVersion']).toBe(6); + expect(record['protocolVersion']).toBe(7); expect(record['pid']).toBe(stack.live.proc.pid); expect(String(record['instanceNonce'])).toMatch(/^[A-Za-z0-9_-]{16,256}$/); }); diff --git a/package-lock.json b/package-lock.json index 8ea68ca4bfb..610540dc8fb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31289,8 +31289,10 @@ "version": "0.1.0", "license": "Apache-2.0", "dependencies": { + "@agentclientprotocol/sdk": "^0.14.1", "@qwen-code/sdk": "file:../sdk-typescript", "ansi-regex": "^6.2.2", + "prompts": "^2.4.2", "proper-lockfile": "^4.1.2", "ws": "^8.18.0" }, @@ -31299,6 +31301,7 @@ }, "devDependencies": { "@types/node": "^22.13.10", + "@types/prompts": "^2.4.9", "@types/proper-lockfile": "^4.1.4", "@types/ws": "^8.5.0", "typescript": "^5.3.3", diff --git a/packages/cli/src/serve/live/live-host-coordinator.ts b/packages/cli/src/serve/live/live-host-coordinator.ts index c59f56a9dd2..7c99dc1bfd4 100644 --- a/packages/cli/src/serve/live/live-host-coordinator.ts +++ b/packages/cli/src/serve/live/live-host-coordinator.ts @@ -292,6 +292,18 @@ function parseHostMessage(text: string): LiveHostMessage | undefined { }; } } + if ( + value['type'] === 'host.playback_started' && + typeof value['epoch'] === 'number' + ) { + return { type: 'host.playback_started', epoch: value['epoch'] }; + } + if ( + value['type'] === 'host.playback_completed' && + typeof value['epoch'] === 'number' + ) { + return { type: 'host.playback_completed', epoch: value['epoch'] }; + } return undefined; } @@ -983,6 +995,15 @@ export class LiveHostCoordinator { this.handleShortcutResult(message); return; } + // v7 playback receipts: accepted but not forwarded to the built-in + // Live session coordinator (which uses byte estimation). The + // standalone qwen-live daemon wires these to its injector. + if ( + message.type === 'host.playback_started' || + message.type === 'host.playback_completed' + ) { + return; + } this.handleAction(message); } diff --git a/packages/cli/src/serve/live/types.ts b/packages/cli/src/serve/live/types.ts index 0d01e7228a4..3eb0b12be58 100644 --- a/packages/cli/src/serve/live/types.ts +++ b/packages/cli/src/serve/live/types.ts @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -export const LIVE_HOST_PROTOCOL_VERSION = 6 as const; +export const LIVE_HOST_PROTOCOL_VERSION = 7 as const; export const LIVE_HOST_BUNDLE_ID = 'com.alibaba.qwen-code.live-host' as const; export const LIVE_INPUT_AUDIO_EPOCH_BYTES = 8; @@ -149,12 +149,24 @@ export type LiveHostScreenContextResult = error: string; }; +export interface LiveHostPlaybackStarted { + type: 'host.playback_started'; + epoch: number; +} + +export interface LiveHostPlaybackCompleted { + type: 'host.playback_completed'; + epoch: number; +} + export type LiveHostMessage = | LiveHostHello | LiveHostAction | LiveHostPong | LiveHostShortcutResult - | LiveHostScreenContextResult; + | LiveHostScreenContextResult + | LiveHostPlaybackStarted + | LiveHostPlaybackCompleted; export type LiveDaemonMessage = | { diff --git a/packages/live-host/src/main/__tests__/protocol.test.ts b/packages/live-host/src/main/__tests__/protocol.test.ts index 4be7e44fd92..a65fe2a1ac4 100644 --- a/packages/live-host/src/main/__tests__/protocol.test.ts +++ b/packages/live-host/src/main/__tests__/protocol.test.ts @@ -142,7 +142,7 @@ describe('Live Host protocol', () => { /LIVE_HOST_BUNDLE_ID = '([^']+)'/u, )?.[1]; - assert.equal(LIVE_PROTOCOL_VERSION, 6); + assert.equal(LIVE_PROTOCOL_VERSION, 7); assert.equal(daemonVersion, LIVE_PROTOCOL_VERSION); assert.equal(daemonBundleId, LIVE_HOST_BUNDLE_ID); assert.equal(Object.values(PROTOCOL_TYPE_PARITY).every(Boolean), true); diff --git a/packages/live-host/src/main/daemon-connection.ts b/packages/live-host/src/main/daemon-connection.ts index 91e13ca8164..e64404ebf81 100644 --- a/packages/live-host/src/main/daemon-connection.ts +++ b/packages/live-host/src/main/daemon-connection.ts @@ -458,6 +458,14 @@ export class LiveDaemonConnection { return true; } + sendPlaybackStarted(epoch: number): boolean { + return this.sendControl({ type: 'host.playback_started', epoch }); + } + + sendPlaybackCompleted(epoch: number): boolean { + return this.sendControl({ type: 'host.playback_completed', epoch }); + } + private async captureScreenContext( requestId: string, epoch: number, diff --git a/packages/live-host/src/main/index.ts b/packages/live-host/src/main/index.ts index 77bd77466cc..63b5deab999 100644 --- a/packages/live-host/src/main/index.ts +++ b/packages/live-host/src/main/index.ts @@ -575,6 +575,30 @@ function registerIpc(): void { }); publishState(); }); + ipcMain.on('live:audio:playback-started', (event, epoch: unknown) => { + if ( + !isTrustedSender(event) || + typeof epoch !== 'number' || + !Number.isSafeInteger(epoch) || + epoch !== daemon.getEpoch() + ) { + return; + } + daemon.sendPlaybackStarted(epoch); + }); + + ipcMain.on('live:audio:playback-completed', (event, epoch: unknown) => { + if ( + !isTrustedSender(event) || + typeof epoch !== 'number' || + !Number.isSafeInteger(epoch) || + epoch !== daemon.getEpoch() + ) { + return; + } + daemon.sendPlaybackCompleted(epoch); + }); + ipcMain.handle('live:set-output-muted', (event, muted: unknown) => { if (!isTrustedSender(event) || typeof muted !== 'boolean') return; const inputMuted = live.inputMuted ?? false; @@ -970,12 +994,13 @@ void app.whenReady().then(() => { }, onOutputAudio: (audio) => { if (nativeServicesActive && !live.outputMuted) { - appendHostAudio(audio, daemon.getEpoch()); + const epoch = daemon.getEpoch(); + appendHostAudio(audio, epoch); writeLiveDiagnostic('output_frame_received', { - epoch: daemon.getEpoch(), + epoch, bytes: audio.byteLength, }); - sendAudioCommand('live:audio:play', audio); + sendAudioCommand('live:audio:play', { audio, epoch }); } }, onClearOutput: () => { diff --git a/packages/live-host/src/preload/audio-engine.ts b/packages/live-host/src/preload/audio-engine.ts index ca533cd3822..5089fe5b34a 100644 --- a/packages/live-host/src/preload/audio-engine.ts +++ b/packages/live-host/src/preload/audio-engine.ts @@ -56,6 +56,8 @@ export class HostAudioEngine { event: string, details: AudioDiagnosticDetails, ) => void = () => {}, + private readonly onPlaybackStarted: () => void = () => {}, + private readonly onPlaybackCompleted: () => void = () => {}, ) {} private readonly handleDeviceChange = (): void => { @@ -255,10 +257,23 @@ export class HostAudioEngine { currentGeneration: this.outputGeneration, remainingSources: this.outputSources.size, }); + // Only fire completion for a natural end (generation matches); + // clearOutput increments generation before stopping sources, + // so a stop-triggered onended sees a mismatch and stays silent. + if ( + this.outputSources.size === 0 && + generation === this.outputGeneration + ) { + this.onPlaybackCompleted(); + } }; + const wasEmpty = this.outputSources.size === 0; this.outputSources.add(source); source.start(schedule.startAt); this.outputCursor = schedule.endAt; + if (wasEmpty) { + this.onPlaybackStarted(); + } this.onDiagnostic('output_frame_scheduled', { bytes: frame.byteLength, generation, diff --git a/packages/live-host/src/preload/index.ts b/packages/live-host/src/preload/index.ts index d5fbb36ad71..b4a98b528d6 100644 --- a/packages/live-host/src/preload/index.ts +++ b/packages/live-host/src/preload/index.ts @@ -14,6 +14,16 @@ const audio = new HostAudioEngine( ipcRenderer.send('live:audio:diagnostic', { event, details }); } }, + () => { + if (currentPlaybackEpoch !== undefined) { + ipcRenderer.send('live:audio:playback-started', currentPlaybackEpoch); + } + }, + () => { + if (currentPlaybackEpoch !== undefined) { + ipcRenderer.send('live:audio:playback-completed', currentPlaybackEpoch); + } + }, ); const invoke = (channel: string, ...args: unknown[]): Promise => @@ -85,14 +95,19 @@ ipcRenderer.on( ipcRenderer.on('live:audio:set-output-muted', (_event, muted: boolean) => { audio.setOutputMuted(muted); }); -ipcRenderer.on('live:audio:play', (_event, frame: Uint8Array) => { - void audio.play(frame).catch(() => { - audio.clearOutput(); - ipcRenderer.send('live:audio:output-error', { - code: 'audio_output_unavailable', +let currentPlaybackEpoch: number | undefined; +ipcRenderer.on( + 'live:audio:play', + (_event, payload: { audio: Uint8Array; epoch: number }) => { + currentPlaybackEpoch = payload.epoch; + void audio.play(payload.audio).catch(() => { + audio.clearOutput(); + ipcRenderer.send('live:audio:output-error', { + code: 'audio_output_unavailable', + }); }); - }); -}); + }, +); ipcRenderer.on('live:audio:clear', () => audio.clearOutput()); let lastPointerInteractive = false; diff --git a/packages/live-host/src/shared/protocol.ts b/packages/live-host/src/shared/protocol.ts index d2dd1e438f3..068ec8d9f9d 100644 --- a/packages/live-host/src/shared/protocol.ts +++ b/packages/live-host/src/shared/protocol.ts @@ -1,4 +1,4 @@ -export const LIVE_PROTOCOL_VERSION = 6; +export const LIVE_PROTOCOL_VERSION = 7; export const LIVE_HOST_BUNDLE_ID = 'com.alibaba.qwen-code.live-host'; export const MAX_CONTROL_FRAME_BYTES = 64 * 1024; export const MAX_INPUT_AUDIO_FRAME_BYTES = 64 * 1024; @@ -112,7 +112,9 @@ export type HostControlMessage = requestId: string; success: false; error: string; - }; + } + | { type: 'host.playback_started'; epoch: number } + | { type: 'host.playback_completed'; epoch: number }; export type DaemonControlMessage = | { diff --git a/packages/qwen-live/README.md b/packages/qwen-live/README.md index cd1a8f29902..87f5ae8910f 100644 --- a/packages/qwen-live/README.md +++ b/packages/qwen-live/README.md @@ -1,47 +1,118 @@ # @qwen-code/qwen-live Standalone Live voice daemon: a realtime voice control plane that orchestrates -Qwen Code sessions. +coding sessions through voice. `qwen-live` connects three parties: -- **Qwen Live Host** (the macOS overlay app) over the Live Host WebSocket - protocol v6 — the same wire protocol and `~/.qwen/live/daemon.json` discovery - file the built-in `qwen serve` Live integration uses, so an already-installed - Host binary connects without changes. +- **Live Host** (the macOS overlay app) over the Live Host WebSocket + protocol v7 — writes `~/.qwen/live/daemon.json` for discovery, so an + already-installed Host connects automatically. - **A DashScope realtime voice model** (`qwen-omni` realtime) that owns the - conversation: VAD, direct answers, and a small tool surface for dispatching - work. -- **Qwen Code sessions** through a `BackendAdaptor`. The first adaptor drives a - running `qwen serve` daemon over its REST/SSE surface. + conversation: VAD, direct answers, and a tool surface for dispatching work + to coding sessions. +- **Coding sessions** through a `BackendAdaptor`. Two adaptors are available: + one drives `qwen serve` over REST/SSE, and one spawns any ACP-compatible + agent (`qwen --acp`, `qodercli --acp`, `gemini --acp`, etc.) as a child + process over JSON-RPC stdio. Multiple backends can coexist with per-session + routing. The live session itself is fully owned by this daemon (JSONL logs under -`~/.qwen-live/sessions/`); backend sessions are ordinary Qwen Code sessions +`~/.qwen-live/sessions/`); backend sessions are ordinary coding sessions that keep running after a call ends. -## Run +## Quick Start + +### 1. Install + +```bash +# From the qwen-code monorepo +cd qwen-code +npm install && npm run build +``` + +### 2. Run the setup wizard + +```bash +node packages/qwen-live/dist/index.js init +``` + +The wizard will: + +- Scan your PATH for installed coding agents (qodercli, qwen, gemini, + claude, codex) and list what it found +- Let you pick a default backend and add additional ones +- Ask for your DashScope realtime API key +- Set a default working directory for coding sessions +- On macOS: check if the Live Host app is installed and offer to install it + +When done, it writes `~/.qwen-live/config.json` and tells you to run +`qwen-live` to start. + +### 3. Start the daemon ```bash -# Requires a running `qwen serve` and a DashScope API key. -DASHSCOPE_API_KEY=sk-... qwen-live +qwen-live +# → qwen-live listening on http://127.0.0.1: ``` -Configuration comes from environment variables (`QWEN_LIVE_*`, -`DASHSCOPE_API_KEY`, `QWEN_SERVER_TOKEN`) with `~/.qwen-live/config.json` as -the file-based fallback. See `src/config.ts` for the full list. +On macOS, open the Live Host app — it reads the discovery file and connects +automatically. Press the global shortcut to start a voice call. -Only one Live daemon may own the Host discovery file at a time. If the -built-in `qwen serve` Live integration is enabled and running, `qwen-live` -fails fast at startup instead of taking over. +On other platforms: the Live Host app is macOS-only (it needs native +microphone, global shortcut, and screen capture). Linux/Windows users cannot +use voice features until a Host is available on their platform. -## Host bootstrap +## Configuration -The Live Host installer is ported: `GET /live/setup` reports the installed -Host's state, `POST /live/setup/install` downloads, verifies (sha256 + -codesign + team identifier), and installs the latest Host release, and -`POST /live/setup/launch` opens it. macOS only (the same surface `qwen -serve` exposes). All three sit behind the daemon's Bearer token; a browser -context (Origin header) is always refused. +Configuration comes from `~/.qwen-live/config.json` (generated by `init`), +with environment variables (`DASHSCOPE_API_KEY`, `QWEN_LIVE_*`) as overrides. + +```jsonc +{ + "realtimeApiKey": "sk-...", + "defaultCwd": "~/work/my-project", + "backends": [ + { + "name": "qodercli", + "kind": "acp", + "command": "/usr/local/bin/qodercli", + "args": ["--acp"], + "default": true, + }, + { + "name": "qwen", + "kind": "acp", + "command": "/usr/bin/qwen", + "args": ["--acp"], + }, + ], +} +``` + +See `src/config.ts` for the full list of options and validation rules. + +### Supported backends + +| Backend | Kind | ACP entry | Notes | +| ----------- | ----------- | ------------------------------------------- | ----------------------------------- | +| Qoder CLI | `acp` | `qodercli --acp` | Hidden flag; uses Qoder's own login | +| Qwen Code | `acp` | `qwen --acp` | Native ACP mode | +| Gemini CLI | `acp` | `gemini --experimental-acp` | Official ACP support | +| Claude Code | `acp` | `npx @agentclientprotocol/claude-agent-acp` | Adapter-based | +| Codex | `acp` | `npx @agentclientprotocol/codex-acp` | Adapter-based | +| qwen serve | `qwen-code` | REST/SSE to `qwen serve` daemon | Legacy; no ACP needed | + +Multiple backends can coexist — the voice model sees all sessions across +all backends in `session_list` and can route `handoff` to a specific one by +name. + +## Host Bootstrap + +The Live Host installer is built in. On macOS, `qwen-live init` checks if +the Host is installed and offers to download and install it (sha256 + +codesign + team identifier verification). The daemon also exposes HTTP +endpoints behind its Bearer token: ```bash TOKEN=$(jq -r .token ~/.qwen/live/daemon.json) @@ -49,9 +120,27 @@ PORT=$(jq -r .url ~/.qwen/live/daemon.json | sed 's/.*://') curl -H "authorization: Bearer $TOKEN" "http://127.0.0.1:$PORT/live/setup" ``` +## Protocol v7: Playback Receipts + +The daemon speaks Live Host protocol v7, which adds **playback receipts**: +the Host app sends `host.playback_started` and `host.playback_completed` +messages when audio actually starts and finishes playing. The injector uses +these real signals instead of estimating playback duration from byte counts. + +A v6 Host still connects (the daemon falls back to a simpler playback model +without byte estimation), but v7 is required for the full "dual-barrier +delivery confirmation" described in the roadmap. + ## Status -Incubating inside the qwen-code monorepo, tracking M1+M2 (plus the M1 -installer, now ported) of the Live split roadmap (issue #10118). The Host app itself, non-qwen-code adaptors (M4), and retiring -the built-in Live integration (M5) are tracked in later milestones of -issue #10118. +Incubating inside the qwen-code monorepo, tracking M1–M5 of the Live split +roadmap (issue #10118): + +- **M1+M2** (merged): daemon, host stack, 7 tools, injector, permissions, + steering, JSONL logs, Host installer +- **M4** (merged): AcpAdaptor, multi-backend routing, capability gating +- **M5** (this PR): protocol v7 playback receipts, `qwen-live init` wizard +- **M3** (blocked): session registry + cross-session messaging — depends on + upstream #9576 +- Built-in Live module retirement: deferred until the standalone daemon is + stable in production diff --git a/packages/qwen-live/package.json b/packages/qwen-live/package.json index 617363c0ea8..7ee2f952381 100644 --- a/packages/qwen-live/package.json +++ b/packages/qwen-live/package.json @@ -41,6 +41,7 @@ "@agentclientprotocol/sdk": "^0.14.1", "@qwen-code/sdk": "file:../sdk-typescript", "ansi-regex": "^6.2.2", + "prompts": "^2.4.2", "proper-lockfile": "^4.1.2", "ws": "^8.18.0" }, @@ -49,7 +50,8 @@ "@types/proper-lockfile": "^4.1.4", "@types/ws": "^8.5.0", "typescript": "^5.3.3", - "vitest": "^3.1.1" + "vitest": "^3.1.1", + "@types/prompts": "^2.4.9" }, "bugs": { "url": "https://github.com/QwenLM/qwen-code/issues" diff --git a/packages/qwen-live/src/agent-detector.ts b/packages/qwen-live/src/agent-detector.ts new file mode 100644 index 00000000000..795e3cec339 --- /dev/null +++ b/packages/qwen-live/src/agent-detector.ts @@ -0,0 +1,178 @@ +/** + * @license + * Copyright 2026 Qwen + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Detects coding agents on the user's PATH that qwen-live can drive as ACP + * backends. Each agent is probed with `--version` (cheap, non-blocking); + * the ACP handshake itself is left to the daemon's preflight. + */ + +import { spawnSync } from 'node:child_process'; +import { accessSync, statSync } from 'node:fs'; +import { delimiter, join } from 'node:path'; + +export interface DetectedAgent { + /** Display name for the prompt. */ + label: string; + /** Backend name for config.json (matches the name pattern). */ + name: string; + /** Resolved command to spawn the agent. */ + command: string; + /** Args to enter ACP mode. */ + args: string[]; + /** Detected version string (may be empty). */ + version: string; + /** Environment variables the agent needs (e.g. CODEX_PATH). */ + env?: Record; +} + +/** Agent definitions: how to detect + how to launch in ACP mode. */ +const AGENT_SPECS: ReadonlyArray<{ + label: string; + name: string; + binary: string; + acpArgs: string[]; + /** If true, the agent itself is the ACP server. If false, use npx adapter. */ + native: boolean; + /** Adapter package for non-native agents. */ + adapterPackage?: string; + /** Env to pass through for non-native agents. */ + envKey?: string; +}> = [ + { + label: 'Qoder CLI', + name: 'qodercli', + binary: 'qodercli', + acpArgs: ['--acp'], + native: true, + }, + { + label: 'Qwen Code', + name: 'qwen', + binary: 'qwen', + acpArgs: ['--acp'], + native: true, + }, + { + label: 'Gemini CLI', + name: 'gemini', + binary: 'gemini', + acpArgs: ['--experimental-acp'], + native: true, + }, + { + label: 'Claude Code', + name: 'claude', + binary: 'claude', + acpArgs: ['-y', '@agentclientprotocol/claude-agent-acp'], + native: false, + adapterPackage: '@agentclientprotocol/claude-agent-acp', + }, + { + label: 'Codex', + name: 'codex', + binary: 'codex', + acpArgs: ['-y', '@agentclientprotocol/codex-acp'], + native: false, + adapterPackage: '@agentclientprotocol/codex-acp', + envKey: 'CODEX_PATH', + }, +]; + +/** + * Resolve a command name to an absolute path on the PATH. + * Returns undefined if not found. Supports PATHEXT on Windows. + */ +function findExecutable(command: string): string | undefined { + // Absolute or relative path — return as-is if it exists. + if (command.includes('/') || command.includes('\\')) { + try { + accessSync(command); + return command; + } catch { + return undefined; + } + } + const pathEnv = process.env['PATH'] ?? ''; + const pathExt = process.env['PATHEXT']; + const extensions = pathExt ? pathExt.split(';') : ['']; + for (const dir of pathEnv.split(delimiter)) { + if (!dir) continue; + for (const ext of extensions) { + const candidate = join(dir, command + ext); + try { + const stat = statSync(candidate); + if (stat.isFile()) return candidate; + } catch { + /* not found, continue */ + } + } + } + return undefined; +} + +/** + * Probe a binary with `--version` (3s timeout). Returns the trimmed + * stdout, or undefined if the probe failed. + */ +function probeVersion(binary: string): string | undefined { + const resolved = findExecutable(binary); + if (!resolved) return undefined; + try { + const result = spawnSync(resolved, ['--version'], { + encoding: 'utf8', + timeout: 3_000, + windowsHide: true, + }); + if (result.status === 0) { + return (result.stdout || result.stderr || '').trim().split('\n')[0]; + } + // Some CLIs exit non-zero on --version but still print to stderr. + const fallback = (result.stderr || '').trim().split('\n')[0]; + return fallback || undefined; + } catch { + return undefined; + } +} + +/** + * Detect all supported coding agents installed on the current machine. + * Returns an array of DetectedAgent, ordered by the spec priority + * (qodercli first, then qwen, gemini, claude, codex). + */ +export function detectAgents(): DetectedAgent[] { + const found: DetectedAgent[] = []; + for (const spec of AGENT_SPECS) { + const version = probeVersion(spec.binary); + if (!version) continue; + if (spec.native) { + const resolved = findExecutable(spec.binary); + if (!resolved) continue; + found.push({ + label: spec.label, + name: spec.name, + command: resolved, + args: spec.acpArgs, + version, + }); + } else { + // Non-native: use npx to run the adapter. The binary itself + // (claude/codex) must be on PATH for the adapter to find it. + const resolved = findExecutable(spec.binary); + if (!resolved) continue; + const npx = findExecutable('npx') ?? 'npx'; + found.push({ + label: spec.label, + name: spec.name, + command: npx, + args: spec.acpArgs, + version, + ...(spec.envKey ? { env: { [spec.envKey]: resolved } } : {}), + }); + } + } + return found; +} diff --git a/packages/qwen-live/src/daemon.ts b/packages/qwen-live/src/daemon.ts index 90a13fe9d0d..9faaac70083 100644 --- a/packages/qwen-live/src/daemon.ts +++ b/packages/qwen-live/src/daemon.ts @@ -155,6 +155,8 @@ export class LiveDaemon { onStart: (call) => session.start(call), onStop: (call) => session.stop(call), onInputAudio: (call) => session.pushAudio(call), + onPlaybackStarted: (call) => session.notePlaybackStarted(call), + onPlaybackCompleted: (call) => session.notePlaybackCompleted(call), }); const port = await this.listen(); diff --git a/packages/qwen-live/src/host/live-host-coordinator.ts b/packages/qwen-live/src/host/live-host-coordinator.ts index 6c68b749998..9cf9332814b 100644 --- a/packages/qwen-live/src/host/live-host-coordinator.ts +++ b/packages/qwen-live/src/host/live-host-coordinator.ts @@ -16,6 +16,8 @@ import { type LiveHostHello, type LiveHostShortcutResult, type LiveHostScreenContextResult, + type LiveHostPlaybackStarted, + type LiveHostPlaybackCompleted, type LiveHostStatus, type LiveHostMessage, type LiveMuteUpdate, @@ -103,6 +105,8 @@ export interface LiveCallHandlers { callId: string; pcm16: Buffer; }) => boolean; + onPlaybackStarted?: (call: { epoch: number }) => void; + onPlaybackCompleted?: (call: { epoch: number }) => void; } export interface LiveHostCoordinatorOptions { @@ -297,6 +301,18 @@ function parseHostMessage(text: string): LiveHostMessage | undefined { }; } } + if ( + value['type'] === 'host.playback_started' && + typeof value['epoch'] === 'number' + ) { + return { type: 'host.playback_started', epoch: value['epoch'] }; + } + if ( + value['type'] === 'host.playback_completed' && + typeof value['epoch'] === 'number' + ) { + return { type: 'host.playback_completed', epoch: value['epoch'] }; + } return undefined; } @@ -996,6 +1012,14 @@ export class LiveHostCoordinator { this.handleShortcutResult(message); return; } + if (message.type === 'host.playback_started') { + this.handlePlaybackStarted(message); + return; + } + if (message.type === 'host.playback_completed') { + this.handlePlaybackCompleted(message); + return; + } this.handleAction(message); } @@ -1093,6 +1117,18 @@ export class LiveHostCoordinator { this.sendState(status); } + private handlePlaybackStarted(message: LiveHostPlaybackStarted): void { + const call = this.call; + if (!call || call.epoch !== message.epoch) return; + this.handlers.onPlaybackStarted?.({ epoch: call.epoch }); + } + + private handlePlaybackCompleted(message: LiveHostPlaybackCompleted): void { + const call = this.call; + if (!call || call.epoch !== message.epoch) return; + this.handlers.onPlaybackCompleted?.({ epoch: call.epoch }); + } + private handleAction(action: LiveHostAction): void { if ( action.epoch !== undefined && diff --git a/packages/qwen-live/src/host/types.ts b/packages/qwen-live/src/host/types.ts index 0d01e7228a4..3eb0b12be58 100644 --- a/packages/qwen-live/src/host/types.ts +++ b/packages/qwen-live/src/host/types.ts @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -export const LIVE_HOST_PROTOCOL_VERSION = 6 as const; +export const LIVE_HOST_PROTOCOL_VERSION = 7 as const; export const LIVE_HOST_BUNDLE_ID = 'com.alibaba.qwen-code.live-host' as const; export const LIVE_INPUT_AUDIO_EPOCH_BYTES = 8; @@ -149,12 +149,24 @@ export type LiveHostScreenContextResult = error: string; }; +export interface LiveHostPlaybackStarted { + type: 'host.playback_started'; + epoch: number; +} + +export interface LiveHostPlaybackCompleted { + type: 'host.playback_completed'; + epoch: number; +} + export type LiveHostMessage = | LiveHostHello | LiveHostAction | LiveHostPong | LiveHostShortcutResult - | LiveHostScreenContextResult; + | LiveHostScreenContextResult + | LiveHostPlaybackStarted + | LiveHostPlaybackCompleted; export type LiveDaemonMessage = | { diff --git a/packages/qwen-live/src/index.ts b/packages/qwen-live/src/index.ts index b94ae67d9ea..8a832caa434 100644 --- a/packages/qwen-live/src/index.ts +++ b/packages/qwen-live/src/index.ts @@ -14,6 +14,7 @@ import { realpathSync, statSync } from 'node:fs'; import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; import { loadConfig } from './config.js'; +import { runInit } from './init.js'; import { LiveDaemon } from './daemon.js'; import { LiveLogger } from './logger.js'; @@ -109,5 +110,11 @@ if (process.argv[1] !== undefined) { } } if (invokedDirectly) { - void main(); + // Subcommand dispatch: `qwen-live init` runs the setup wizard, + // everything else starts the daemon. + if (process.argv[2] === 'init') { + void runInit(); + } else { + void main(); + } } diff --git a/packages/qwen-live/src/init.ts b/packages/qwen-live/src/init.ts new file mode 100644 index 00000000000..5ad04efc305 --- /dev/null +++ b/packages/qwen-live/src/init.ts @@ -0,0 +1,256 @@ +/** + * @license + * Copyright 2026 Qwen + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * `qwen-live init` — interactive setup wizard. + * + * Scans the user's machine for supported coding agents, lets them pick a + * default backend, collects their DashScope API key, checks/installs the + * Live Host app, and writes ~/.qwen-live/config.json. + */ + +/* eslint-disable no-console -- this is a CLI wizard; console is its UI */ + +import { homedir } from 'node:os'; +import { join } from 'node:path'; +import { + existsSync, + mkdirSync, + readFileSync, + renameSync, + writeFileSync, +} from 'node:fs'; +import prompts from 'prompts'; +import { detectAgents, type DetectedAgent } from './agent-detector.js'; +import { LiveHostInstaller } from './host/live-host-installer.js'; + +const CONFIG_DIR = join(homedir(), '.qwen-live'); +const CONFIG_PATH = join(CONFIG_DIR, 'config.json'); + +interface RawBackend { + name: string; + kind: 'acp'; + command: string; + args: string[]; + env?: Record; + cwd?: string; + default?: boolean; +} + +interface RawConfig { + realtimeApiKey?: string; + realtimeEndpoint?: string; + realtimeModel?: string; + voice?: string; + defaultCwd?: string; + backends?: RawBackend[]; + port?: number; +} + +export async function runInit(): Promise { + console.log('\n qwen-live setup\n ================\n'); + + // 1. Check existing config + if (existsSync(CONFIG_PATH)) { + const existing = readFileSync(CONFIG_PATH, 'utf8'); + const overwrite = await prompts({ + type: 'confirm', + name: 'value', + message: 'A config.json already exists. Overwrite?', + initial: false, + }); + if (!overwrite.value) { + console.log('\n Keeping existing config. Run `qwen-live` to start.\n'); + return; + } + void existing; // suppress unused + } + + // 2. Scan for agents + console.log(' Scanning for installed coding agents...\n'); + const agents = detectAgents(); + if (agents.length === 0) { + console.log(' No supported coding agents found on your PATH.'); + console.log( + ' Install at least one of: qodercli, qwen, gemini, claude, codex\n', + ); + console.log( + ' You can create ~/.qwen-live/config.json manually instead.\n', + ); + return; + } + for (const agent of agents) { + console.log(` ✓ ${agent.label} (${agent.version})`); + } + console.log(); + + // 3. Select default backend + const defaultChoice = await prompts({ + type: 'select', + name: 'value', + message: 'Which agent should be the default backend?', + choices: agents.map((agent) => ({ + title: `${agent.label} (${agent.version})`, + value: agent.name, + })), + initial: 0, + }); + if (defaultChoice.value === undefined) { + console.log('\n Cancelled.\n'); + return; + } + + // 4. Add additional backends + const backends: RawBackend[] = []; + const remaining = agents.filter((a) => a.name !== defaultChoice.value); + let addMore = remaining.length > 0; + const available = [...remaining]; + while (addMore && available.length > 0) { + const more = await prompts({ + type: 'confirm', + name: 'value', + message: `Add another backend? (${available.length} remaining)`, + initial: false, + }); + if (!more.value) { + addMore = false; + break; + } + const pick = await prompts({ + type: 'select', + name: 'value', + message: 'Which agent?', + choices: available.map((agent) => ({ + title: `${agent.label} (${agent.version})`, + value: agent.name, + })), + initial: 0, + }); + if (pick.value !== undefined) { + const agent = available.find((a) => a.name === pick.value)!; + backends.push(toRawBackend(agent, false)); + const idx = available.indexOf(agent); + if (idx !== -1) available.splice(idx, 1); + } + } + + // Build the default backend + const defaultAgent = agents.find((a) => a.name === defaultChoice.value)!; + backends.unshift(toRawBackend(defaultAgent, true)); + + // 5. API key + const envKey = + process.env['DASHSCOPE_API_KEY'] ?? + process.env['QWEN_LIVE_REALTIME_API_KEY']; + let apiKey: string | undefined; + if (envKey) { + const useEnv = await prompts({ + type: 'confirm', + name: 'value', + message: `Use DASHSCOPE_API_KEY from environment (${envKey.slice(0, 8)}...)?`, + initial: true, + }); + if (useEnv.value) { + apiKey = envKey; + } + } + if (!apiKey) { + const keyPrompt = await prompts({ + type: 'password', + name: 'value', + message: 'DashScope realtime API key (sk-...):', + validate: (val: string) => + val.trim().length > 0 || 'Please enter your API key', + }); + apiKey = keyPrompt.value?.trim(); + } + if (!apiKey) { + console.log('\n Cancelled — API key is required.\n'); + return; + } + + // 6. Working directory + const cwdPrompt = await prompts({ + type: 'text', + name: 'value', + message: 'Default working directory for coding sessions:', + initial: process.cwd(), + }); + const defaultCwd = cwdPrompt.value || process.cwd(); + + // 7. Host app (macOS only) + let hostStatus = 'skipped'; + if (process.platform === 'darwin') { + console.log('\n Checking Live Host app...'); + const installer = new LiveHostInstaller(); + const status = await installer.refresh(); + if (status.state === 'installed') { + console.log(` ✓ Live Host ${status.version} is installed.`); + hostStatus = 'installed'; + } else if (status.state === 'missing') { + const install = await prompts({ + type: 'confirm', + name: 'value', + message: 'Live Host is not installed. Install now?', + initial: true, + }); + if (install.value) { + console.log(' Installing Live Host (this may take a minute)...'); + const result = await installer.ensureInstalled(); + if (result.state === 'installed') { + console.log(` ✓ Live Host ${result.version} installed.`); + hostStatus = 'installed'; + } else { + console.log( + ` ✗ Installation failed: ${result.message ?? 'unknown error'}`, + ); + hostStatus = 'failed'; + } + } else { + hostStatus = 'skipped'; + } + } else { + console.log(` ! Host check failed: ${status.message ?? 'unknown'}`); + hostStatus = 'error'; + } + } else { + console.log( + '\n Live Host app is macOS-only. Voice features require a Mac.', + ); + hostStatus = 'unsupported'; + } + + // 8. Write config + const config: RawConfig = { + realtimeApiKey: apiKey, + defaultCwd, + backends, + }; + + mkdirSync(CONFIG_DIR, { recursive: true }); + const tmpPath = CONFIG_PATH + '.tmp'; + writeFileSync(tmpPath, JSON.stringify(config, null, 2) + '\n', { + mode: 0o600, + }); + renameSync(tmpPath, CONFIG_PATH); + + // 9. Done + console.log(`\n ✓ Config written to ${CONFIG_PATH}`); + console.log(` ✓ Default backend: ${defaultAgent.label}`); + console.log(` ✓ Host: ${hostStatus}`); + console.log('\n Run `qwen-live` to start the daemon.\n'); +} + +function toRawBackend(agent: DetectedAgent, isDefault: boolean): RawBackend { + return { + name: agent.name, + kind: 'acp', + command: agent.command, + args: agent.args, + ...(Object.keys(agent.env ?? {}).length > 0 ? { env: agent.env } : {}), + ...(isDefault ? { default: true } : {}), + }; +} diff --git a/packages/qwen-live/src/orchestrator/injector.test.ts b/packages/qwen-live/src/orchestrator/injector.test.ts index 56e785d2724..e07ac9ac58d 100644 --- a/packages/qwen-live/src/orchestrator/injector.test.ts +++ b/packages/qwen-live/src/orchestrator/injector.test.ts @@ -101,17 +101,18 @@ describe('Injector window conditions', () => { expect(injector.pendingCount).toBe(0); }); - it('reports estimated playback pending when user speech starts', () => { - injector.noteOutputAudio(48_000); + it('reports playback in progress when user speech starts', () => { + injector.notePlaybackStarted(); expect(injector.noteSpeechStarted()).toBe(true); injector.noteOutputCleared(); expect(injector.noteSpeechStarted()).toBe(false); }); - it('drops the old quiet gap when speech starts after playback ends', () => { - injector.noteOutputAudio(48_000); - vi.advanceTimersByTime(1_100); + it('drops the quiet gap when speech starts after playback completes', () => { + injector.notePlaybackStarted(); + injector.notePlaybackCompleted(); + vi.advanceTimersByTime(100); expect(injector.noteSpeechStarted()).toBe(false); injector.enqueue(complete('arrived while speaking', 'Result ready.')); @@ -134,35 +135,41 @@ describe('Injector window conditions', () => { expect(sink.contextCalls).toEqual(['build finished']); }); - it('waits out the estimated playback plus the quiet gap', () => { - // 48,000 bytes of 24 kHz mono PCM16 ≈ 1,000 ms of audio. - injector.noteOutputAudio(48_000); + it('holds items while playback is in progress and delivers after completion + quiet gap', () => { + injector.notePlaybackStarted(); injector.enqueue(complete('done')); expect(sink.contextCalls).toEqual([]); - // One tick before playback end + quiet gap: still closed. - vi.advanceTimersByTime(1_000 + QUIET_GAP_MS - 1); + injector.notePlaybackCompleted(); + // Quiet gap still applies after completion. + vi.advanceTimersByTime(QUIET_GAP_MS - 1); expect(sink.contextCalls).toEqual([]); vi.advanceTimersByTime(1); expect(sink.contextCalls).toEqual(['done']); }); - it('stacks playback estimates for consecutive audio chunks', () => { - injector.noteOutputAudio(48_000); - injector.noteOutputAudio(48_000); + it('holds items through multiple playback chunks until completion', () => { + injector.notePlaybackStarted(); + // Additional chunks arrive while playback is in progress — the + // window stays closed until the Host reports completion. + injector.notePlaybackStarted(); injector.enqueue(complete('done')); - vi.advanceTimersByTime(2_000 + QUIET_GAP_MS - 1); + vi.advanceTimersByTime(QUIET_GAP_MS + 5_000); + expect(sink.contextCalls).toEqual([]); + + injector.notePlaybackCompleted(); + vi.advanceTimersByTime(QUIET_GAP_MS - 1); expect(sink.contextCalls).toEqual([]); vi.advanceTimersByTime(1); expect(sink.contextCalls).toEqual(['done']); }); - it('reopens the window immediately when output audio is cleared', () => { - injector.noteOutputAudio(48_000 * 60); + it('reopens the window immediately when output is cleared during playback', () => { + injector.notePlaybackStarted(); injector.enqueue(complete('interrupted')); vi.advanceTimersByTime(500); expect(sink.contextCalls).toEqual([]); diff --git a/packages/qwen-live/src/orchestrator/injector.ts b/packages/qwen-live/src/orchestrator/injector.ts index 9783ec28ec8..9eeda12edc5 100644 --- a/packages/qwen-live/src/orchestrator/injector.ts +++ b/packages/qwen-live/src/orchestrator/injector.ts @@ -26,8 +26,6 @@ const RECHECK_MIN_MS = 100; const PROGRESS_THROTTLE_MS = 5 * 60_000; const MAX_SPOKEN_CHARS = 280; const MAX_CONTEXT_CHARS = 6_000; -/** 24 kHz mono PCM16. */ -const PLAYBACK_BYTES_PER_MS = 48; export type InjectorItemKind = | 'complete' @@ -79,7 +77,8 @@ export class Injector { private queue: InjectorItem[] = []; private speechInProgress = false; private responseInFlight = false; - private playbackDeadline = 0; + private playbackInProgress = false; + private playbackCompletedAt = 0; private lastProgressAt = new Map(); private timer: ReturnType | undefined; private disposed = false; @@ -95,8 +94,9 @@ export class Injector { // -- window state signals (fed by the orchestrator) ---------------------- noteSpeechStarted(): boolean { - const outputWasPlaying = this.playbackDeadline > this.now(); - this.playbackDeadline = 0; + const outputWasPlaying = this.playbackInProgress; + this.playbackInProgress = false; + this.playbackCompletedAt = 0; this.speechInProgress = true; // Barge-in semantics: pending progress is stale the moment the user // speaks; conclusions and permission asks stay queued. A dropped item @@ -128,13 +128,20 @@ export class Injector { this.poke(); } - noteOutputAudio(bytes: number): void { - const start = Math.max(this.now(), this.playbackDeadline); - this.playbackDeadline = start + bytes / PLAYBACK_BYTES_PER_MS; + notePlaybackStarted(): void { + this.playbackInProgress = true; + this.playbackCompletedAt = 0; + } + + notePlaybackCompleted(): void { + this.playbackInProgress = false; + this.playbackCompletedAt = this.now(); + this.poke(); } noteOutputCleared(): void { - this.playbackDeadline = 0; + this.playbackInProgress = false; + this.playbackCompletedAt = 0; this.poke(); } @@ -194,9 +201,13 @@ export class Injector { private windowClosedForMs(): number { if (this.speechInProgress || this.responseInFlight) return -1; - const quietAt = this.playbackDeadline + this.quietGapMs; - const wait = quietAt - this.now(); - return wait > 0 ? wait : 0; + if (this.playbackInProgress) return -1; + if (this.playbackCompletedAt > 0) { + const quietAt = this.playbackCompletedAt + this.quietGapMs; + const wait = quietAt - this.now(); + return wait > 0 ? wait : 0; + } + return 0; } private poke(): void { diff --git a/packages/qwen-live/src/orchestrator/live-session.test.ts b/packages/qwen-live/src/orchestrator/live-session.test.ts index c942c553d45..682cc974f48 100644 --- a/packages/qwen-live/src/orchestrator/live-session.test.ts +++ b/packages/qwen-live/src/orchestrator/live-session.test.ts @@ -1260,9 +1260,12 @@ describe('LiveSession', () => { expect(host.clearOutput).toHaveBeenCalledTimes(1); }); - it('clears estimated playback tail when speech starts after response.done', async () => { - const { callbacks, host } = await startSession(); + it('clears playback tail when speech starts after response.done', async () => { + const { session, callbacks, host } = await startSession(); + // Playback receipts arrive via coordinator handlers (not realtime + // callbacks) — call the session methods directly as daemon.ts does. + session.notePlaybackStarted({ epoch: 1 }); callbacks.onOutputAudioDelta?.({ callEpoch: 1, responseId: 'resp_tail', diff --git a/packages/qwen-live/src/orchestrator/live-session.ts b/packages/qwen-live/src/orchestrator/live-session.ts index bee4140966c..ff40d1c5232 100644 --- a/packages/qwen-live/src/orchestrator/live-session.ts +++ b/packages/qwen-live/src/orchestrator/live-session.ts @@ -376,6 +376,20 @@ export class LiveSession { }); } + /** LiveCallHandlers.onPlaybackStarted */ + notePlaybackStarted(call: { epoch: number }): void { + const context = this.active; + if (!context || context.epoch !== call.epoch) return; + context.injector.notePlaybackStarted(); + } + + /** LiveCallHandlers.onPlaybackCompleted */ + notePlaybackCompleted(call: { epoch: number }): void { + const context = this.active; + if (!context || context.epoch !== call.epoch) return; + context.injector.notePlaybackCompleted(); + } + /** LiveCallHandlers.onInputAudio */ pushAudio(call: { epoch: number; callId: string; pcm16: Buffer }): boolean { const context = this.active; @@ -466,7 +480,6 @@ export class LiveSession { onOutputAudioDelta: (event: { audio: Uint8Array }) => { if (!current()) return; this.host.sendOutputAudio(context.epoch, event.audio); - context.injector.noteOutputAudio(event.audio.byteLength); }, onResponseCreated: (event: { responseId: string; authority: string }) => { if (!current()) return;