Skip to content
Closed
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
95 commits
Select commit Hold shift + click to select a range
ae8a8d7
feat(cli): add audio bridge for attachments
DragonnZhang Aug 1, 2026
5a007e5
fix(cli): address audio bridge review feedback (#8332)
Aug 2, 2026
411007f
Merge upstream/main into dragon/audio-bridge
DragonnZhang Aug 2, 2026
31489f6
fix(cli): harden audio bridge contracts
DragonnZhang Aug 2, 2026
2bf04b2
fix(cli): preserve audio bridge abort semantics
DragonnZhang Aug 2, 2026
b3aeb24
test(cli): cover audio bridge guard paths
DragonnZhang Aug 2, 2026
17c3eec
test(cli): pin audio bridge skip and text-only guard branches (#8332)
qwen-code-ci-bot Aug 2, 2026
544258b
Merge branch 'dragon/audio-bridge' of https://github.com/DragonnZhang…
qwen-code-dev-bot Aug 2, 2026
753b9e6
fix(cli): address audio bridge review feedback (#8332)
Aug 2, 2026
b141afc
test(cli): cover audio and vision bridge composition
DragonnZhang Aug 2, 2026
28ff0e0
test(cli): strengthen audio bridge review assertions (#8332)
Aug 2, 2026
aeab4fe
Merge branch 'dragon/audio-bridge' of https://github.com/DragonnZhang…
qwen-code-dev-bot Aug 2, 2026
48c90dc
test(cli): pin audio bridge injection-defense label (#8332)
Aug 3, 2026
6b84b25
fix(cli): state audio attachment limit in overflow note (#8332)
Aug 3, 2026
ed7bc1c
Merge branch 'main' into dragon/audio-bridge
qwen-code-dev-bot Aug 3, 2026
6937de4
Merge branch 'main' into dragon/audio-bridge
wenshao Aug 3, 2026
64dbfcf
Merge branch 'main' into dragon/audio-bridge
qwen-code-dev-bot Aug 4, 2026
15ce5f4
fix audio handling for model overrides
DragonnZhang Aug 4, 2026
ab14e5d
fix(cli): harden audio bridge fail-closed paths (#8332)
qwen-code-dev-bot Aug 4, 2026
b9ed4fd
fix(core): bound bridge audio reads
DragonnZhang Aug 5, 2026
3cda2d5
refactor(core): share inline media limit
DragonnZhang Aug 5, 2026
811b64c
fix(cli): honor audio target capabilities
DragonnZhang Aug 5, 2026
5bcf6d3
chore: merge main to resolve conflicts
DragonnZhang Aug 5, 2026
a7f855e
fix(cli): preserve direct ACP audio blocks for the audio bridge (#8332)
DragonnZhang Aug 5, 2026
ce34a75
Merge branch 'dragon/audio-bridge' of https://github.com/DragonnZhang…
qwen-code-dev-bot Aug 5, 2026
e4b9a69
Merge branch 'main' into dragon/audio-bridge
qwen-code-dev-bot Aug 5, 2026
9a3e34f
fix(cli): close audio bridge size, routing, and guard gaps (#8332)
qwen-code-dev-bot Aug 5, 2026
2e1d7dc
fix(cli): decode RFC 6052 NAT64 addresses safely
DragonnZhang Aug 6, 2026
2ba71fb
fix(cli): clamp spoofed ACP audio payloads
DragonnZhang Aug 6, 2026
76d1aa8
fix(cli): preserve audio route during vision bridging
DragonnZhang Aug 6, 2026
0fd885f
chore: merge main to resolve conflicts
DragonnZhang Aug 6, 2026
fb0e109
fix(cli): preserve audio across vision routing
DragonnZhang Aug 7, 2026
88656cb
test: expose audio bridge limit in core mock
DragonnZhang Aug 7, 2026
b7b8332
fix(cli): tighten audio override handling
DragonnZhang Aug 7, 2026
e582f1a
fix(cli): honor audio route capabilities
DragonnZhang Aug 13, 2026
e374028
fix(cli): preserve valid transcript unicode
DragonnZhang Aug 13, 2026
430e023
merge: sync upstream main
DragonnZhang Aug 14, 2026
6316689
merge: sync upstream main
DragonnZhang Aug 14, 2026
40fdf3a
fix(acp): report unavailable audio routes
DragonnZhang Aug 14, 2026
6ca977c
fix(cli): fail closed on unresolved audio routes
DragonnZhang Aug 14, 2026
36e1847
fix(cli): rebridge drained images after route fallback
DragonnZhang Aug 15, 2026
789134b
fix(cli): keep failed media routes sticky
DragonnZhang Aug 15, 2026
5ba2b44
fix(cli): reconcile media with final model route
DragonnZhang Aug 15, 2026
b8867ac
fix(cli): keep media fallback on primary route
DragonnZhang Aug 15, 2026
15b7f1a
fix(cli): reconcile media across model routes
DragonnZhang Aug 16, 2026
68126df
fix(cli): preserve media across route fallback
DragonnZhang Aug 16, 2026
af781cf
fix(cli): recover audio after route resolution failures
DragonnZhang Aug 17, 2026
99532e6
Merge upstream main into audio bridge
DragonnZhang Aug 17, 2026
a9d0975
Merge remote-tracking branch 'origin/main' into dragon/audio-bridge
DragonnZhang Aug 18, 2026
c7d7cf6
fix(cli): address media-routing review criticals
DragonnZhang Aug 19, 2026
0e330b2
Merge remote-tracking branch 'origin/main' into dragon/audio-bridge
DragonnZhang Aug 19, 2026
94483f3
Merge remote-tracking branch 'origin/main' into tmp-merge-8332
DragonnZhang Aug 20, 2026
4e38d79
fix(cli): keep media intact through aborts and bridge failures, clear…
DragonnZhang Aug 20, 2026
e35b13b
fix(cli): keep media intact through mid-bridge aborts and route resol…
DragonnZhang Aug 20, 2026
f0042dc
fix(cli): keep drained audio retryable and scope exact-route marker t…
DragonnZhang Aug 21, 2026
534c119
test(cli): give marker-scoping regression tests production-faithful p…
DragonnZhang Aug 21, 2026
596c616
fix(cli): deliver failed drained steers once and clamp resolution-fai…
DragonnZhang Aug 21, 2026
40b037f
Merge remote-tracking branch 'origin/main' into pr-8332
DragonnZhang Aug 21, 2026
21a0aab
fix(cli): keep steer recovery single-channel and capability-check ove…
DragonnZhang Aug 22, 2026
8587295
test(cli): cover the accept-settled recovery channel for failed drain…
DragonnZhang Aug 22, 2026
ed5afd6
fix(cli): capture pristine images for retry and fail-close them on abort
DragonnZhang Aug 22, 2026
e8e2391
Merge remote-tracking branch 'origin/main' into pr-8332
DragonnZhang Aug 22, 2026
81e1549
fix(cli): keep media-routed continuations on the drain-time route and…
DragonnZhang Aug 22, 2026
75a74c9
Merge remote-tracking branch 'origin/main' into pr-8332
DragonnZhang Aug 22, 2026
e5b375f
fix(cli): align restored ask_user_question continuation with the mode…
DragonnZhang Aug 22, 2026
ba56d83
fix(cli): fail-close nested tool-result media under full-turn selecto…
DragonnZhang Aug 22, 2026
68c0295
fix(cli): keep media fail-closes retryable, exact-routed, and lossles…
DragonnZhang Aug 23, 2026
1911ef0
Merge remote-tracking branch 'origin/main' into pr-8332
DragonnZhang Aug 23, 2026
5709c7d
fix(cli): keep routed media case-insensitive, clamped, probed, and ex…
DragonnZhang Aug 23, 2026
397a54c
Merge remote-tracking branch 'origin/main' into pr-8332
DragonnZhang Aug 23, 2026
91fbcf7
fix(cli): preserve structured tool output when fail-closing nested media
DragonnZhang Aug 24, 2026
8e64e5c
Merge remote-tracking branch 'origin/main' into pr-8332
DragonnZhang Aug 24, 2026
4f270ad
fix(core): sink audio bridge byte limit into utils leaf layer
DragonnZhang Aug 24, 2026
89c1fa2
fix(cli): gate headless tool-result media, preserve verbatim response…
DragonnZhang Aug 24, 2026
96fe03e
fix(cli): recover accepted steers on send failure; fail-closed media …
DragonnZhang Aug 24, 2026
ba76096
fix(cli): settle-aware steer recovery; model-facing marker for withhe…
DragonnZhang Aug 25, 2026
2f432a4
fix(cli): police unstamped overrides and keep drain media fail-closed…
DragonnZhang Aug 25, 2026
325dc65
Merge remote-tracking branch 'origin/main' into pr-8332
DragonnZhang Aug 26, 2026
cb7f361
fix(cli): address R49 review — stamp core-driven drains, recover stee…
DragonnZhang Aug 26, 2026
61a5983
Merge remote-tracking branch 'upstream/main' into dragon/audio-bridge
DragonnZhang Aug 26, 2026
02897f9
fix(cli): widen the retry gate fallback member so mediaRouted access …
DragonnZhang Aug 26, 2026
58fd18f
fix(cli): close the R49 media-gate holes (foreign MIME, headless stam…
DragonnZhang Aug 27, 2026
cb58dc1
Merge remote-tracking branch 'origin/main' into pr-8332
DragonnZhang Aug 27, 2026
e1cb25a
Merge remote-tracking branch 'origin/main' into dragon/audio-bridge
DragonnZhang Aug 28, 2026
091b535
fix(cli): address round-51 critical findings on media routing (#8332)
DragonnZhang Aug 28, 2026
a459192
Merge remote-tracking branch 'origin/main' into dragon/audio-bridge
DragonnZhang Aug 29, 2026
cf53207
Merge remote-tracking branch 'origin/main' into dragon/audio-bridge
DragonnZhang Aug 29, 2026
cc84601
fix(cli): fire settlement-carrier hand-back hooks and carry drain rou…
DragonnZhang Aug 30, 2026
153421e
fix(core): compare MIME case-insensitively in Gemini media conversion…
DragonnZhang Aug 30, 2026
d416820
fix(cli): fail-close still-raw audio on all vision-bridge abort exits…
DragonnZhang Aug 30, 2026
7014597
fix(cli): fail-close staged tool media at the surviving-override abor…
DragonnZhang Aug 30, 2026
d201321
fix(cli): type the R52-33 runAudioBridge test mocks against PartListU…
DragonnZhang Aug 30, 2026
4ac6ac4
Merge remote-tracking branch 'origin/main' into dragon/audio-bridge
DragonnZhang Aug 30, 2026
8c4a61c
fix(cli): strip the steer via a composition-site snapshot, not suffix…
DragonnZhang Aug 30, 2026
11ad92d
fix(cli): defer the Idle teammate drain while a retryable failure is …
DragonnZhang Aug 30, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions docs/design/2026-08-01-audio-bridge.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# Audio bridge for user attachments

## Context

Qwen Code already transcribes ACP audio content blocks through the configured
`voiceModel` when the primary model does not support audio. Interactive and
headless `@` attachments do not use that path: file processing replaces an
unsupported audio file with a placeholder before the CLI can transcribe it.

## Design

Extract the ACP-only conversion into a shared CLI audio-bridge service. The
service accepts normalized prompt parts, keeps audio unchanged when the active
primary model supports it, and otherwise replaces every audio part with an
untrusted machine transcript produced by `voiceModel`. Missing configuration,
oversized audio, empty transcripts, request failures, and cancellation never
forward raw audio to a text-only primary model.

Interactive and headless `@` resolution preserve unsupported audio only when a
batch-capable `voiceModel` is selected, then run the shared bridge before the
existing vision bridge. ACP direct audio blocks use the same service and keep
their existing agent-message disclosure. A bridge notice identifies the voice
model whenever audio left the machine, including failed or empty responses.

The existing `voiceModel` setting and `/model --voice` command remain the sole
configuration surface. Voice dictation behavior is unchanged.

## Scope

This change covers user-supplied ACP audio blocks and interactive/headless
`@audio` attachments. Audio nested in tool results, channel-specific attachment
plumbing, video, and realtime dictation are unchanged.
Comment thread
DragonnZhang marked this conversation as resolved.
6 changes: 3 additions & 3 deletions docs/users/configuration/settings.md
Original file line number Diff line number Diff line change
Expand Up @@ -284,9 +284,9 @@ The `extra_body` field allows you to add custom parameters to the request body s

#### voiceModel

| Setting | Type | Description | Default |
| ------------ | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- |
| `voiceModel` | string | Model used for voice transcription. Leave empty to keep voice dictation disabled until a voice model is selected. Can also be set via `/model --voice`. | `""` |
| Setting | Type | Description | Default |
| ------------ | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- |
| `voiceModel` | string | Model used for voice transcription and as the audio bridge when a text-only main model receives an audio attachment. Leave empty to keep voice dictation and audio bridging disabled until a voice model is selected. Can also be set via `/model --voice`. | `""` |

#### modelFallbacks

Expand Down
2 changes: 1 addition & 1 deletion docs/users/features/commands.md
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ Commands for managing AI tools and models.
| → `yolo` | Auto-approve everything (quick prototyping) | `/approval-mode yolo` |
| `/model` | Switch model used in current session | `/model`, `/model <model-id>` (switch immediately) |
| `/model --fast` | Set a lighter model for prompt suggestions | `/model --fast qwen3-coder-flash` |
| `/model --voice` | Set the model used for voice transcription | `/model --voice <model-id>` |
| `/model --voice` | Set the model used for voice transcription and audio bridging | `/model --voice <model-id>` |
| `/model --vision` | Set the vision-bridge model used to transcribe images for a text-only main model | `/model --vision <model-id>` |
| `/model --compaction` | Set the model used for chat compression | `/model --compaction <model-id>`, `/model --compaction clear` |
| `/model --image` | Set an image-only model for the built-in image generation tool | `/model --image <model-id>` |
Expand Down
109 changes: 104 additions & 5 deletions packages/cli/src/acp-integration/session/Session.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4414,7 +4414,7 @@ describe('Session', () => {
}
});

it('routes ACP audio prompts through the voice bridge for text-only primary models', async () => {
it('routes ACP audio prompts through the audio bridge for text-only primary models', async () => {
mockConfig.getEffectiveInputModalities = vi.fn().mockReturnValue({});
mockChat.sendMessageStream = vi
.fn()
Expand Down Expand Up @@ -4465,7 +4465,7 @@ describe('Session', () => {
);
});

it('does not run the voice bridge when the primary model supports audio', async () => {
it('does not run the audio bridge when the primary model supports audio', async () => {
mockConfig.getEffectiveInputModalities = vi
.fn()
.mockReturnValue({ audio: true });
Expand Down Expand Up @@ -4570,7 +4570,7 @@ describe('Session', () => {
);
});

it('rejects oversized ACP audio before decoding for the voice bridge', async () => {
it('rejects oversized ACP audio before decoding for the audio bridge', async () => {
const ENV_KEY = 'QWEN_CODE_MAX_INLINE_MEDIA_BYTES';
const original = process.env[ENV_KEY];
process.env[ENV_KEY] = String(20 * 1024 * 1024);
Expand Down Expand Up @@ -4610,7 +4610,7 @@ describe('Session', () => {
);
});

it('falls back to text-only parts when voice bridge transcription fails', async () => {
it('falls back to text-only parts when audio bridge transcription fails', async () => {
mockConfig.getEffectiveInputModalities = vi.fn().mockReturnValue({});
mockChat.sendMessageStream = vi
.fn()
Expand Down Expand Up @@ -5122,6 +5122,59 @@ describe('Session', () => {
}
});

it('preserves unsupported audio @ paths for the audio bridge', async () => {
const tempDir = await fs.realpath(
await fs.mkdtemp(path.join(os.tmpdir(), 'qwen-acp-audio-')),
);
const audioPath = path.join(tempDir, 'recording.wav');
await fs.writeFile(audioPath, 'audio');
mockConfig.getProjectRoot = vi.fn().mockReturnValue(tempDir);
mockConfig.getWorkspaceContext = vi.fn().mockReturnValue({
isPathWithinWorkspace: (pathSpec: string) =>
path.resolve(tempDir, pathSpec).startsWith(`${tempDir}${path.sep}`),
});
mockConfig.getEffectiveInputModalities = vi.fn().mockReturnValue({});
Object.assign(mockSettings.merged as Record<string, unknown>, {
voiceModel: 'qwen3-asr-flash',
});
const readManyFilesSpy = vi
.spyOn(core, 'readManyFiles')
.mockResolvedValue({
contentParts: {
inlineData: { mimeType: 'audio/wav', data: 'UklGRg==' },
},
} as Awaited<ReturnType<typeof core.readManyFiles>>);
transcribeVoiceAudioSpy.mockResolvedValue('audio transcript');
mockChat.sendMessageStream = vi
.fn()
.mockResolvedValue(createEmptyStream());

try {
await session.prompt({
sessionId: 'test-session-id',
prompt: [{ type: 'text', text: `listen to @${audioPath}` }],
});

expect(readManyFilesSpy).toHaveBeenCalledWith(
mockConfig,
expect.objectContaining({
paths: [audioPath],
preserveUnsupportedAudioForBridge: true,
validatedPathIdentities: expect.any(Map),
}),
);
expect(textParts(firstSentMessage())).toContainEqual(
expect.stringContaining('audio transcript'),
);
expect(firstSentMessage().some((part) => 'inlineData' in part)).toBe(
false,
);
} finally {
readManyFilesSpy.mockRestore();
await fs.rm(tempDir, { recursive: true, force: true });
}
});

it('resolves image @ paths from ACP text through the vision bridge', async () => {
const tempDir = await fs.realpath(
await fs.mkdtemp(path.join(os.tmpdir(), 'qwen-acp-image-')),
Expand Down Expand Up @@ -5185,6 +5238,52 @@ describe('Session', () => {
}
});

it('reads image @ paths for an image-capable primary model without the vision bridge', async () => {
const tempDir = await fs.realpath(
await fs.mkdtemp(path.join(os.tmpdir(), 'qwen-acp-imgcap-')),
);
const imagePath = path.join(tempDir, 'shot.png');
await fs.writeFile(imagePath, 'image');
mockConfig.getProjectRoot = vi.fn().mockReturnValue(tempDir);
mockConfig.getWorkspaceContext = vi.fn().mockReturnValue({
isPathWithinWorkspace: (pathSpec: string) =>
path.resolve(tempDir, pathSpec).startsWith(`${tempDir}${path.sep}`),
});
mockConfig.getEffectiveInputModalities = vi
.fn()
.mockReturnValue({ image: true });
const readManyFilesSpy = vi
.spyOn(core, 'readManyFiles')
.mockResolvedValue({
contentParts: {
inlineData: { mimeType: 'image/png', data: 'iVBORw0KGgo=' },
},
} as Awaited<ReturnType<typeof core.readManyFiles>>);
mockChat.sendMessageStream = vi
.fn()
.mockResolvedValue(createEmptyStream());

try {
await session.prompt({
sessionId: 'test-session-id',
prompt: [{ type: 'text', text: `look at @${imagePath}` }],
});

expect(readManyFilesSpy).toHaveBeenCalledWith(
mockConfig,
expect.objectContaining({
paths: [imagePath],
}),
);
expect(firstSentMessage().some((part) => 'inlineData' in part)).toBe(
true,
);
} finally {
readManyFilesSpy.mockRestore();
await fs.rm(tempDir, { recursive: true, force: true });
}
});

it('ignores non-image and relative ACP text @ paths', async () => {
const tempDir = await fs.realpath(
await fs.mkdtemp(path.join(os.tmpdir(), 'qwen-acp-paths-')),
Expand Down Expand Up @@ -7145,7 +7244,7 @@ describe('Session', () => {
});

const audioFallbackPart = {
text: '[Voice bridge could not transcribe attached audio: no voice model is configured. The audio content is unavailable; do not assume or invent what it says.]',
text: '[Audio bridge could not transcribe attached audio: no voice model is configured. The audio content is unavailable; do not assume or invent what it says.]',
};
const midTurnParts: Part[] = [
{
Expand Down
Loading
Loading