Skip to content

Commit 1a96521

Browse files
justinchubyCopilot
justinchuby
and
Copilot
committed
Add speech-language L5 generation test + Qwen3-ASR forced language variant
Implement _run_speech_language_generation() for the 3-model speech pipeline (audio_encoder → embedding → decoder). Supports 3D position_ids for models like Qwen3-ASR. Add two Qwen3-ASR test cases: - qwen3-asr: auto language detection (no text prompt) - qwen3-asr-en: forced English via decoder prefix tokens Update generate_golden.py to support forced language prefix for Qwen3-ASR when prompts are specified in the YAML. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Justin Chu <justinchu@microsoft.com>
1 parent f5eb0ce commit 1a96521

8 files changed

Lines changed: 446 additions & 19 deletions

File tree

‎scripts/generate_golden.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -648,12 +648,18 @@ def _prepare_speech_language_inputs(
648648
text_prompt = processor.apply_chat_template(
649649
messages, add_generation_prompt=True, tokenize=False
650650
)
651+
# If prompts are provided, append as forced decoder prefix
652+
# (e.g. "language English<asr_text>" to skip language detection).
653+
force_prefix = ""
654+
if case.prompts:
655+
force_prefix = case.prompts[0]
656+
text_prompt = text_prompt + force_prefix
651657
processed = processor(
652658
text=text_prompt,
653659
audio=[audio_array],
654660
return_tensors="pt",
655661
).to(device)
656-
prompt_for_golden = str(audio_path)
662+
prompt_for_golden = force_prefix or str(audio_path)
657663
else:
658664
# Gemma4-style: text prompt + audio
659665
prompt_text = case.prompts[0]

‎testdata/cases/speech/qwen3-asr-en.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
model_id: "Qwen/Qwen3-ASR-0.6B"
2+
revision: "main"
3+
task_type: "speech-language"
4+
dtype: "float32"
5+
trust_remote_code: true
6+
7+
inputs:
8+
prompts:
9+
- "language English<asr_text>"
10+
audio:
11+
- "652-129742-0006.flac"
12+
13+
generation:
14+
max_new_tokens: 50
15+
do_sample: false
16+
17+
level: "L4+L5"
18+
19+
notes: "Qwen3-ASR with forced English language. Prompt is appended to assistant turn as decoder prefix to skip language detection."

‎testdata/cases/speech/qwen3-asr.yaml‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@ dtype: "float32"
55
trust_remote_code: true
66

77
inputs:
8-
prompts:
9-
- "Transcribe this audio."
108
audio:
119
- "652-129742-0006.flac"
1210

@@ -16,4 +14,4 @@ generation:
1614

1715
level: "L4+L5"
1816

19-
notes: "Qwen3-ASR speech recognition. 3-model split: audio_encoder + embedding + decoder."
17+
notes: "Qwen3-ASR speech recognition (auto language detection). 3-model split: audio_encoder + embedding + decoder. Uses fixed chat template with audio-only input."

‎testdata/golden/speech/qwen3-asr-en.json‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
{
2+
"top1_id": 34,
3+
"top2_id": 15265,
4+
"top10_ids": [
5+
34,
6+
15265,
7+
6127,
8+
89915,
9+
95870,
10+
1143,
11+
3882,
12+
8852,
13+
7339,
14+
98180
15+
],
16+
"top10_logits": [
17+
"0x1.b1da520000000p+4",
18+
"0x1.7323c80000000p+4",
19+
"0x1.71f90c0000000p+4",
20+
"0x1.2e0cb20000000p+4",
21+
"0x1.2a4f400000000p+4",
22+
"0x1.20b6e20000000p+4",
23+
"0x1.1a96840000000p+4",
24+
"0x1.176bf80000000p+4",
25+
"0x1.1714900000000p+4",
26+
"0x1.1638a80000000p+4"
27+
],
28+
"logits_summary": [
29+
"0x1.b1da520000000p+4",
30+
"-0x1.792c420000000p+4",
31+
"-0x1.9f2cfc1fa659bp+2",
32+
"0x1.1e455b8e0c75cp+2"
33+
],
34+
"input_ids": [
35+
151644,
36+
8948,
37+
198,
38+
151645,
39+
198,
40+
151644,
41+
872,
42+
198,
43+
151669,
44+
151676,
45+
151676,
46+
151676,
47+
151676,
48+
151676,
49+
151676,
50+
151676,
51+
151676,
52+
151676,
53+
151676,
54+
151676,
55+
151676,
56+
151676,
57+
151676,
58+
151676,
59+
151676,
60+
151676,
61+
151676,
62+
151676,
63+
151676,
64+
151676,
65+
151676,
66+
151676,
67+
151676,
68+
151676,
69+
151676,
70+
151676,
71+
151676,
72+
151676,
73+
151676,
74+
151676,
75+
151676,
76+
151676,
77+
151676,
78+
151676,
79+
151676,
80+
151676,
81+
151676,
82+
151676,
83+
151676,
84+
151676,
85+
151676,
86+
151676,
87+
151676,
88+
151676,
89+
151676,
90+
151676,
91+
151676,
92+
151676,
93+
151676,
94+
151676,
95+
151676,
96+
151676,
97+
151676,
98+
151676,
99+
151676,
100+
151676,
101+
151676,
102+
151676,
103+
151676,
104+
151676,
105+
151676,
106+
151676,
107+
151676,
108+
151676,
109+
151676,
110+
151676,
111+
151676,
112+
151676,
113+
151676,
114+
151676,
115+
151676,
116+
151676,
117+
151676,
118+
151676,
119+
151676,
120+
151676,
121+
151676,
122+
151676,
123+
151676,
124+
151676,
125+
151676,
126+
151676,
127+
151676,
128+
151676,
129+
151676,
130+
151676,
131+
151676,
132+
151676,
133+
151676,
134+
151676,
135+
151676,
136+
151676,
137+
151676,
138+
151676,
139+
151676,
140+
151676,
141+
151676,
142+
151676,
143+
151676,
144+
151676,
145+
151676,
146+
151676,
147+
151676,
148+
151676,
149+
151676,
150+
151676,
151+
151676,
152+
151676,
153+
151676,
154+
151676,
155+
151676,
156+
151676,
157+
151676,
158+
151676,
159+
151676,
160+
151676,
161+
151676,
162+
151676,
163+
151670,
164+
151645,
165+
198,
166+
151644,
167+
77091,
168+
198,
169+
11528,
170+
6364,
171+
151704
172+
]
173+
}

‎testdata/golden/speech/qwen3-asr-en_generation.json‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{
2+
"model_id": "Qwen/Qwen3-ASR-0.6B",
3+
"prompt": "language English<asr_text>",
4+
"generated_tokens": [
5+
34,
6+
4943,
7+
76773,
8+
1231,
9+
13459,
10+
1064,
11+
25,
12+
11778,
13+
9255,
14+
65085,
15+
95870,
16+
11,
17+
1438,
18+
1119,
19+
23091,
20+
11,
21+
7842,
22+
12021,
23+
11,
24+
24353,
25+
11,
26+
323,
27+
46105,
28+
311,
29+
3200,
30+
13,
31+
151645
32+
],
33+
"generated_text": "Cauliflower mayonnaise: Take cold boiled cauliflower, break into branches, adding salt, pepper, and vinegar to season."
34+
}

‎testdata/golden/speech/qwen3-asr.json‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,22 +14,22 @@
1414
60740
1515
],
1616
"top10_logits": [
17-
"0x1.bdd5f80000000p+4",
18-
"0x1.3ade700000000p+4",
19-
"0x1.25e2640000000p+4",
20-
"0x1.f50dda0000000p+3",
21-
"0x1.e830be0000000p+3",
22-
"0x1.e3a7700000000p+3",
23-
"0x1.e15bca0000000p+3",
24-
"0x1.d6fb280000000p+3",
25-
"0x1.cc90c20000000p+3",
26-
"0x1.bea7380000000p+3"
17+
"0x1.bdd5c00000000p+4",
18+
"0x1.3ade6a0000000p+4",
19+
"0x1.25e2300000000p+4",
20+
"0x1.f50d980000000p+3",
21+
"0x1.e82fc40000000p+3",
22+
"0x1.e3a6e60000000p+3",
23+
"0x1.e15ae80000000p+3",
24+
"0x1.d6faea0000000p+3",
25+
"0x1.cc90e80000000p+3",
26+
"0x1.bea70c0000000p+3"
2727
],
2828
"logits_summary": [
29-
"-0x1.2516140000000p+1",
30-
"0x1.d376820000000p+1",
31-
"-0x1.1ff44e0000000p+4",
32-
"0x1.bdd5f80000000p+4"
29+
"0x1.bdd5c00000000p+4",
30+
"-0x1.1ff45c0000000p+4",
31+
"-0x1.251727ddff51dp+1",
32+
"0x1.d3765f0f5c1a9p+1"
3333
],
3434
"input_ids": [
3535
151644,

‎testdata/golden/speech/qwen3-asr_generation.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,4 +34,4 @@
3434
151645
3535
],
3636
"generated_text": "language English<asr_text>Cauliflower mayonnaise: Take cold boiled cauliflower, break into branches, adding salt, pepper, and vinegar to season."
37-
}
37+
}

0 commit comments

Comments
 (0)