From f6a22324b7ca8ac8a978eb050a2dfe81dd19b0cd Mon Sep 17 00:00:00 2001 From: Liangyx2 Date: Fri, 3 Jul 2026 14:16:36 +0800 Subject: [PATCH 1/5] Support Fish Speech TTS inference on XPU platform Motivation: - Fish Speech (FishSpeechSlowARForConditionalGeneration) model cannot run on Intel XPU due to CUDA-specific API calls and memory constraints. - The offline inference script lacks --deploy-config support, inconsistent with other model examples in the framework. - Streaming mode loses audio data due to DELTA output draining. Technical details: 1. fish_kvcache_attn.py: Guard torch.cuda.is_current_stream_capturing() with torch.cuda.is_available() check to avoid RuntimeError on XPU where torch.cuda is a dummy module. 2. fish_qwen3_omni.yaml: Add platforms.xpu overrides reducing max_model_len to 2048 and increasing gpu_memory_utilization to 0.8 for stage-0, fitting within XPU memory constraints when two stages share a single device. 3. end2end.py: Add --deploy-config argument aligned with other offline inference scripts. Fix streaming mode to accumulate audio chunks across DELTA outputs instead of only saving the final (empty) chunk. Performance impact: - Enables Fish Speech S2 Pro end-to-end TTS on single Intel XPU (BMG/ARC). - No regression on CUDA path (original defaults unchanged). Workload mapping: - Fish Speech S2 Pro (fishaudio/s2-pro) text-to-speech on XPU. Signed-off-by: Liangyx2 --- .../text_to_speech/fish_speech/end2end.py | 60 ++++++++++++++++--- vllm_omni/attention/fish_kvcache_attn.py | 2 + vllm_omni/deploy/fish_qwen3_omni.yaml | 11 ++++ 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/examples/offline_inference/text_to_speech/fish_speech/end2end.py b/examples/offline_inference/text_to_speech/fish_speech/end2end.py index 4d6855246a5..2ae61df1194 100644 --- a/examples/offline_inference/text_to_speech/fish_speech/end2end.py +++ b/examples/offline_inference/text_to_speech/fish_speech/end2end.py @@ -127,7 +127,6 @@ def main(args): output_dir = args.output_dir os.makedirs(output_dir, exist_ok=True) - stage_configs_path = args.stage_configs_path or DEFAULT_STAGE_CONFIG model_name = args.model or DEFAULT_MODEL inputs = [ @@ -139,12 +138,17 @@ def main(args): ) ] - omni = Omni( + omni_kwargs = dict( model=model_name, - stage_configs_path=stage_configs_path, log_stats=args.log_stats, stage_init_timeout=args.stage_init_timeout, ) + if args.deploy_config: + omni_kwargs["deploy_config"] = args.deploy_config + else: + omni_kwargs["stage_configs_path"] = args.stage_configs_path or DEFAULT_STAGE_CONFIG + + omni = Omni(**omni_kwargs) t_start = time.perf_counter() for stage_outputs in omni.generate(inputs): @@ -165,7 +169,6 @@ async def main_streaming(args): output_dir = args.output_dir os.makedirs(output_dir, exist_ok=True) - stage_configs_path = args.stage_configs_path or DEFAULT_STAGE_CONFIG model_name = args.model or DEFAULT_MODEL prompt = build_prompt( @@ -175,23 +178,39 @@ async def main_streaming(args): model_name=model_name, ) - omni = AsyncOmni( + omni_kwargs = dict( model=model_name, - stage_configs_path=stage_configs_path, log_stats=args.log_stats, stage_init_timeout=args.stage_init_timeout, ) + if args.deploy_config: + omni_kwargs["deploy_config"] = args.deploy_config + else: + omni_kwargs["stage_configs_path"] = args.stage_configs_path or DEFAULT_STAGE_CONFIG + + omni = AsyncOmni(**omni_kwargs) request_id = "0" t_start = time.perf_counter() t_prev = t_start chunk_idx = 0 + all_audio_chunks: list[torch.Tensor] = [] + sample_rate = None async for stage_output in omni.generate(prompt, request_id=request_id): mm = stage_output.request_output.outputs[0].multimodal_output if not stage_output.finished: t_now = time.perf_counter() audio = mm.get("audio") + if audio is not None: + if isinstance(audio, list): + all_audio_chunks.extend(audio) + else: + all_audio_chunks.append(audio) + sr_raw = mm.get("sr") + if sr_raw is not None: + sr_val = sr_raw[-1] if isinstance(sr_raw, list) and sr_raw else sr_raw + sample_rate = sr_val.item() if hasattr(sr_val, "item") else int(sr_val) n = len(audio) if isinstance(audio, list) else (0 if audio is None else 1) dt_ms = (t_now - t_prev) * 1000 ttfa_ms = (t_now - t_start) * 1000 @@ -202,10 +221,29 @@ async def main_streaming(args): t_prev = t_now chunk_idx += 1 else: + # Collect any remaining audio in the final message + audio = mm.get("audio") if mm else None + if audio is not None: + if isinstance(audio, list): + all_audio_chunks.extend(audio) + else: + all_audio_chunks.append(audio) + sr_raw = mm.get("sr") if mm else None + if sr_raw is not None: + sr_val = sr_raw[-1] if isinstance(sr_raw, list) and sr_raw else sr_raw + sample_rate = sr_val.item() if hasattr(sr_val, "item") else int(sr_val) + t_end = time.perf_counter() total_ms = (t_end - t_start) * 1000 logger.info("Request %s: done total=%.1fms chunks=%d", request_id, total_ms, chunk_idx) - _save_wav(output_dir, request_id, mm) + + if all_audio_chunks and sample_rate: + audio_tensor = torch.cat(all_audio_chunks, dim=-1) + out_wav = os.path.join(output_dir, f"output_{request_id}.wav") + sf.write(out_wav, audio_tensor.float().cpu().numpy().flatten(), samplerate=sample_rate, format="WAV") + logger.info("Request %s: saved audio to %s (sr=%d)", request_id, out_wav, sample_rate) + else: + logger.warning("Request %s: no audio chunks received", request_id) def parse_args(): @@ -236,11 +274,17 @@ def parse_args(): default=None, help=f"HuggingFace model path (default: {DEFAULT_MODEL}).", ) + parser.add_argument( + "--deploy-config", + type=str, + default=None, + help="Deploy config YAML path (under vllm_omni/deploy/).", + ) parser.add_argument( "--stage-configs-path", type=str, default=None, - help="Path to stage configs YAML.", + help="Path to stage configs YAML (legacy, prefer --deploy-config).", ) parser.add_argument( "--output-dir", diff --git a/vllm_omni/attention/fish_kvcache_attn.py b/vllm_omni/attention/fish_kvcache_attn.py index 9e552e76210..e7235168cc3 100644 --- a/vllm_omni/attention/fish_kvcache_attn.py +++ b/vllm_omni/attention/fish_kvcache_attn.py @@ -111,6 +111,8 @@ def can_use_fish_kvcache_attn( def _is_cuda_graph_capturing() -> bool: + if not torch.cuda.is_available(): + return False return bool(torch.cuda.is_current_stream_capturing()) diff --git a/vllm_omni/deploy/fish_qwen3_omni.yaml b/vllm_omni/deploy/fish_qwen3_omni.yaml index 5b0c44988a0..e8e5279786c 100644 --- a/vllm_omni/deploy/fish_qwen3_omni.yaml +++ b/vllm_omni/deploy/fish_qwen3_omni.yaml @@ -63,3 +63,14 @@ stages: max_tokens: 65536 seed: 42 repetition_penalty: 1.0 + +platforms: + xpu: + stages: + - stage_id: 0 + gpu_memory_utilization: 0.8 + max_model_len: 2048 + max_num_batched_tokens: 2048 + - stage_id: 1 + max_model_len: 2048 + max_num_batched_tokens: 2048 From aaf96fa7095fb0ce7639dce0e2eb8a8c5d2e3663 Mon Sep 17 00:00:00 2001 From: Yuxiang Liang Date: Fri, 3 Jul 2026 14:38:00 +0800 Subject: [PATCH 2/5] Update supported_models.md Signed-off-by: Yuxiang Liang --- docs/models/supported_models.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 95f93f5c780..bf0f4e95e12 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -82,7 +82,7 @@ th { | `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | | | `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | | | `Flux2Pipeline` | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | ✅︎ | ✅︎ | | | -| `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | | +| `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | ✅︎ | | `DreamIDOmniPipeline` | DreamID-Omni | `XuGuo699/DreamID-Omni` | ✅︎ | ✅︎ | | | | `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only) | `SenseNova/SenseNova-U1-8B-MoT` | ✅︎ | | | | | `LancePipeline` | Lance | `bytedance-research/Lance` | ✅︎ | | | | From 583c0fb411f4c14a4328c8b8301d52e1729f3974 Mon Sep 17 00:00:00 2001 From: Liangyx2 Date: Wed, 8 Jul 2026 10:18:39 +0800 Subject: [PATCH 3/5] Refactor Fish Speech end2end.py: extract helper functions to reduce duplication Extract _build_omni_kwargs() and _collect_audio_chunk() helpers to eliminate duplicated logic between main() and main_streaming() paths. Signed-off-by: Liangyx2 --- .../text_to_speech/fish_speech/end2end.py | 84 +++++++++---------- 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/examples/offline_inference/text_to_speech/fish_speech/end2end.py b/examples/offline_inference/text_to_speech/fish_speech/end2end.py index 2ae61df1194..5661cccd3b2 100644 --- a/examples/offline_inference/text_to_speech/fish_speech/end2end.py +++ b/examples/offline_inference/text_to_speech/fish_speech/end2end.py @@ -122,6 +122,43 @@ def _save_wav(output_dir: str, request_id: str, mm: dict) -> None: logger.info("Request %s: saved audio to %s (sr=%d)", request_id, out_wav, sr) +def _collect_audio_chunk( + mm: dict, + all_audio_chunks: list[torch.Tensor], + sample_rate: int | None, +) -> tuple[int | None, int]: + """Extract audio and sample rate from a multimodal output chunk. + + Returns updated sample_rate and the number of audio segments in this chunk. + """ + audio = mm.get("audio") if mm else None + if audio is not None: + if isinstance(audio, list): + all_audio_chunks.extend(audio) + else: + all_audio_chunks.append(audio) + sr_raw = mm.get("sr") if mm else None + if sr_raw is not None and not (isinstance(sr_raw, (list, tuple)) and len(sr_raw) == 0): + sr_val = sr_raw[-1] if isinstance(sr_raw, list) and sr_raw else sr_raw + sample_rate = sr_val.item() if hasattr(sr_val, "item") else int(sr_val) + n = len(audio) if isinstance(audio, list) else (0 if audio is None else 1) + return sample_rate, n + + +def _build_omni_kwargs(args, model_name: str) -> dict: + """Build keyword arguments for Omni/AsyncOmni initialization.""" + omni_kwargs = dict( + model=model_name, + log_stats=args.log_stats, + stage_init_timeout=args.stage_init_timeout, + ) + if args.deploy_config: + omni_kwargs["deploy_config"] = args.deploy_config + else: + omni_kwargs["stage_configs_path"] = args.stage_configs_path or DEFAULT_STAGE_CONFIG + return omni_kwargs + + def main(args): """Run offline inference with Omni.""" output_dir = args.output_dir @@ -138,17 +175,7 @@ def main(args): ) ] - omni_kwargs = dict( - model=model_name, - log_stats=args.log_stats, - stage_init_timeout=args.stage_init_timeout, - ) - if args.deploy_config: - omni_kwargs["deploy_config"] = args.deploy_config - else: - omni_kwargs["stage_configs_path"] = args.stage_configs_path or DEFAULT_STAGE_CONFIG - - omni = Omni(**omni_kwargs) + omni = Omni(**_build_omni_kwargs(args, model_name)) t_start = time.perf_counter() for stage_outputs in omni.generate(inputs): @@ -178,17 +205,7 @@ async def main_streaming(args): model_name=model_name, ) - omni_kwargs = dict( - model=model_name, - log_stats=args.log_stats, - stage_init_timeout=args.stage_init_timeout, - ) - if args.deploy_config: - omni_kwargs["deploy_config"] = args.deploy_config - else: - omni_kwargs["stage_configs_path"] = args.stage_configs_path or DEFAULT_STAGE_CONFIG - - omni = AsyncOmni(**omni_kwargs) + omni = AsyncOmni(**_build_omni_kwargs(args, model_name)) request_id = "0" t_start = time.perf_counter() @@ -201,17 +218,7 @@ async def main_streaming(args): mm = stage_output.request_output.outputs[0].multimodal_output if not stage_output.finished: t_now = time.perf_counter() - audio = mm.get("audio") - if audio is not None: - if isinstance(audio, list): - all_audio_chunks.extend(audio) - else: - all_audio_chunks.append(audio) - sr_raw = mm.get("sr") - if sr_raw is not None: - sr_val = sr_raw[-1] if isinstance(sr_raw, list) and sr_raw else sr_raw - sample_rate = sr_val.item() if hasattr(sr_val, "item") else int(sr_val) - n = len(audio) if isinstance(audio, list) else (0 if audio is None else 1) + sample_rate, n = _collect_audio_chunk(mm, all_audio_chunks, sample_rate) dt_ms = (t_now - t_prev) * 1000 ttfa_ms = (t_now - t_start) * 1000 if chunk_idx == 0: @@ -222,16 +229,7 @@ async def main_streaming(args): chunk_idx += 1 else: # Collect any remaining audio in the final message - audio = mm.get("audio") if mm else None - if audio is not None: - if isinstance(audio, list): - all_audio_chunks.extend(audio) - else: - all_audio_chunks.append(audio) - sr_raw = mm.get("sr") if mm else None - if sr_raw is not None: - sr_val = sr_raw[-1] if isinstance(sr_raw, list) and sr_raw else sr_raw - sample_rate = sr_val.item() if hasattr(sr_val, "item") else int(sr_val) + sample_rate, _ = _collect_audio_chunk(mm, all_audio_chunks, sample_rate) t_end = time.perf_counter() total_ms = (t_end - t_start) * 1000 From b29042657a348864ef30aa6b3348593739ae3d60 Mon Sep 17 00:00:00 2001 From: Liangyx2 Date: Thu, 9 Jul 2026 10:21:13 +0800 Subject: [PATCH 4/5] Add XPU HW validation note for Fish Speech deploy config Add comment noting XPU platform overrides are verified on Intel Arc B60 (BMG, 24GB, 800 EUs). Signed-off-by: Liangyx2 --- vllm_omni/deploy/fish_qwen3_omni.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/vllm_omni/deploy/fish_qwen3_omni.yaml b/vllm_omni/deploy/fish_qwen3_omni.yaml index e8e5279786c..539b3777507 100644 --- a/vllm_omni/deploy/fish_qwen3_omni.yaml +++ b/vllm_omni/deploy/fish_qwen3_omni.yaml @@ -67,6 +67,7 @@ stages: platforms: xpu: stages: + # Verified on Intel Arc B60. - stage_id: 0 gpu_memory_utilization: 0.8 max_model_len: 2048 From 1f275d3ae7331de83e63d1621a2b6c72b21ff7c1 Mon Sep 17 00:00:00 2001 From: Yuxiang Date: Fri, 10 Jul 2026 09:09:32 +0800 Subject: [PATCH 5/5] Trigger CI Signed-off-by: Yuxiang