diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index 6006a5a90243..0596ed4a6927 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -2401,6 +2401,8 @@ def is_generation_model(model_architectures: List[str], is_embedding: bool = Fal "Qwen3_5MoeForConditionalGeneration", "Qwen4ExpForConditionalGeneration", "MuseGlimmerForConditionalGeneration", + "Qwen3VLForConditionalGeneration", + "Qwen3VLMoeForConditionalGeneration", "MuseSparkForConditionalGeneration", "KimiK3ForConditionalGeneration", "KimiK25ForConditionalGeneration", diff --git a/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py b/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py index 385f0990901c..b0f1b5b51c3a 100644 --- a/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py +++ b/python/sglang/srt/model_executor/cuda_graph_buffer_registry.py @@ -831,6 +831,7 @@ def build_prefill_registry( is_multimodal: bool = False, hidden_size: int = 0, embed_dtype: Optional[torch.dtype] = None, + deepstack_replay_width: int = 0, enable_mamba_track: bool = False, enable_num_token_non_padded: bool = False, require_gathered_buffer: bool = False, @@ -925,6 +926,17 @@ def _bs(bs: int, _mt: int) -> Tuple[int, ...]: copy_from_fb=False, ) ) + if is_multimodal and deepstack_replay_width > 0: + slots.append( + GraphSlot( + "input_deepstack_embeds", + lambda _bs2, mt: (mt, deepstack_replay_width), + embed_dtype, + axis="tokens", + padding_policy=PaddingPolicy.ZERO, + copy_from_fb=False, + ) + ) if enable_mamba_track: slots.append(GraphSlot("mamba_track_indices", _bs, torch.int64, axis="bs")) slots.append(GraphSlot("mamba_track_mask", _bs, torch.bool, axis="bs")) diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index fc7e93b9f22e..6eb6c8fb1a83 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -44,7 +44,15 @@ from collections.abc import Sequence from contextlib import contextmanager, nullcontext from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Optional, + Protocol, + Union, + runtime_checkable, +) import torch import tqdm @@ -241,6 +249,57 @@ def _slice_output_rows(output: Any, num_tokens: int) -> Any: raise TypeError(f"Unsupported full prefill CUDA graph output: {type(output)}") +@runtime_checkable +class _SupportsDeepStackReplay(Protocol): + def get_num_deepstack_embeddings(self) -> int: ... + + +def _resolve_deepstack_replay_width( + *, model: Any, hidden_size: int, uses_eager_prefill_tail: bool +) -> int: + # Only the eager-tail backends (BREAKABLE/FULL) bind and refresh this slot; + # tc_piecewise captures the outer forward and never reads it. + if not uses_eager_prefill_tail: + return 0 + if isinstance(model, _SupportsDeepStackReplay): + return hidden_size * model.get_num_deepstack_embeddings() + return 0 + + +def _refresh_deepstack_replay_slot( + slot: torch.Tensor, deepstack_embeds: Optional[torch.Tensor] +) -> None: + """Sync the captured DeepStack slot for this replay. + + Absence zeroes the slot so previous rows cannot bleed through the LM's + ``add_``; a non-empty tensor that does not fit raises instead — replaying + with zeros would silently serve the request without its visual part. + """ + if deepstack_embeds is None or deepstack_embeds.numel() == 0: + slot.zero_() + return + + num_tokens = slot.shape[0] + if ( + deepstack_embeds.ndim != slot.ndim + or deepstack_embeds.shape[0] > num_tokens + or deepstack_embeds.shape[1:] != slot.shape[1:] + or deepstack_embeds.dtype != slot.dtype + or deepstack_embeds.device != slot.device + ): + raise RuntimeError( + "input_deepstack_embeds does not fit the captured BCG replay slot: " + f"got shape={tuple(deepstack_embeds.shape)}, " + f"dtype={deepstack_embeds.dtype}, device={deepstack_embeds.device}; " + f"slot holds up to {num_tokens} rows of {tuple(slot.shape[1:])} " + f"{slot.dtype} on {slot.device}." + ) + + num_rows = deepstack_embeds.shape[0] + slot[:num_rows].copy_(deepstack_embeds) + slot[num_rows:].zero_() + + @dataclass(frozen=True) class _ChunkedPrefixCaptureBuffers: """Runner-owned tensors shared by every prefix and token-bucket variant. @@ -430,6 +489,12 @@ def __init__(self, model_runner: ModelRunner): self.mamba_track_enabled = self._is_mamba_track_enabled() + deepstack_replay_width = _resolve_deepstack_replay_width( + model=self.model_runner.model, + hidden_size=self.model_runner.model_config.hidden_size, + uses_eager_prefill_tail=self._uses_eager_prefill_tail(), + ) + # --- buffers --------------------------------------------------- # `hidden_size` here sizes only the multimodal `input_embeds` buffer, # which `general_mm_embed_routine` copies the merged text+media @@ -446,6 +511,7 @@ def __init__(self, model_runner: ModelRunner): hidden_size=input_embeds_hidden_size, dtype=self.model_runner.dtype, enable_mamba_track=self.mamba_track_enabled, + deepstack_replay_width=deepstack_replay_width, enable_input_embeds=(self.is_multimodal or self._use_draft_input_embeds), pp_size=self.pp_size, is_first_pp_rank=self.model_runner.pp_group.is_first_rank, @@ -471,6 +537,7 @@ def __init__(self, model_runner: ModelRunner): is_multimodal=self.is_multimodal, hidden_size=input_embeds_hidden_size, embed_dtype=self.model_runner.dtype, + deepstack_replay_width=deepstack_replay_width, enable_mamba_track=self.mamba_track_enabled, # FullCG always pads to a capture bucket. Models that mask padded # hidden rows need the live boundary even without expert parallelism. @@ -874,6 +941,13 @@ def _run_forward(self, forward_batch: ForwardBatch, num_tokens: int): kwargs = _build_layer_model_forward_kwargs( self.layer_model, forward_batch, pp_proxy_tensors ) + # DeepStack replay slot: bound by name like the other optional + # transformer inputs above, but sourced from the runner's + # capture buffer rather than the forward batch. + if self.buffer_registry.has_slot("input_deepstack_embeds"): + kwargs["input_deepstack_embeds"] = self.buffer_registry.get_slot( + "input_deepstack_embeds" + ).slice_for(1, num_tokens) if pp_proxy_tensors is not None: input_ids = None for embeds_name in ("input_embeds", "inputs_embeds"): @@ -2183,6 +2257,13 @@ def replay_layer_forward(*args, **layer_kwargs): and self.buffer_registry.has_slot("input_embeds") ): self._fill_input_embeds_slot(args, layer_kwargs, static_num_tokens) + if self.buffer_registry.has_slot("input_deepstack_embeds"): + _refresh_deepstack_replay_slot( + slot=self.buffer_registry.get_slot( + "input_deepstack_embeds" + ).slice_for(1, static_num_tokens), + deepstack_embeds=layer_kwargs.get("input_deepstack_embeds"), + ) hs = self.backend.replay(shape_key, static_forward_batch, **kwargs) hs = self._restore_qwen_bcg_hc_output(hs) return _slice_output_rows(hs, raw_num_tokens) if full_path else hs diff --git a/python/sglang/srt/model_executor/runner_utils/buffers.py b/python/sglang/srt/model_executor/runner_utils/buffers.py index 93d6a3b68565..6dae901b3e9f 100644 --- a/python/sglang/srt/model_executor/runner_utils/buffers.py +++ b/python/sglang/srt/model_executor/runner_utils/buffers.py @@ -280,6 +280,7 @@ class PrefillInputBuffers(ForwardInputBuffers): positions: torch.Tensor input_embeds: Optional[torch.Tensor] mrope_positions: Optional[torch.Tensor] + input_deepstack_embeds: Optional[torch.Tensor] pp_proxy_tensors: Optional[Dict[str, torch.Tensor]] @classmethod @@ -294,6 +295,7 @@ def create( hidden_size: int, dtype: torch.dtype, enable_mamba_track: bool, + deepstack_replay_width: int = 0, enable_input_embeds: Optional[bool] = None, pp_size: int = 1, is_first_pp_rank: bool = False, @@ -331,8 +333,15 @@ def create( if is_multimodal: mrope_positions = torch.zeros((3, max_num_tokens), dtype=torch.int64) + if deepstack_replay_width > 0: + input_deepstack_embeds = torch.zeros( + (max_num_tokens, deepstack_replay_width), dtype=dtype + ) + else: + input_deepstack_embeds = None else: mrope_positions = None + input_deepstack_embeds = None pp_proxy_tensors = ( allocate_pp_proxy_tensors( @@ -359,5 +368,6 @@ def create( positions=positions, input_embeds=input_embeds, mrope_positions=mrope_positions, + input_deepstack_embeds=input_deepstack_embeds, pp_proxy_tensors=pp_proxy_tensors, ) diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py index 171777a77e87..cc32a5e53e19 100644 --- a/python/sglang/srt/models/qwen3_vl.py +++ b/python/sglang/srt/models/qwen3_vl.py @@ -1411,6 +1411,10 @@ def __init__( # For EAGLE3 support self.capture_aux_hidden_states = False + # Opt-in for the prefill graph runner's DeepStack replay slot; 0 means no slot. + def get_num_deepstack_embeddings(self) -> int: + return self.num_deepstack_embeddings + def separate_deepstack_embeds(self, embedding): assert embedding.shape[-1] % (1 + self.num_deepstack_embeddings) == 0, ( f"hidden_state of {embedding.shape} should be divisible by ({1 + self.num_deepstack_embeddings})" diff --git a/test/registered/unit/model_executor/test_deepstack_replay_slot.py b/test/registered/unit/model_executor/test_deepstack_replay_slot.py new file mode 100644 index 000000000000..6aba9f0c13bf --- /dev/null +++ b/test/registered/unit/model_executor/test_deepstack_replay_slot.py @@ -0,0 +1,324 @@ +"""Unit tests for the optional DeepStack BCG replay slot. + +Covers the allocation contract (registry gating, buffer allocation, +buffer-to-registry adoption), the model capability opt-in, and the +per-replay refresh of the slot (``_refresh_deepstack_replay_slot``). + +CPU-only; the logic under test is GPU-agnostic. +""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock + +import torch + +from sglang.srt.model_executor.cuda_graph_buffer_registry import ( + build_prefill_registry, +) +from sglang.srt.model_executor.runner.prefill_cuda_graph_runner import ( + PrefillCudaGraphRunner, + _refresh_deepstack_replay_slot, + _resolve_deepstack_replay_width, + _SupportsDeepStackReplay, +) +from sglang.srt.model_executor.runner_utils.buffers import PrefillInputBuffers +from sglang.srt.models.qwen2_5_vl import Qwen2_5_VLForConditionalGeneration +from sglang.srt.models.qwen3 import Qwen3ForCausalLM +from sglang.srt.models.qwen3_vl import Qwen3VLForConditionalGeneration +from sglang.srt.models.qwen3_vl_moe import Qwen3VLMoeForConditionalGeneration +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +_DEVICE = torch.device("cpu") + + +def _reg(**overrides): + base = dict( + device=_DEVICE, + max_bs=1, + max_num_token=128, + cache_loc_dtype=torch.int32, + is_multimodal=True, + hidden_size=64, + embed_dtype=torch.bfloat16, + deepstack_replay_width=0, + ) + base.update(overrides) + return build_prefill_registry(**base) + + +def _buffers(**overrides): + base = dict( + device=_DEVICE, + max_bs=1, + max_num_tokens=128, + cache_loc_dtype=torch.int32, + is_multimodal=True, + hidden_size=64, + dtype=torch.bfloat16, + enable_mamba_track=False, + deepstack_replay_width=0, + ) + base.update(overrides) + return PrefillInputBuffers.create(**base) + + +class TestDeepStackReplaySlotRegistration(CustomTestCase): + def test_slot_registration_follows_the_gates(self): + cases = [ + (dict(), False), + (dict(deepstack_replay_width=192), True), + (dict(deepstack_replay_width=192, is_multimodal=False), False), + (dict(deepstack_replay_width=192, register_input_embeds=False), False), + ] + for overrides, expected in cases: + with self.subTest(**overrides): + reg = _reg(**overrides) + self.assertEqual(reg.has_slot("input_deepstack_embeds"), expected) + + def test_slot_shape_and_dtype_match_contract(self): + buf = _reg(deepstack_replay_width=192).get_slot("input_deepstack_embeds").buffer + self.assertEqual(buf.shape[-1], 192) + self.assertEqual(buf.dtype, torch.bfloat16) + + +class TestPrefillInputBuffersDeepStackField(CustomTestCase): + def test_buffer_allocation_follows_the_gates(self): + cases = [ + (dict(), False), + (dict(deepstack_replay_width=192), True), + (dict(deepstack_replay_width=192, is_multimodal=False), False), + ] + for overrides, expected in cases: + with self.subTest(**overrides): + buf = _buffers(**overrides).input_deepstack_embeds + if expected: + self.assertEqual(buf.shape, (128, 192)) + self.assertEqual(buf.dtype, torch.bfloat16) + else: + self.assertIsNone(buf) + + def test_registry_adopts_the_buffer_tensor(self): + # Adoption looks the slot up by name on the source; a field rename + # silently breaks the wiring without this pin. + buf = _buffers(deepstack_replay_width=192) + reg = build_prefill_registry( + device=_DEVICE, + max_bs=1, + max_num_token=128, + cache_loc_dtype=torch.int32, + is_multimodal=True, + hidden_size=64, + embed_dtype=torch.bfloat16, + deepstack_replay_width=192, + source=buf, + ) + self.assertIs( + reg.get_slot("input_deepstack_embeds").buffer, + buf.input_deepstack_embeds, + ) + + +class TestQwen3VLCapabilityOptIn(CustomTestCase): + def test_only_deepstack_capable_models_opt_in(self): + for cls, expected in [ + (Qwen3VLForConditionalGeneration, True), + (Qwen3VLMoeForConditionalGeneration, True), + (Qwen2_5_VLForConditionalGeneration, False), + (Qwen3ForCausalLM, False), + ]: + with self.subTest(cls=cls.__name__): + self.assertEqual(issubclass(cls, _SupportsDeepStackReplay), expected) + + def test_replay_width_requires_eager_tail_and_model_support(self): + class Model: + def get_num_deepstack_embeddings(self): + return 3 + + for model, uses_eager_prefill_tail, expected in ( + (Model(), True, 192), + # tc_piecewise never binds or refreshes the slot, so it gets none. + (Model(), False, 0), + (object(), True, 0), + ): + with self.subTest( + uses_eager_prefill_tail=uses_eager_prefill_tail, expected=expected + ): + self.assertEqual( + _resolve_deepstack_replay_width( + model=model, + hidden_size=64, + uses_eager_prefill_tail=uses_eager_prefill_tail, + ), + expected, + ) + + +class TestDeepStackReplaySlotRefresh(CustomTestCase): + """The slot persists across requests sharing a token bucket, so each + replay must fully define its contents.""" + + NUM_TOKENS = 8 + WIDTH = 192 + DTYPE = torch.bfloat16 + + def _slot(self) -> torch.Tensor: + return torch.zeros( + (self.NUM_TOKENS, self.WIDTH), dtype=self.DTYPE, device=_DEVICE + ) + + def _embeds(self, num_rows, value, dtype=None) -> torch.Tensor: + return torch.full( + (num_rows, self.WIDTH), value, dtype=dtype or self.DTYPE, device=_DEVICE + ) + + def test_no_stale_rows_survive_into_the_next_request(self): + # The LM applies the slot with ``add_``, not through attention, so + # uncleared rows corrupt real tokens instead of being masked out. + slot = self._slot() + _refresh_deepstack_replay_slot( + slot=slot, deepstack_embeds=self._embeds(self.NUM_TOKENS, 3.0) + ) + _refresh_deepstack_replay_slot(slot=slot, deepstack_embeds=self._embeds(3, 5.0)) + self.assertTrue(torch.all(slot[:3] == 5.0)) + self.assertTrue(torch.all(slot[3:] == 0.0)) + + _refresh_deepstack_replay_slot(slot=slot, deepstack_embeds=None) + self.assertTrue(torch.all(slot == 0.0)) + + def _malformed_inputs(self): + return [ + self._embeds(4, 1.0, dtype=torch.float32), + torch.full((4, 1), 1.0, dtype=self.DTYPE, device=_DEVICE), + self._embeds(self.NUM_TOKENS + 1, 1.0), + torch.tensor(1.0, dtype=self.DTYPE, device=_DEVICE), + ] + + def test_malformed_deepstack_fails_closed(self): + # copy_ silently casts dtype drift and broadcasts an (n, 1) source + # across the full width, so the explicit guard is the only check. + for bad in self._malformed_inputs(): + with self.subTest(shape=tuple(bad.shape), dtype=bad.dtype): + with self.assertRaises(RuntimeError): + _refresh_deepstack_replay_slot( + slot=self._slot(), deepstack_embeds=bad + ) + + def test_fail_closed_leaves_slot_unmodified(self): + # copy_ would have written every rejected input above (cast or + # broadcast), so they are what prove validate-before-write. + for bad in self._malformed_inputs(): + with self.subTest(shape=tuple(bad.shape), dtype=bad.dtype): + slot = self._slot() + _refresh_deepstack_replay_slot( + slot=slot, deepstack_embeds=self._embeds(self.NUM_TOKENS, 3.0) + ) + with self.assertRaises(RuntimeError): + _refresh_deepstack_replay_slot(slot=slot, deepstack_embeds=bad) + self.assertTrue(torch.all(slot == 3.0)) + + def test_empty_tensor_clears_like_none(self): + slot = self._slot() + _refresh_deepstack_replay_slot( + slot=slot, deepstack_embeds=self._embeds(self.NUM_TOKENS, 3.0) + ) + _refresh_deepstack_replay_slot(slot=slot, deepstack_embeds=self._embeds(0, 1.0)) + self.assertTrue(torch.all(slot == 0.0)) + + +class TestDeepStackReplaySlotWiring(CustomTestCase): + """Drives ``_execute_body_capture`` so the slot is exercised through the + replay closure as wired; the helper tests above cannot catch the refresh + call or the capture binding being dropped.""" + + NUM_TOKENS = 8 + WIDTH = 6 + DTYPE = torch.bfloat16 + + def _runner(self, replay): + runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner) + runner._is_full_backend = False + runner._qwen_bcg_hc_sidechannel = False + runner._use_draft_input_embeds = False + runner._input_embeds_arg_idx = None + runner.buffer_registry = build_prefill_registry( + device=_DEVICE, + max_bs=1, + max_num_token=self.NUM_TOKENS, + cache_loc_dtype=torch.int32, + is_multimodal=True, + hidden_size=4, + embed_dtype=self.DTYPE, + deepstack_replay_width=self.WIDTH, + ) + runner._fill_input_embeds_slot = Mock() + runner.backend = SimpleNamespace(replay=replay) + runner.layer_model = SimpleNamespace(forward=None) + runner._prefill_forward_context = lambda *_args, **_kwargs: nullcontext() + return runner + + def _drive(self, runner, deepstack_embeds): + def model_forward(ids, positions, batch, **_kwargs): + return runner.layer_model.forward( + ids, positions, batch, input_deepstack_embeds=deepstack_embeds + ) + + runner.model_runner = SimpleNamespace( + pp_group=SimpleNamespace(is_first_rank=True), + model=SimpleNamespace(forward=model_forward), + ) + batch = SimpleNamespace(input_ids=None, positions=None, mm_input_embeds=None) + return runner._execute_body_capture( + batch, + batch, + static_num_tokens=self.NUM_TOKENS, + raw_num_tokens=3, + shape_key=object(), + ) + + def _slot_contents(self, runner): + return runner.buffer_registry.get_slot("input_deepstack_embeds").buffer.clone() + + def test_replay_refreshes_the_captured_slot(self): + seen = {} + + def replay(*_args, **_kwargs): + seen["slot"] = self._slot_contents(runner) + return "replayed" + + runner = self._runner(replay) + full = torch.full((3, self.WIDTH), 3.0, dtype=self.DTYPE, device=_DEVICE) + self.assertEqual(self._drive(runner, full), "replayed") + self.assertTrue(torch.all(seen["slot"][:3] == 3.0)) + self.assertTrue(torch.all(seen["slot"][3:] == 0.0)) + + short = torch.full((2, self.WIDTH), 5.0, dtype=self.DTYPE, device=_DEVICE) + self._drive(runner, short) + self.assertTrue(torch.all(seen["slot"][:2] == 5.0)) + self.assertTrue(torch.all(seen["slot"][2:] == 0.0)) + + self._drive(runner, None) + self.assertTrue(torch.all(seen["slot"] == 0.0)) + + def test_malformed_input_prevents_replay(self): + replayed = [] + + def replay(*_args, **_kwargs): + replayed.append(True) + return "replayed" + + runner = self._runner(replay) + bad = torch.full((3, self.WIDTH), 1.0, dtype=torch.float32, device=_DEVICE) + with self.assertRaises(RuntimeError): + self._drive(runner, bad) + self.assertFalse(replayed) + self.assertIsNone(runner.layer_model.forward) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py index 37b07ba2cb0a..7b639e9d1488 100644 --- a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py +++ b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py @@ -537,7 +537,9 @@ def test_bcg_mtp_draft_replay_fills_hc_width_input_embeds_slot(self): runner._qwen_bcg_hc_sidechannel = False runner._qwen_bcg_mtp_draft = True runner._input_embeds_arg_idx = 3 - runner.buffer_registry = SimpleNamespace(has_slot=lambda _name: True) + runner.buffer_registry = SimpleNamespace( + has_slot=lambda name: name == "input_embeds" + ) slot = torch.empty((8, 2 * 4)) runner._fill_input_embeds_slot = Mock( side_effect=lambda args, *_: slot[:1].copy_(args[3]) diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index 023fb5b8b5b4..0e2543f7f13c 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -3076,7 +3076,7 @@ def _handled_args(self, *, architectures, is_multimodal, allowlisted): def test_multimodal_arch_disables_prefill_breakable(self): args = self._handled_args( - architectures=["Qwen3VLForConditionalGeneration"], + architectures=["Qwen2_5_VLForConditionalGeneration"], is_multimodal=True, allowlisted=False, ) @@ -3101,19 +3101,17 @@ def test_allowlist_membership(self): is_multimodal_breakable_cuda_graph_supported, ) - self.assertTrue( - is_multimodal_breakable_cuda_graph_supported( - ["Qwen3_5MoeForConditionalGeneration"] - ) - ) - self.assertTrue( - is_multimodal_breakable_cuda_graph_supported( - ["Qwen3_5ForConditionalGeneration"] - ) - ) + for arch in ( + "Qwen3_5MoeForConditionalGeneration", + "Qwen3_5ForConditionalGeneration", + "Qwen3VLForConditionalGeneration", + "Qwen3VLMoeForConditionalGeneration", + ): + with self.subTest(arch=arch): + self.assertTrue(is_multimodal_breakable_cuda_graph_supported([arch])) self.assertFalse( is_multimodal_breakable_cuda_graph_supported( - ["Qwen3VLForConditionalGeneration"] + ["Qwen2_5_VLForConditionalGeneration"] ) )