diff --git a/docs/source/models/supported-models.md b/docs/source/models/supported-models.md index 3771da0496b8..16db3e338971 100644 --- a/docs/source/models/supported-models.md +++ b/docs/source/models/supported-models.md @@ -114,6 +114,7 @@ Note: Support for other models may vary. Features marked "N/A" are not applicabl | `Step3p7ForConditionalGeneration` | Yes | Yes | Untested | Yes | Untested | Untested | Untested | Untested | L + I | | `MiniMaxM3SparseForConditionalGeneration` [^12] | Yes | Yes | Untested | Yes | Untested | No | Untested | Untested | L + I + V | | `Cosmos3ForConditionalGeneration` [^13] | Yes | Yes | Yes | Yes | Yes | Yes | Untested | Untested | L + I + V | +| `Qwen3_5ForConditionalGeneration` | Yes | Yes | Untested | Yes | Yes | No | Untested | Yes | L + I + V | | `Qwen3_5MoeForConditionalGeneration` | Yes | Yes | Untested | Yes | Yes | No | Untested | Yes | L + I + V | Note: diff --git a/tensorrt_llm/_torch/models/__init__.py b/tensorrt_llm/_torch/models/__init__.py index 3506906b46dd..e16beb5093f9 100644 --- a/tensorrt_llm/_torch/models/__init__.py +++ b/tensorrt_llm/_torch/models/__init__.py @@ -49,7 +49,7 @@ from .modeling_qwen2vl import Qwen2_5_VLModel, Qwen2VLModel from .modeling_qwen3 import Qwen3ForCausalLM from .modeling_qwen3_5 import (Qwen3_5ForCausalLM, Qwen3_5MoeForCausalLM, - Qwen3_5MoeVLModel) + Qwen3_5MoeVLModel, Qwen3_5VLModel) from .modeling_qwen3_moe import Qwen3MoeForCausalLM from .modeling_qwen3_next import Qwen3NextForCausalLM from .modeling_qwen3vl import Qwen3VLModel @@ -117,6 +117,7 @@ "Qwen3_5MoeForCausalLM", "QwenImageBenchModel", "Qwen3_5MoeVLModel", + "Qwen3_5VLModel", "Qwen3NextForCausalLM", "Qwen3MoeVLModel", "GptOssForCausalLM", diff --git a/tensorrt_llm/_torch/models/checkpoints/hf/qwen3_5_weight_mapper.py b/tensorrt_llm/_torch/models/checkpoints/hf/qwen3_5_weight_mapper.py index 71e46c1c9872..36570149aebd 100644 --- a/tensorrt_llm/_torch/models/checkpoints/hf/qwen3_5_weight_mapper.py +++ b/tensorrt_llm/_torch/models/checkpoints/hf/qwen3_5_weight_mapper.py @@ -20,6 +20,7 @@ @register_mapper("HF", "Qwen3_5MoeForCausalLM") @register_mapper("HF", "Qwen3_5MoeForConditionalGeneration") @register_mapper("HF", "Qwen3_5ForCausalLM") +@register_mapper("HF", "Qwen3_5ForConditionalGeneration") class Qwen3_5MoeHfWeightMapper(Qwen3NextHfWeightMapper): """Weight mapper for Qwen3.5 MoE text checkpoints. diff --git a/tensorrt_llm/_torch/models/modeling_qwen3_5.py b/tensorrt_llm/_torch/models/modeling_qwen3_5.py index 0b2c2a93b8fe..25a984ec8f8c 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3_5.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3_5.py @@ -330,16 +330,33 @@ def _normalize_qwen35_quantization_config(model_config) -> None: quantization_config["modules_to_not_convert"] = sorted(set(normalized_modules)) -def _normalize_qwen35_moe_vl_config(model_config) -> None: - """Adapt HF Qwen3.5-MoE VLM config to TRT-LLM runtime conventions.""" +# Map the inner (text) causal-LM arch to the outer VLM arch, used only for the +# defensive fallback when a config arrives without an `architectures` field. +_INNER_TO_OUTER_VL_ARCH = { + "Qwen3_5MoeForCausalLM": "Qwen3_5MoeForConditionalGeneration", + "Qwen3_5ForCausalLM": "Qwen3_5ForConditionalGeneration", +} + + +def _normalize_qwen35_vl_config(model_config, inner_arch: str) -> None: + """Adapt an HF Qwen3.5 VLM config (MoE or dense) to TRT-LLM conventions. + + Shared by both the MoE (`Qwen3_5MoeForConditionalGeneration` -> + `Qwen3_5MoeForCausalLM`) and dense (`Qwen3_5ForConditionalGeneration` -> + `Qwen3_5ForCausalLM`) VLM paths. The only difference between the two is the + inner causal-LM arch string written onto `text_config`; everything else + (mRoPE flattening, Qwen3Next text aliases, quantization exclude-module + rewrite) is identical. `_normalize_qwen35_qwen3next_text_aliases` is a no-op + for dense (its native `intermediate_size` is already present). + """ if not getattr(model_config, "architectures", None): - model_config.architectures = ["Qwen3_5MoeForConditionalGeneration"] + model_config.architectures = [_INNER_TO_OUTER_VL_ARCH.get(inner_arch, inner_arch)] text_config = getattr(model_config, "text_config", None) if text_config is None: - raise ValueError("Qwen3.5-MoE VLM config is missing text_config") + raise ValueError("Qwen3.5 VLM config is missing text_config") - text_config.architectures = ["Qwen3_5MoeForCausalLM"] + text_config.architectures = [inner_arch] _normalize_qwen35_qwen3next_text_aliases(text_config) _normalize_qwen35_mrope_config(text_config) @@ -347,6 +364,11 @@ def _normalize_qwen35_moe_vl_config(model_config) -> None: _normalize_qwen35_quantization_config(model_config) +def _normalize_qwen35_moe_vl_config(model_config) -> None: + """Adapt HF Qwen3.5-MoE VLM config to TRT-LLM runtime conventions.""" + _normalize_qwen35_vl_config(model_config, inner_arch="Qwen3_5MoeForCausalLM") + + def _normalize_qwen35_exclude_modules(model_config): """Normalize NVFP4/FP8 exclude_modules from HF naming to TRT-LLM naming. @@ -435,25 +457,41 @@ def __init__(self, model_config): super().__init__(model_config) -# TODO: Add tests for disaggregated support. -@support_multimodal_disaggregated -@register_vision_encoder(Qwen3VisionModelBase, vlm_base_model=Qwen3VisionModel) -@register_auto_model("Qwen3_5MoeForConditionalGeneration") -@register_input_processor( - Qwen3VLInputProcessorBase, - model_type="qwen3_5_moe", - placeholder_metadata=MultimodalPlaceholderMetadata( - placeholder_map={ - "image": "<|vision_start|><|image_pad|><|vision_end|>", - "video": "<|vision_start|><|video_pad|><|vision_end|>", - }, - placeholder_placement=MultimodalPlaceholderPlacement.BEFORE_TEXT, - placeholders_separator="", - content_format=ContentFormat.STRING, - ), +# Shared placeholder metadata for both Qwen3.5 VLM variants. The image/video +# placeholder layout is identical for MoE and dense; only the registration +# `model_type` differs (set per concrete class below). +_QWEN3_5_VL_PLACEHOLDER_METADATA = MultimodalPlaceholderMetadata( + placeholder_map={ + "image": "<|vision_start|><|image_pad|><|vision_end|>", + "video": "<|vision_start|><|video_pad|><|vision_end|>", + }, + placeholder_placement=MultimodalPlaceholderPlacement.BEFORE_TEXT, + placeholders_separator="", + content_format=ContentFormat.STRING, ) -class Qwen3_5MoeVLModel(Qwen3VLModelBase): - """VLM wrapper composing Qwen3 vision encoder with Qwen3.5 MoE text decoder.""" + + +class _Qwen3_5VLModel(Qwen3VLModelBase): + """Shared VLM wrapper composing the Qwen3 vision encoder with a Qwen3.5 + (Qwen3Next-based) text decoder. + + MoE and dense differ only in the inner causal-LM the config normalizer + selects (`Qwen3_5MoeForCausalLM` vs `Qwen3_5ForCausalLM`) — both reuse the + same vision tower, weight mapper, and forward path, so the wrapper body is + shared here. The concrete subclasses below carry only the registration + decorators (outer arch string + input-processor `model_type`). + """ + + @classmethod + def get_model_defaults(cls, llm_args): + # `ModelLoader` applies `get_model_defaults()` on the resolved outer + # model class (this VLM wrapper), not on the inner decoder. Both + # inner LMs (`Qwen3_5MoeForCausalLM` / `Qwen3_5ForCausalLM`) inherit + # `Qwen3NextForCausalLM`'s defaults unchanged, so delegate to it to + # propagate `enable_block_reuse=False` — the hybrid Mamba/SSM path + # doesn't support KV-cache block reuse. Without this the VLM path + # would silently fall back to the global default (block reuse on). + return Qwen3NextForCausalLM.get_model_defaults(llm_args) def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, **kwargs): kwargs["vision_model_class"] = Qwen3VisionModel @@ -481,3 +519,35 @@ def load_weights(self, weights: Dict[str, torch.Tensor], weight_mapper: BaseWeig r"^model\.language_model\.(.*)$": r"model.\1", } self.llm.load_weights(filtered_weights, weight_mapper, params_map=params_map) + + +# TODO(TRTLLM-13417): Add tests for disaggregated support. +@support_multimodal_disaggregated +@register_vision_encoder(Qwen3VisionModelBase, vlm_base_model=Qwen3VisionModel) +@register_auto_model("Qwen3_5MoeForConditionalGeneration") +@register_input_processor( + Qwen3VLInputProcessorBase, + model_type="qwen3_5_moe", + placeholder_metadata=_QWEN3_5_VL_PLACEHOLDER_METADATA, +) +class Qwen3_5MoeVLModel(_Qwen3_5VLModel): + """VLM wrapper composing Qwen3 vision encoder with Qwen3.5 MoE text decoder.""" + + +# TODO(TRTLLM-13417): Add tests for disaggregated support. +@support_multimodal_disaggregated +@register_vision_encoder(Qwen3VisionModelBase, vlm_base_model=Qwen3VisionModel) +@register_auto_model("Qwen3_5ForConditionalGeneration") +@register_input_processor( + Qwen3VLInputProcessorBase, + model_type="qwen3_5", + placeholder_metadata=_QWEN3_5_VL_PLACEHOLDER_METADATA, +) +class Qwen3_5VLModel(_Qwen3_5VLModel): + """VLM wrapper composing Qwen3 vision encoder with dense Qwen3.5 text decoder. + + Dense sibling of `Qwen3_5MoeVLModel` (arch `Qwen3_5ForConditionalGeneration`, + `model_type="qwen3_5"`). Same hybrid Qwen3Next runtime, with `GatedMLP` + instead of `SparseMoeBlock` (the dense text config has a native `intermediate_size` + and no `num_experts`). + """ diff --git a/tensorrt_llm/_torch/models/modeling_qwen3vl.py b/tensorrt_llm/_torch/models/modeling_qwen3vl.py index fec1f1cbf59d..9c4e46321704 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3vl.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3vl.py @@ -1151,6 +1151,7 @@ def __init__( "QwenImageBenchForConditionalGeneration": "Qwen3_5ForCausalLM", "Cosmos3ForConditionalGeneration": "Qwen3ForCausalLM", "Qwen3_5MoeForConditionalGeneration": "Qwen3_5MoeForCausalLM", + "Qwen3_5ForConditionalGeneration": "Qwen3_5ForCausalLM", } llm_arch = vlm_to_llm_arch.get(self.original_arch) if llm_arch is None: @@ -1225,6 +1226,26 @@ def vocab_size_padded(self) -> int: def infer_max_seq_len(self) -> int: return self.llm.infer_max_seq_len() + # Draft-model (two-model speculative decoding, e.g. DFlash / Eagle3) + # delegation: `ModelLoader.load` reads `draft_config` / `draft_model` and + # calls `load_draft_weights` on the *outer* model it resolved, but the + # spec-decoding wrapper (`SpecDecOneEngineForCausalLM`) is applied to the + # inner `self.llm` when this VLM composes it. Composite checkpoints + # (e.g. Qwen3.5-4B publishes text_config + vision_config) route text-only + # spec tests through this wrapper, so surface the inner LM's draft state. + # Note: `load_draft_weights` must keep an explicit signature — the loader + # dispatches kwargs via `inspect.getfullargspec`. + @property + def draft_config(self): + return self.llm.draft_config + + @property + def draft_model(self): + return self.llm.draft_model + + def load_draft_weights(self, weights: Dict, weight_mapper: Optional[BaseWeightMapper] = None): + return self.llm.load_draft_weights(weights, weight_mapper=weight_mapper) + def apply_llm_torch_compile(self, *, backend: Any, fullgraph: bool) -> None: # TODO: Move this hook to MultimodalModelMixin once multimodal models # consistently expose an LLM compile contract. diff --git a/tensorrt_llm/_torch/pyexecutor/config_utils.py b/tensorrt_llm/_torch/pyexecutor/config_utils.py index 806c757ee40c..a0980bd4cea5 100644 --- a/tensorrt_llm/_torch/pyexecutor/config_utils.py +++ b/tensorrt_llm/_torch/pyexecutor/config_utils.py @@ -418,10 +418,24 @@ def load_pretrained_config(model_name_or_path: str, and architectures[0] == "Qwen3_5MoeForConditionalGeneration"))): # Qwen3.5-MoE VLM: HF native composite config + model-side normalizer. from tensorrt_llm._torch.models.modeling_qwen3_5 import \ - _normalize_qwen35_moe_vl_config + _normalize_qwen35_vl_config model_config = transformers.Qwen3_5MoeConfig.from_pretrained( model_name_or_path, **kwargs) - _normalize_qwen35_moe_vl_config(model_config) + _normalize_qwen35_vl_config(model_config, + inner_arch="Qwen3_5MoeForCausalLM") + elif (model_type == "qwen3_5" and + (("text_config" in config_dict and "vision_config" in config_dict) or + (architectures + and architectures[0] == "Qwen3_5ForConditionalGeneration"))): + # Qwen3.5 dense VLM: HF native composite config + model-side normalizer. + # Must precede the text-only `qwen3_5` branch below so the composite + # config isn't flattened and vision_config dropped. + from tensorrt_llm._torch.models.modeling_qwen3_5 import \ + _normalize_qwen35_vl_config + model_config = transformers.Qwen3_5Config.from_pretrained( + model_name_or_path, **kwargs) + _normalize_qwen35_vl_config(model_config, + inner_arch="Qwen3_5ForCausalLM") elif model_type in _CONFIG_REGISTRY: config_class = _CONFIG_REGISTRY[model_type] model_config = config_class.from_pretrained(model_name_or_path, diff --git a/tests/integration/defs/accuracy/references/mmmu.yaml b/tests/integration/defs/accuracy/references/mmmu.yaml index a7a0cc0aa8ce..8134b6e5965c 100644 --- a/tests/integration/defs/accuracy/references/mmmu.yaml +++ b/tests/integration/defs/accuracy/references/mmmu.yaml @@ -71,6 +71,8 @@ Qwen/Qwen3-VL-8B-Instruct: - accuracy: 55.11 mistralai/Mistral-Small-3.1-24B-Instruct-2503: - accuracy: 57.0 +Qwen/Qwen3.5-27B: + - accuracy: 62.222 Qwen/Qwen3.5-35B-A3B: # The default accuracy for `test_auto_dtype` tests. - accuracy: 59.0 diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py index 6a551b7b7348..a26f330537c8 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py @@ -518,6 +518,36 @@ def test_fp8_prequantized(self) -> None: task.evaluate(llm, sampling_params=self.sampling_params) +@skip_pre_hopper +@pytest.mark.skip_less_device_memory(80000) +class TestQwen3_5_27B_VL(LlmapiAccuracyTestHarness): + MODEL_NAME = "Qwen/Qwen3.5-27B" + MODEL_PATH = f"{llm_models_root()}/Qwen3.5-27B" + MAX_NUM_TOKENS = 16384 + MAX_BATCH_SIZE = 32 + + sampling_params = SamplingParams( + max_tokens=MAX_NUM_TOKENS, + truncate_prompt_tokens=MMMU.MAX_INPUT_LEN, + stop="<|endoftext|>", + ) + + kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.6, enable_block_reuse=False) + + def _make_llm(self, model_path: str) -> LLM: + return LLM( + model_path, + max_num_tokens=self.MAX_NUM_TOKENS, + max_batch_size=self.MAX_BATCH_SIZE, + kv_cache_config=self.kv_cache_config, + ) + + def test_auto_dtype(self) -> None: + with self._make_llm(self.MODEL_PATH) as llm: + task = MMMU(self.MODEL_NAME) + task.evaluate(llm, sampling_params=self.sampling_params) + + class TestQwen3VL(LlmapiAccuracyTestHarness): MODEL_NAME = "Qwen/Qwen3-VL-8B-Instruct" MODEL_PATH = f"{llm_models_root()}/Qwen3/Qwen3-VL-8B-Instruct" diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 18938020d2d2..d2b07d90d3e0 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -823,6 +823,7 @@ accuracy/test_llm_api_pytorch_multimodal.py::TestStep3_7::test_nvfp4[mtp_nextn=0 accuracy/test_llm_api_pytorch_multimodal.py::TestStep3_7::test_nvfp4[mtp_nextn=3] TIMEOUT (120) accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3_5_35B_A3B_VL::test_auto_dtype accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3_5_35B_A3B_VL::test_fp8_prequantized +accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3_5_27B_VL::test_auto_dtype accuracy/test_llm_api_pytorch_multimodal.py::TestVILA1_5_3B::test_auto_dtype accuracy/test_llm_api_pytorch_ray.py::TestLlama3_1_8BInstruct::test_pp2_ray unittest/disaggregated/test_openai_disagg_server.py diff --git a/tests/integration/test_lists/test-db/l0_h100.yml b/tests/integration/test_lists/test-db/l0_h100.yml index cc9f83b31d43..5a23db5804a1 100644 --- a/tests/integration/test_lists/test-db/l0_h100.yml +++ b/tests/integration/test_lists/test-db/l0_h100.yml @@ -65,6 +65,8 @@ l0_h100: # test must run on Hopper-or-newer GPUs. Peer Qwen3-VL / Qwen3-VL-MoE # tests stay on L40s because they're pure attention and don't trigger the GDN kernel. - unittest/_torch/modeling/test_modeling_qwen3_5_vl_moe.py::TestQwen3_5MoeVL::test_all + # Dense Qwen3.5-VL is the same hybrid family (sm90+ GDN kernel), so it lands here too. + - unittest/_torch/modeling/test_modeling_qwen3_5_vl.py::TestQwen3_5VL::test_all - unittest/disaggregated/test_disagg_utils.py - unittest/disaggregated/test_router.py - unittest/disaggregated/test_remoteDictionary.py diff --git a/tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl.py b/tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl.py new file mode 100644 index 000000000000..fe097dabd79a --- /dev/null +++ b/tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl.py @@ -0,0 +1,425 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json +import os +from copy import deepcopy +from pathlib import Path +from typing import List, Optional + +import torch +import transformers +from test_modeling_multimodal import MultimodalScenario, TestModelingMultimodal +from transformers import Qwen3_5ForConditionalGeneration as HFQwen3_5ForConditionalGeneration +from utils.llm_data import llm_models_root +from utils.util import skip_pre_hopper + +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.models import Qwen3_5VLModel +from tensorrt_llm._torch.models.checkpoints.auto_mapper import AutoCheckpointMapper +from tensorrt_llm._torch.models.checkpoints.hf.qwen3_5_weight_mapper import Qwen3_5MoeHfWeightMapper +from tensorrt_llm._torch.models.modeling_auto import AutoModelForCausalLM +from tensorrt_llm._torch.models.modeling_qwen3_5 import _normalize_qwen35_vl_config +from tensorrt_llm._torch.pyexecutor.config_utils import ( + extract_mamba_kv_cache_params, + load_pretrained_config, +) +from tensorrt_llm._torch.pyexecutor.model_loader import validate_and_set_mamba_ssm_cache_dtype +from tensorrt_llm.inputs import ContentFormat +from tensorrt_llm.inputs.registry import MULTIMODAL_PLACEHOLDER_REGISTRY + +# Dense sibling of test_modeling_qwen3_5_vl_moe.py. The dense Qwen3.5-VL +# (Qwen/Qwen3.5-27B, arch Qwen3_5ForConditionalGeneration, model_type qwen3_5) +# reuses the same hybrid Qwen3Next runtime as the MoE variant, differing only in +# the feed-forward block (GatedMLP instead of SparseMoeBlock). This file mirrors +# the MoE routing + parity tests with a dense synthetic config. +# +# Two dense-specific differences from the MoE config: +# - No MoE fields (num_experts / moe_intermediate_size / ...); a native +# `intermediate_size` is present and must be preserved by the normalizer +# (the Qwen3Next-alias synthesis is a no-op for dense). +# - `deepstack_visual_indexes: []` matches the real dense checkpoint (as it +# does for the MoE one — the Qwen3.5 family dropped Qwen3-VL's deepstack). +# `use_deepstack` stays truthy via `hasattr`, but +# `deepstack_num_level == 0` makes the split a no-op. +# - `attn_output_gate: true` mirrors the real config and matches TRT-LLM's +# Qwen3NextAttention, which hardcodes output gating. + + +def _write_qwen35_dense_vl_config(tmp_path: Path) -> Path: + config = { + "architectures": ["Qwen3_5ForConditionalGeneration"], + "image_token_id": 248056, + "model_type": "qwen3_5", + "text_config": { + "attention_bias": False, + "attention_dropout": 0.0, + "attn_output_gate": True, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 2048, + "intermediate_size": 2048, + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "mlp_only_layers": [], + "model_type": "qwen3_5_text", + "num_attention_heads": 16, + "num_hidden_layers": 2, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-6, + "rope_parameters": { + "mrope_interleaved": True, + "mrope_section": [11, 11, 10], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000.0, + "rope_type": "default", + }, + "use_cache": True, + "vocab_size": 248320, + }, + "tie_word_embeddings": False, + "video_token_id": 248057, + "vision_config": { + "deepstack_visual_indexes": [], + "depth": 2, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "in_channels": 3, + "intermediate_size": 4304, + "model_type": "qwen3_5", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2048, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2, + }, + "vision_end_token_id": 248054, + "vision_start_token_id": 248053, + } + (tmp_path / "config.json").write_text(json.dumps(config), encoding="utf-8") + return tmp_path + + +def test_qwen35_dense_vl_config_preserves_vlm_architecture( + tmp_path: Path, +) -> None: + config = load_pretrained_config(str(_write_qwen35_dense_vl_config(tmp_path))) + + assert isinstance(config, transformers.Qwen3_5Config) + assert config.architectures == ["Qwen3_5ForConditionalGeneration"] + assert config.text_config.architectures == ["Qwen3_5ForCausalLM"] + # Dense: native intermediate_size is preserved (no MoE synthesis). + assert config.text_config.intermediate_size == 2048 + assert getattr(config.text_config, "num_experts", 0) in (0, None) + # Qwen3.5 family contract: deepstack stays disabled (empty) — both real + # checkpoints (dense 27B and MoE 35B-A3B) publish []. Normalization must + # not synthesize indices; the placeholder path depends on this staying + # empty (see re-greening item D in the MoE testing-notes doc). + assert config.vision_config.deepstack_visual_indexes == [] + assert config.text_config.rope_theta == 10000000.0 + assert config.text_config.partial_rotary_factor == 0.25 + assert config.text_config.rope_scaling["type"] == "mrope" + assert config.text_config.rope_scaling["mrope_section"] == [11, 11, 10] + # mrope_interleaved must survive normalization: the fused QK-norm-RoPE op + # gates the mRoPE path on it, and without it position_ids gets flattened + # to 3*num_tokens and mismatches the QKV token count. + assert config.text_config.rope_scaling["mrope_interleaved"] is True + assert config.text_config.mamba_ssm_dtype == "float32" + assert config.get_text_config() is config.text_config + + +def test_qwen35_dense_vl_resolves_mamba_ssm_cache_dtype( + tmp_path: Path, +) -> None: + config = load_pretrained_config(str(_write_qwen35_dense_vl_config(tmp_path))) + model_config = ModelConfig(pretrained_config=config) + + validate_and_set_mamba_ssm_cache_dtype(model_config, "auto") + assert model_config.quant_config.mamba_ssm_cache_dtype is torch.float32 + + mamba_params = extract_mamba_kv_cache_params( + config.text_config, + quant_config=model_config.quant_config, + ) + assert mamba_params.dtype is torch.bfloat16 + assert mamba_params.mamba_ssm_cache_dtype is torch.float32 + + +def test_qwen35_dense_vl_resolves_model_and_mapper(tmp_path: Path) -> None: + config = load_pretrained_config(str(_write_qwen35_dense_vl_config(tmp_path))) + model_config = ModelConfig(pretrained_config=config) + + assert AutoModelForCausalLM._resolve_class(model_config) is Qwen3_5VLModel + assert isinstance( + AutoCheckpointMapper.get("HF", "Qwen3_5ForConditionalGeneration"), + Qwen3_5MoeHfWeightMapper, + ) + + +def test_qwen35_dense_vl_placeholder_metadata_registered() -> None: + metadata = MULTIMODAL_PLACEHOLDER_REGISTRY.get_placeholder_metadata("qwen3_5") + + assert metadata.placeholder_map == { + "image": "<|vision_start|><|image_pad|><|vision_end|>", + "video": "<|vision_start|><|video_pad|><|vision_end|>", + } + assert metadata.placeholders_separator == "" + assert metadata.content_format is ContentFormat.STRING + + +# --- Layered parity test scaffold ------------------------------------------- +# +# Tiny synthetic config used by TestQwen3_5VL below. Same architecture as the +# real Qwen/Qwen3.5-27B checkpoint but with much smaller dimensions. The shape +# constraints are identical to the MoE parity config (see +# test_modeling_qwen3_5_vl_moe.py) except: +# +# - dense MLP: native `intermediate_size` (no MoE fields), so Qwen3NextModel +# selects GatedMLP for the feed-forward layers. +# - `deepstack_visual_indexes=[]` matches the real dense checkpoint (same +# as the MoE parity config — the Qwen3.5 family dropped deepstack); +# `depth=2` keeps the tower tiny since nothing pins its depth, and the +# config exercises the `deepstack_num_level == 0` no-op path. +# +# `_name_or_path` points at the real checkpoint dir so the test can load the +# tokenizer/processor (only the processor; not the full model weights). +QWEN3_5_VL_DENSE_PARITY_CONFIG = { + "architectures": ["Qwen3_5ForConditionalGeneration"], + "image_token_id": 248056, + "model_type": "qwen3_5", + "text_config": { + "attention_bias": False, + "attention_dropout": 0.0, + "attn_output_gate": True, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 2, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 2048, + "intermediate_size": 2048, + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 8192, + "mlp_only_layers": [], + "model_type": "qwen3_5_text", + "num_attention_heads": 16, + "num_hidden_layers": 2, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-6, + "rope_parameters": { + "mrope_interleaved": True, + "mrope_section": [11, 11, 10], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000.0, + "rope_type": "default", + }, + "use_cache": True, + "vocab_size": 248320, + }, + "tie_word_embeddings": False, + "video_token_id": 248057, + "vision_config": { + "deepstack_visual_indexes": [], + "depth": 2, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "model_type": "qwen3_5", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2048, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2, + }, + "vision_end_token_id": 248054, + "vision_start_token_id": 248053, + "_name_or_path": str(os.path.join(llm_models_root(), "Qwen3.5-27B")), +} + + +@skip_pre_hopper +class TestQwen3_5VL(TestModelingMultimodal): + """Forward-parity test for dense Qwen3.5-VL against HuggingFace. + + Dense sibling of `TestQwen3_5MoeVL`: both stacks are constructed from + `QWEN3_5_VL_DENSE_PARITY_CONFIG` (2 LM layers, 1 linear + 1 full attention, + dense GatedMLP, 2 vision layers, deepstack disabled), HF weights are copied + into TRT-LLM via `Qwen3_5MoeHfWeightMapper`, then `test_all` sweeps the + default `MultimodalScenario`s comparing last-position logits. + + Two-config design (same as the MoE test): `self.hf_config` stays raw HF + schema; TRT-LLM gets a deep-copied + normalized copy via the + `get_trtllm_pretrained_config` override, mirroring production + `load_pretrained_config` + (`_normalize_qwen35_vl_config(..., inner_arch="Qwen3_5ForCausalLM")`). + """ + + def get_model_config(self): + return QWEN3_5_VL_DENSE_PARITY_CONFIG + + def get_trtllm_model_class(self): + return Qwen3_5VLModel + + def get_hf_model_class(self): + return HFQwen3_5ForConditionalGeneration + + def get_weight_mapper_class(self): + return Qwen3_5MoeHfWeightMapper + + def get_model_type(self): + return "qwen3_5" + + def get_model_config_class(self): + return transformers.Qwen3_5Config + + def get_trtllm_pretrained_config(self) -> transformers.PretrainedConfig: + """Return a normalized config copy for TRT-LLM model construction. + + Mirrors the MoE test but passes `inner_arch="Qwen3_5ForCausalLM"` to the + shared normalizer so the dense text decoder is selected. + """ + trtllm_config = deepcopy(self.hf_config) + _normalize_qwen35_vl_config(trtllm_config, inner_arch="Qwen3_5ForCausalLM") + return trtllm_config + + def _dummy_request_kwargs(self, scenario): + """Qwen3.5-VL uses mRoPE; the cache manager needs the mRoPE + position-id buffer allocated at dummy-request time.""" + return {"use_mrope": True} + + def get_tolerance(self): + """Tighten `rtol` to `0.1` (4x tighter than the base 0.4 default) + while keeping `atol` at `0.4` to absorb single-logit tail outliers. + Same band as the MoE parity test. + """ + return 0.4, 0.1 + + def get_trtllm_inputs( + self, + input_ids, + multimodal_params_list, + is_gen: bool = False, + num_cached_tokens_per_seq: Optional[List[int]] = None, + total_prompt_len: Optional[int] = None, + ): + """Override position_ids with mRoPE position IDs from the multimodal + params. Identical to the MoE test — the VLM wrapper feeds mRoPE-shaped + position IDs to the decoder, not the base class's range-based default. + """ + trtllm_inputs = super().get_trtllm_inputs( + input_ids, + multimodal_params_list, + is_gen, + num_cached_tokens_per_seq, + total_prompt_len=total_prompt_len, + ) + + if is_gen: + mrope_gen_position_ids = [] + for multimodal_param in multimodal_params_list: + mrope_gen_position_ids.append( + multimodal_param.multimodal_data["mrope_config"]["mrope_position_deltas"] + ) + mrope_gen_position_ids = torch.cat(mrope_gen_position_ids, dim=-1).to(self.device) + trtllm_inputs["position_ids"] = ( + (trtllm_inputs["position_ids"] + mrope_gen_position_ids) + .expand(3, -1, 1) + .to(self.device) + ) + gen_multimodal_params_list = [] + for multimodal_param in multimodal_params_list: + multimodal_param.strip_for_generation() + multimodal_param.to_device( + "multimodal_data", + self.device, + pin_memory=True, + target_keywords=["mrope_config.mrope_position_deltas"], + ) + gen_multimodal_params_list.append(multimodal_param) + trtllm_inputs["multimodal_params"] = gen_multimodal_params_list + # Cached-mRoPE read slots (added in #11943): the decode path reads + # per-request deltas from the cache by seq slot. + trtllm_inputs["mrope_delta_read_seq_slots"] = torch.arange( + len(multimodal_params_list), device=self.device, dtype=torch.long + ) + else: + # Mrope position ids. For chunked prefill / KV cache reuse we must + # mirror production `PyTorchModelEngine` and slice each request's + # full `mrope_position_ids` to the current chunk's token range — + # the fused QK-norm-RoPE op requires position_ids tokens to match + # the QKV token count. + chunk_len = input_ids.shape[-1] + if num_cached_tokens_per_seq is None: + begin_offsets = [0] * len(multimodal_params_list) + elif isinstance(num_cached_tokens_per_seq, int): + begin_offsets = [num_cached_tokens_per_seq] * len(multimodal_params_list) + else: + begin_offsets = list(num_cached_tokens_per_seq) + mrope_position_ids = [] + for multimodal_param, begin in zip(multimodal_params_list, begin_offsets): + full_mrope = multimodal_param.multimodal_data["mrope_config"]["mrope_position_ids"] + mrope_position_ids.append(full_mrope[:, :, begin : begin + chunk_len]) + position_ids = torch.cat(mrope_position_ids, dim=-1).to(self.device) + trtllm_inputs["position_ids"] = position_ids + # Cached-mRoPE write slots (added in #11943): the context path + # writes per-request deltas into the cache by seq slot. + trtllm_inputs["mrope_delta_write_seq_slots"] = torch.arange( + len(multimodal_params_list), device=self.device, dtype=torch.long + ) + + return trtllm_inputs + + def get_scenarios(self) -> List[MultimodalScenario]: + """Modality-sanity sweep (image / multiple_image / video). + + CUDA-graph capture is intentionally not exercised — the hybrid + (Mamba + attention) cache's SSM state buffer isn't threaded through the + harness graph-capture path. Same limitation as the MoE parity test. + """ + return [ + MultimodalScenario( + modality="image", + use_cuda_graph=False, + chunked_prefill=False, + kv_cache_reuse=False, + ), + MultimodalScenario( + modality="multiple_image", + use_cuda_graph=False, + chunked_prefill=False, + kv_cache_reuse=False, + ), + MultimodalScenario( + modality="video", + use_cuda_graph=False, + chunked_prefill=False, + kv_cache_reuse=False, + ), + ] + + def test_construction_and_weight_loading_smoke(self): + """Smoke test: setUp built HF + TRT-LLM models and copied HF weights + into TRT-LLM via the weight mapper. Detailed assertions on the + normalizer's outputs live in the routing tests above — this one just + confirms construction reached the end without exception. + """ + self.assertIsNotNone(self.hf_model) + self.assertIsNotNone(self.trtllm_model) + self.assertIsNotNone(self.model_config)