diff --git a/.github/vllm-main-verified.commit b/.github/vllm-main-verified.commit index 6be422f3d04b..3bc0c33c8aaa 100644 --- a/.github/vllm-main-verified.commit +++ b/.github/vllm-main-verified.commit @@ -1 +1 @@ -85c09e9885e346ea1612da30ebff5a75f67d2350 +54503ecec0f3ac31e5ecfc5f28652e4cc42307b5 diff --git a/vllm_ascend/distributed/kv_transfer/kv_p2p/mooncake_connector.py b/vllm_ascend/distributed/kv_transfer/kv_p2p/mooncake_connector.py index 9d8a0cf27e18..7405a8a9cdd2 100644 --- a/vllm_ascend/distributed/kv_transfer/kv_p2p/mooncake_connector.py +++ b/vllm_ascend/distributed/kv_transfer/kv_p2p/mooncake_connector.py @@ -1108,7 +1108,11 @@ def _get_group_kv_caches(self, group_idx: int, layer_indices: list[int] | None = if layer_indices is None: _, layer_indices = self.kv_group2layeridx[group_idx] layer_index_set = set(layer_indices) - num_attn_module = 2 if self.vllm_config.model_config.hf_text_config.model_type == "longcat_flash" else 1 + num_attn_module = ( + 2 + if self.vllm_config.model_config.hf_text_config.model_type in ("longcat_flash", "longcat_flash_ngram") + else 1 + ) from vllm.v1.worker.utils import extract_layer_index def layer_in_group(layer_name: str) -> bool: @@ -2081,7 +2085,11 @@ def _build_kv_group2layeridx(self) -> dict[int, tuple[dict[str, Any], list[int]] from vllm.v1.worker.utils import extract_layer_index kv_group2layeridx: dict[int, tuple[dict[str, Any], list[int]]] = {} - num_attn_module = 2 if self.vllm_config.model_config.hf_text_config.model_type == "longcat_flash" else 1 + num_attn_module = ( + 2 + if self.vllm_config.model_config.hf_text_config.model_type in ("longcat_flash", "longcat_flash_ngram") + else 1 + ) next_mtp_layer_idx = self.total_layers transfer_group_id = 0 for kv_cache_group_id, group_spec in enumerate(self.kv_cache_config.kv_cache_groups): diff --git a/vllm_ascend/distributed/kv_transfer/kv_p2p/mooncake_layerwise_connector.py b/vllm_ascend/distributed/kv_transfer/kv_p2p/mooncake_layerwise_connector.py index 2d779994dc37..801b4b904d2e 100644 --- a/vllm_ascend/distributed/kv_transfer/kv_p2p/mooncake_layerwise_connector.py +++ b/vllm_ascend/distributed/kv_transfer/kv_p2p/mooncake_layerwise_connector.py @@ -1336,7 +1336,11 @@ def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): if use_kv_buffer: self.create_kv_buffer(kv_buffer) - num_attn_module = 2 if self.vllm_config.model_config.hf_text_config.model_type == "longcat_flash" else 1 + num_attn_module = ( + 2 + if self.vllm_config.model_config.hf_text_config.model_type in ("longcat_flash", "longcat_flash_ngram") + else 1 + ) mtp_layer_name = "" for layer_name in kv_caches: if "mtp" in layer_name: diff --git a/vllm_ascend/patch/hunyuan_vl_processor_compat.py b/vllm_ascend/patch/hunyuan_vl_processor_compat.py index e25565e52f4f..315709365fa0 100644 --- a/vllm_ascend/patch/hunyuan_vl_processor_compat.py +++ b/vllm_ascend/patch/hunyuan_vl_processor_compat.py @@ -196,6 +196,66 @@ def call_hf_processor( hunyuan_vision.HunYuanVLMultiModalProcessor._call_hf_processor = call_hf_processor +def _patch_xdrope_input_positions(hunyuan_vision: Any) -> None: + """Guard IndexError when image_start_indices > image_grid_thw entries.""" + + def get_xdrope_input_positions( + self: Any, + input_tokens: list[int], + mm_features: list[Any], + ) -> Any: + from vllm.multimodal.inputs import MultiModalFeatureSpec + + kwargs = MultiModalFeatureSpec.gather_kwargs( + mm_features, + {"image_grid_thw"}, + ) + image_grid_thw = [item.tolist() for item in kwargs.get("image_grid_thw", [])] + + hf_config = self.config + image_start_token_id = hf_config.image_start_token_id + spatial_merge_size = hf_config.vision_config.spatial_merge_size + xd_num = len(hf_config.rope_scaling["xdrope_section"]) + + import torch + + input_tokens_tensor = torch.tensor(input_tokens) + image_start_indices = torch.argwhere(input_tokens_tensor == image_start_token_id).squeeze(1) + + p_index = torch.arange(len(input_tokens_tensor)) + w_index = torch.arange(len(input_tokens_tensor)) + h_index = torch.arange(len(input_tokens_tensor)) + t_index = torch.arange(len(input_tokens_tensor)) + for image_index in range(len(image_start_indices)): + if image_index >= len(image_grid_thw): + continue + pos = image_start_indices[image_index] + 2 + t, h, w = image_grid_thw[image_index] + _, llm_grid_h, llm_grid_w = ( + t, + h // spatial_merge_size, + w // spatial_merge_size, + ) + + token_num = (llm_grid_w + 1) * llm_grid_h + w_index[pos : pos + token_num].copy_( + torch.arange(0, llm_grid_w + 1).reshape(1, -1).expand(llm_grid_h, -1).reshape(-1) + ) + h_index[pos : pos + token_num].copy_( + torch.arange(0, llm_grid_h).reshape(-1, 1).expand(-1, llm_grid_w + 1).reshape(-1) + ) + t_index[pos : pos + token_num] = image_index + + if xd_num == 4: + llm_positions = torch.stack([p_index, w_index, h_index, t_index]) + elif xd_num == 3: + llm_positions = torch.stack([w_index, h_index, t_index]) + + return llm_positions + + hunyuan_vision.HunYuanVLForConditionalGeneration.get_xdrope_input_positions = get_xdrope_input_positions + + def install_hunyuan_vl_processor_compat() -> None: """Align both supported vLLM refs with Transformers 5.13 Hunyuan APIs.""" # Keep each target's native, image-token-only prompt replacement. The @@ -208,8 +268,8 @@ def install_hunyuan_vl_processor_compat() -> None: _patch_v024_processor_methods(v024_hunyuan_vision) return - if not _remove_stale_registry_entries(): - return + _remove_stale_registry_entries() from vllm.model_executor.models import hunyuan_vision as main_hunyuan_vision _patch_hunyuan_processor_loader(main_hunyuan_vision) + _patch_xdrope_input_positions(main_hunyuan_vision) diff --git a/vllm_ascend/patch/platform/patch_speculative_config.py b/vllm_ascend/patch/platform/patch_speculative_config.py index c27986884ccd..7ba0eaa0f472 100644 --- a/vllm_ascend/patch/platform/patch_speculative_config.py +++ b/vllm_ascend/patch/platform/patch_speculative_config.py @@ -111,7 +111,7 @@ def hf_config_override(hf_config: PretrainedConfig) -> PretrainedConfig: "architectures": ["Qwen3_5MoeMTP" if is_moe else "Qwen3_5MTP"], } ) - if hf_config.model_type == "longcat_flash": + if hf_config.model_type in ("longcat_flash", "longcat_flash_ngram"): hf_config.model_type = "longcat_flash_mtp" n_predict = getattr(hf_config, "num_nextn_predict_layers", 1) hf_config.update({"n_predict": n_predict, "architectures": ["LongCatFlashMTPModel"]}) diff --git a/vllm_ascend/quantization/modelslim_config.py b/vllm_ascend/quantization/modelslim_config.py index 14dd69d2ad65..99674395e28c 100644 --- a/vllm_ascend/quantization/modelslim_config.py +++ b/vllm_ascend/quantization/modelslim_config.py @@ -235,6 +235,11 @@ def _is_fused_moe_layer(layer: torch.nn.Module) -> bool: "experts": ["experts.0.gate_proj", "experts.0.up_proj", "experts.0.down_proj"], "fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"], }, + "longcat_flash_ngram": { + "gate_up_proj": ["gate_proj", "up_proj"], + "experts": ["experts.0.gate_proj", "experts.0.up_proj", "experts.0.down_proj"], + "fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"], + }, "minimax_m2": { "qkv_proj": [ "q_proj", diff --git a/vllm_ascend/worker/model_runner_v1.py b/vllm_ascend/worker/model_runner_v1.py index b8834178b762..fdc561e825d4 100644 --- a/vllm_ascend/worker/model_runner_v1.py +++ b/vllm_ascend/worker/model_runner_v1.py @@ -3984,7 +3984,7 @@ def initialize_kv_cache_tensors(self, kv_cache_config: KVCacheConfig) -> dict[st else: from vllm.v1.worker.utils import bind_kv_cache - num_attn_module = 2 if self.model_config.hf_text_config.model_type == "longcat_flash" else 1 + num_attn_module = 2 if self.model_config.hf_text_config.model_type in ("longcat_flash", "longcat_flash_ngram") else 1 bind_kv_cache( kv_caches, self.compilation_config.static_forward_context,