From 2ac8f5a4327b19be7641cfed4988b5a5a0872208 Mon Sep 17 00:00:00 2001 From: Meihan-chen Date: Mon, 2 Mar 2026 15:10:50 +0800 Subject: [PATCH 01/11] fix: adapt to upstream vLLM changes (15d76f7..6290470) Root causes: - CudagraphDispatcher.dispatch() disable_full replaced with valid_modes/invalid_modes (PR #34102) - compile_or_warm_up_model() now returns float compilation_time (PR #35503) - MMEncoderAttention forward methods added sequence_lengths param (PR #35564) - Removed auto-forcing of +rms_norm for sequence parallelism (PR #35410) Upstream commit range: 15d76f74e2fdb12a95ea00f0ca283acf6219a2b7..6290470843c131681e3e1318ae71070a34f33225 Co-Authored-By: Claude Code Signed-off-by: MrZ20 <2609716663@qq.com> --- .github/workflows/bot_pr_create.yaml | 2 +- .github/workflows/pr_test_full.yaml | 2 +- .github/workflows/pr_test_light.yaml | 6 +++--- .github/workflows/schedule_codecov_refresh.yaml | 2 +- vllm_ascend/ops/mm_encoder_attention.py | 1 + vllm_ascend/platform.py | 6 ++++++ vllm_ascend/worker/model_runner_v1.py | 9 +++++---- vllm_ascend/worker/worker.py | 3 ++- 8 files changed, 20 insertions(+), 11 deletions(-) diff --git a/.github/workflows/bot_pr_create.yaml b/.github/workflows/bot_pr_create.yaml index bb956c2a8fd..f2f878d46e2 100644 --- a/.github/workflows/bot_pr_create.yaml +++ b/.github/workflows/bot_pr_create.yaml @@ -37,7 +37,7 @@ jobs: steps: - name: Get vLLM version run: | - VLLM_COMMIT=15d76f74e2fdb12a95ea00f0ca283acf6219a2b7 + VLLM_COMMIT=6290470843c131681e3e1318ae71070a34f33225 echo "VLLM_COMMIT=https://github.com/vllm-project/vllm/commit/$VLLM_COMMIT" >> "$GITHUB_ENV" - name: Checkout repository diff --git a/.github/workflows/pr_test_full.yaml b/.github/workflows/pr_test_full.yaml index 2e9f78ced81..eaa4699f24e 100644 --- a/.github/workflows/pr_test_full.yaml +++ b/.github/workflows/pr_test_full.yaml @@ -75,7 +75,7 @@ jobs: name: e2e-full strategy: matrix: - vllm_version: [15d76f74e2fdb12a95ea00f0ca283acf6219a2b7, v0.16.0] + vllm_version: [6290470843c131681e3e1318ae71070a34f33225, v0.16.0] needs: [changes] if: ${{ needs.changes.outputs.e2e_tracker == 'true' || needs.changes.outputs.e2e_tracker == true }} uses: ./.github/workflows/_e2e_test.yaml diff --git a/.github/workflows/pr_test_light.yaml b/.github/workflows/pr_test_light.yaml index a95ffec46d4..a5e5c328bbb 100644 --- a/.github/workflows/pr_test_light.yaml +++ b/.github/workflows/pr_test_light.yaml @@ -41,7 +41,7 @@ jobs: lint: uses: ./.github/workflows/_pre_commit.yml with: - vllm: 15d76f74e2fdb12a95ea00f0ca283acf6219a2b7 + vllm: 6290470843c131681e3e1318ae71070a34f33225 changes: runs-on: linux-aarch64-a2b3-0 outputs: @@ -89,7 +89,7 @@ jobs: if: ${{ needs.lint.result == 'success' && (needs.changes.outputs.e2e_tracker == 'true' || needs.changes.outputs.ut_tracker == 'true') }} strategy: matrix: - vllm_version: [15d76f74e2fdb12a95ea00f0ca283acf6219a2b7, v0.16.0] + vllm_version: [6290470843c131681e3e1318ae71070a34f33225, v0.16.0] uses: ./.github/workflows/_unit_test.yaml with: vllm: ${{ matrix.vllm_version }} @@ -101,7 +101,7 @@ jobs: name: e2e-light strategy: matrix: - vllm_version: [15d76f74e2fdb12a95ea00f0ca283acf6219a2b7, v0.16.0] + vllm_version: [6290470843c131681e3e1318ae71070a34f33225, v0.16.0] # Note (yikun): If CI resource are limited we can split job into two chain jobs needs: [lint, changes] # only trigger e2e test after lint passed and the change is e2e related with pull request. diff --git a/.github/workflows/schedule_codecov_refresh.yaml b/.github/workflows/schedule_codecov_refresh.yaml index e693cf7bd18..4611e0db93d 100644 --- a/.github/workflows/schedule_codecov_refresh.yaml +++ b/.github/workflows/schedule_codecov_refresh.yaml @@ -33,7 +33,7 @@ jobs: name: refresh codecov strategy: matrix: - vllm_version: [15d76f74e2fdb12a95ea00f0ca283acf6219a2b7] + vllm_version: [6290470843c131681e3e1318ae71070a34f33225] uses: ./.github/workflows/_unit_test.yaml with: vllm: ${{ matrix.vllm_version }} diff --git a/vllm_ascend/ops/mm_encoder_attention.py b/vllm_ascend/ops/mm_encoder_attention.py index 4122c7f178e..9d497059698 100644 --- a/vllm_ascend/ops/mm_encoder_attention.py +++ b/vllm_ascend/ops/mm_encoder_attention.py @@ -96,6 +96,7 @@ def forward_oot( value: torch.Tensor, cu_seqlens: torch.Tensor | None = None, max_seqlen: torch.Tensor | None = None, # Only used for Flash Attention + sequence_lengths: torch.Tensor | None = None, ): bsz, q_len = query.size()[:2] kv_len = key.size(1) diff --git a/vllm_ascend/platform.py b/vllm_ascend/platform.py index 2d161a42eb2..0fbb335df22 100644 --- a/vllm_ascend/platform.py +++ b/vllm_ascend/platform.py @@ -361,6 +361,12 @@ def check_and_update_config(cls, vllm_config: VllmConfig) -> None: if get_ascend_device_type() != AscendDeviceType._310P: compilation_config.custom_ops = ["all"] + # Upstream removed auto-forcing of +rms_norm for SP (PR #35410). + # On Ascend, SP requires rms_norm to go through forward_oot to avoid + # calling the CUDA-only torch.ops._C.rms_norm kernel. + if enable_sp(vllm_config) and "+rms_norm" not in compilation_config.custom_ops: + compilation_config.custom_ops.append("+rms_norm") + if ascend_config.recompute_scheduler_enable: from vllm_ascend.core.recompute_scheduler import RecomputeSchedulerConfig diff --git a/vllm_ascend/worker/model_runner_v1.py b/vllm_ascend/worker/model_runner_v1.py index 0bbe13f5a23..91370ec5b48 100644 --- a/vllm_ascend/worker/model_runner_v1.py +++ b/vllm_ascend/worker/model_runner_v1.py @@ -1827,16 +1827,17 @@ def _determine_batch_execution_and_padding( # ruff: noqa: E731 dispatch_cudagraph = ( - lambda num_tokens, disable_full: self.cudagraph_dispatcher.dispatch( + lambda num_tokens, disable_full=False, valid_modes=None: self.cudagraph_dispatcher.dispatch( num_tokens=num_tokens, has_lora=has_lora, uniform_decode=uniform_decode, - disable_full=disable_full, + valid_modes={CUDAGraphMode.NONE} if force_eager else valid_modes, + invalid_modes={CUDAGraphMode.FULL} if disable_full else None, ) if not force_eager else (CUDAGraphMode.NONE, BatchDescriptor(num_tokens_padded)) ) - cudagraph_mode, batch_descriptor = dispatch_cudagraph(num_tokens_padded, use_cascade_attn or has_encoder_output) + cudagraph_mode, batch_descriptor = dispatch_cudagraph(num_tokens_padded, disable_full=use_cascade_attn or has_encoder_output) num_tokens_padded = batch_descriptor.num_tokens if enable_sp(self.vllm_config): assert batch_descriptor.num_tokens % self.vllm_config.parallel_config.tensor_parallel_size == 0, ( @@ -1858,7 +1859,7 @@ def _determine_batch_execution_and_padding( # Re-dispatch with DP padding cudagraph_mode, batch_descriptor = dispatch_cudagraph( num_tokens_padded, - disable_full=synced_cudagraph_mode <= CUDAGraphMode.PIECEWISE.value, + valid_modes={CUDAGraphMode(synced_cudagraph_mode)}, ) # Assert to make sure the agreed upon token count is correct otherwise # num_tokens_across_dp will no-longer be valid diff --git a/vllm_ascend/worker/worker.py b/vllm_ascend/worker/worker.py index d746b1cdde2..b8cbbdbd27a 100644 --- a/vllm_ascend/worker/worker.py +++ b/vllm_ascend/worker/worker.py @@ -430,7 +430,7 @@ def load_model(self) -> None: with context, set_current_vllm_config(self.vllm_config): self.model_runner.load_model() - def compile_or_warm_up_model(self) -> None: + def compile_or_warm_up_model(self) -> float: # Note: need to adapt for graph mode. warmup_sizes = (self.vllm_config.compilation_config.compile_sizes or []).copy() if not self.model_config.enforce_eager: @@ -462,6 +462,7 @@ def compile_or_warm_up_model(self) -> None: # Reset the seed to ensure that the random state is not affected by # the model initialization and profiling. set_random_seed(self.model_config.seed) + return self.vllm_config.compilation_config.compilation_time def _warm_up_atb(self): x = torch.rand((2, 4), dtype=torch.float16).npu() From 55d700a25dfed2fc7e94e00b8a19804d03deede6 Mon Sep 17 00:00:00 2001 From: Meihan-chen Date: Mon, 2 Mar 2026 19:21:04 +0800 Subject: [PATCH 02/11] fix: adapt to upstream vLLM changes (15d76f74e..6290470843) Root causes: - CudagraphDispatcher.dispatch() API changed: disable_full -> valid_modes/invalid_modes (#34102) - compile_or_warm_up_model() must return float compilation_time (#35503) - MMEncoderAttention.forward_oot() gained new sequence_lengths param (#34580) - +rms_norm no longer auto-forced for SP, breaks Ascend without CUDA _C ops (#35410) Upstream commit range: 15d76f74e2fdb12a95ea00f0ca283acf6219a2b7..6290470843c131681e3e1318ae71070a34f33225 Co-Authored-By: Claude Code Signed-off-by: MrZ20 <2609716663@qq.com> --- vllm_ascend/core/recompute_scheduler.py | 2 - vllm_ascend/platform.py | 13 +++++-- vllm_ascend/worker/model_runner_v1.py | 52 +++++++++++++++++-------- vllm_ascend/worker/worker.py | 2 + 4 files changed, 47 insertions(+), 22 deletions(-) diff --git a/vllm_ascend/core/recompute_scheduler.py b/vllm_ascend/core/recompute_scheduler.py index b1aaff7c88a..c4f39f02e78 100644 --- a/vllm_ascend/core/recompute_scheduler.py +++ b/vllm_ascend/core/recompute_scheduler.py @@ -22,7 +22,6 @@ from collections import defaultdict from dataclasses import dataclass, fields -from vllm._bc_linter import bc_linter_include from vllm.config import SchedulerConfig, VllmConfig from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorMetadata from vllm.distributed.kv_events import KVEventBatch @@ -73,7 +72,6 @@ class RecomputeReqInfo: client_index: int = 0 -@bc_linter_include @dataclass class RecomputeSchedulerOutput(SchedulerOutput): recomputed_reqs: list[RecomputeReqInfo] | None = None diff --git a/vllm_ascend/platform.py b/vllm_ascend/platform.py index 0fbb335df22..117bcd68a26 100644 --- a/vllm_ascend/platform.py +++ b/vllm_ascend/platform.py @@ -361,10 +361,15 @@ def check_and_update_config(cls, vllm_config: VllmConfig) -> None: if get_ascend_device_type() != AscendDeviceType._310P: compilation_config.custom_ops = ["all"] - # Upstream removed auto-forcing of +rms_norm for SP (PR #35410). - # On Ascend, SP requires rms_norm to go through forward_oot to avoid - # calling the CUDA-only torch.ops._C.rms_norm kernel. - if enable_sp(vllm_config) and "+rms_norm" not in compilation_config.custom_ops: + # Upstream removed automatic +rms_norm forcing for SP (PR #35410), + # but Ascend needs it because torch.ops._C.rms_norm (CUDA kernel) is + # not available. Re-force it so the custom op wrapper routes to the + # OOT (NPU) implementation. + if ( + compilation_config.pass_config.enable_sp + and "+rms_norm" not in compilation_config.custom_ops + and "-rms_norm" not in compilation_config.custom_ops + ): compilation_config.custom_ops.append("+rms_norm") if ascend_config.recompute_scheduler_enable: diff --git a/vllm_ascend/worker/model_runner_v1.py b/vllm_ascend/worker/model_runner_v1.py index 91370ec5b48..e800868af74 100644 --- a/vllm_ascend/worker/model_runner_v1.py +++ b/vllm_ascend/worker/model_runner_v1.py @@ -120,6 +120,7 @@ is_moe_model, lmhead_tp_enable, set_weight_prefetch_method, + vllm_version_is, ) from vllm_ascend.worker.npu_input_batch import NPUInputBatch from vllm_ascend.worker.pcp_utils import PCPManager @@ -1826,18 +1827,31 @@ def _determine_batch_execution_and_padding( has_lora = len(self.input_batch.lora_id_to_lora_request) > 0 if force_has_lora is None else force_has_lora # ruff: noqa: E731 - dispatch_cudagraph = ( - lambda num_tokens, disable_full=False, valid_modes=None: self.cudagraph_dispatcher.dispatch( - num_tokens=num_tokens, - has_lora=has_lora, - uniform_decode=uniform_decode, - valid_modes={CUDAGraphMode.NONE} if force_eager else valid_modes, - invalid_modes={CUDAGraphMode.FULL} if disable_full else None, - ) - if not force_eager - else (CUDAGraphMode.NONE, BatchDescriptor(num_tokens_padded)) - ) - cudagraph_mode, batch_descriptor = dispatch_cudagraph(num_tokens_padded, disable_full=use_cascade_attn or has_encoder_output) + if vllm_version_is("0.16.0"): + + def dispatch_cudagraph(num_tokens, disable_full=False, valid_modes=None): + if force_eager: + return (CUDAGraphMode.NONE, BatchDescriptor(num_tokens_padded)) + return self.cudagraph_dispatcher.dispatch( + num_tokens=num_tokens, + has_lora=has_lora, + uniform_decode=uniform_decode, + disable_full=disable_full, + ) + else: + + def dispatch_cudagraph(num_tokens, disable_full=False, valid_modes=None): + if force_eager: + return (CUDAGraphMode.NONE, BatchDescriptor(num_tokens_padded)) + return self.cudagraph_dispatcher.dispatch( + num_tokens=num_tokens, + has_lora=has_lora, + uniform_decode=uniform_decode, + valid_modes=valid_modes, + invalid_modes={CUDAGraphMode.FULL} if disable_full else None, + ) + + cudagraph_mode, batch_descriptor = dispatch_cudagraph(num_tokens_padded, use_cascade_attn or has_encoder_output) num_tokens_padded = batch_descriptor.num_tokens if enable_sp(self.vllm_config): assert batch_descriptor.num_tokens % self.vllm_config.parallel_config.tensor_parallel_size == 0, ( @@ -1857,10 +1871,16 @@ def _determine_batch_execution_and_padding( dp_rank = self.parallel_config.data_parallel_rank num_tokens_padded = int(num_tokens_across_dp[dp_rank].item()) # Re-dispatch with DP padding - cudagraph_mode, batch_descriptor = dispatch_cudagraph( - num_tokens_padded, - valid_modes={CUDAGraphMode(synced_cudagraph_mode)}, - ) + if vllm_version_is("0.16.0"): + cudagraph_mode, batch_descriptor = dispatch_cudagraph( + num_tokens_padded, + disable_full=synced_cudagraph_mode <= CUDAGraphMode.PIECEWISE.value, + ) + else: + cudagraph_mode, batch_descriptor = dispatch_cudagraph( + num_tokens_padded, + valid_modes={CUDAGraphMode(synced_cudagraph_mode)}, + ) # Assert to make sure the agreed upon token count is correct otherwise # num_tokens_across_dp will no-longer be valid assert batch_descriptor.num_tokens == num_tokens_padded diff --git a/vllm_ascend/worker/worker.py b/vllm_ascend/worker/worker.py index b8cbbdbd27a..423b6509849 100644 --- a/vllm_ascend/worker/worker.py +++ b/vllm_ascend/worker/worker.py @@ -464,6 +464,8 @@ def compile_or_warm_up_model(self) -> float: set_random_seed(self.model_config.seed) return self.vllm_config.compilation_config.compilation_time + return self.vllm_config.compilation_config.compilation_time + def _warm_up_atb(self): x = torch.rand((2, 4), dtype=torch.float16).npu() weight = torch.rand((2, 4), dtype=torch.float16).npu() From b93ec4d648b0fb9a57f2675d226fdc9d66382354 Mon Sep 17 00:00:00 2001 From: MrZ20 <2609716663@qq.com> Date: Tue, 3 Mar 2026 11:34:53 +0800 Subject: [PATCH 03/11] update Signed-off-by: MrZ20 <2609716663@qq.com> --- .github/workflows/bot_pr_create.yaml | 2 +- .github/workflows/pr_test_full.yaml | 2 +- .github/workflows/pr_test_light.yaml | 6 +++--- .github/workflows/schedule_codecov_refresh.yaml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/bot_pr_create.yaml b/.github/workflows/bot_pr_create.yaml index f2f878d46e2..4bf3fc082ef 100644 --- a/.github/workflows/bot_pr_create.yaml +++ b/.github/workflows/bot_pr_create.yaml @@ -37,7 +37,7 @@ jobs: steps: - name: Get vLLM version run: | - VLLM_COMMIT=6290470843c131681e3e1318ae71070a34f33225 + VLLM_COMMIT=4034c3d32e30d01639459edd3ab486f56993876d echo "VLLM_COMMIT=https://github.com/vllm-project/vllm/commit/$VLLM_COMMIT" >> "$GITHUB_ENV" - name: Checkout repository diff --git a/.github/workflows/pr_test_full.yaml b/.github/workflows/pr_test_full.yaml index eaa4699f24e..ffe53d18759 100644 --- a/.github/workflows/pr_test_full.yaml +++ b/.github/workflows/pr_test_full.yaml @@ -75,7 +75,7 @@ jobs: name: e2e-full strategy: matrix: - vllm_version: [6290470843c131681e3e1318ae71070a34f33225, v0.16.0] + vllm_version: [4034c3d32e30d01639459edd3ab486f56993876d, v0.16.0] needs: [changes] if: ${{ needs.changes.outputs.e2e_tracker == 'true' || needs.changes.outputs.e2e_tracker == true }} uses: ./.github/workflows/_e2e_test.yaml diff --git a/.github/workflows/pr_test_light.yaml b/.github/workflows/pr_test_light.yaml index a5e5c328bbb..d3c9a88148b 100644 --- a/.github/workflows/pr_test_light.yaml +++ b/.github/workflows/pr_test_light.yaml @@ -41,7 +41,7 @@ jobs: lint: uses: ./.github/workflows/_pre_commit.yml with: - vllm: 6290470843c131681e3e1318ae71070a34f33225 + vllm: 4034c3d32e30d01639459edd3ab486f56993876d changes: runs-on: linux-aarch64-a2b3-0 outputs: @@ -89,7 +89,7 @@ jobs: if: ${{ needs.lint.result == 'success' && (needs.changes.outputs.e2e_tracker == 'true' || needs.changes.outputs.ut_tracker == 'true') }} strategy: matrix: - vllm_version: [6290470843c131681e3e1318ae71070a34f33225, v0.16.0] + vllm_version: [4034c3d32e30d01639459edd3ab486f56993876d, v0.16.0] uses: ./.github/workflows/_unit_test.yaml with: vllm: ${{ matrix.vllm_version }} @@ -101,7 +101,7 @@ jobs: name: e2e-light strategy: matrix: - vllm_version: [6290470843c131681e3e1318ae71070a34f33225, v0.16.0] + vllm_version: [4034c3d32e30d01639459edd3ab486f56993876d, v0.16.0] # Note (yikun): If CI resource are limited we can split job into two chain jobs needs: [lint, changes] # only trigger e2e test after lint passed and the change is e2e related with pull request. diff --git a/.github/workflows/schedule_codecov_refresh.yaml b/.github/workflows/schedule_codecov_refresh.yaml index 4611e0db93d..dd4f2c8484b 100644 --- a/.github/workflows/schedule_codecov_refresh.yaml +++ b/.github/workflows/schedule_codecov_refresh.yaml @@ -33,7 +33,7 @@ jobs: name: refresh codecov strategy: matrix: - vllm_version: [6290470843c131681e3e1318ae71070a34f33225] + vllm_version: [4034c3d32e30d01639459edd3ab486f56993876d] uses: ./.github/workflows/_unit_test.yaml with: vllm: ${{ matrix.vllm_version }} From 08108561868749782f6d01bc74a48095e9433dd7 Mon Sep 17 00:00:00 2001 From: MrZ20 <2609716663@qq.com> Date: Tue, 3 Mar 2026 16:15:02 +0800 Subject: [PATCH 04/11] fix unit test Signed-off-by: MrZ20 <2609716663@qq.com> --- .../_310p/quantization/test_modelslim_config_310.py | 1 + tests/ut/distributed/test_communicator.py | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/ut/_310p/quantization/test_modelslim_config_310.py b/tests/ut/_310p/quantization/test_modelslim_config_310.py index 7e614e21f83..268b98907b3 100644 --- a/tests/ut/_310p/quantization/test_modelslim_config_310.py +++ b/tests/ut/_310p/quantization/test_modelslim_config_310.py @@ -70,6 +70,7 @@ def test_get_quant_method_for_fused_moe_310(self): fused_moe_layer = MagicMock(spec=FusedMoE) fused_moe_layer.moe = MagicMock(spec=FusedMoEConfig) fused_moe_layer.moe_config = MagicMock(spec=FusedMoEConfig) + fused_moe_layer.moe_config.moe_backend = "auto" fused_moe_layer.moe_config.moe_parallel_config = MagicMock(spec=FusedMoEParallelConfig) fused_moe_layer.moe_config.moe_parallel_config.use_ep = True fused_moe_layer.moe_config.moe_parallel_config.dp_size = 1 diff --git a/tests/ut/distributed/test_communicator.py b/tests/ut/distributed/test_communicator.py index c929741eb71..33325e57803 100644 --- a/tests/ut/distributed/test_communicator.py +++ b/tests/ut/distributed/test_communicator.py @@ -38,13 +38,14 @@ def patched_all_to_all(output_tensor_list, torch.tensor([50, 60]) ]) - torch.distributed.all_to_all = patched_all_to_all + dist.all_to_all = patched_all_to_all scatter_sizes = [2, 2] gather_sizes = [2, 2] input_ = torch.tensor([10, 20, 30, 40]) - comm = NPUCommunicator(cpu_group=dist.group.WORLD) + with patch.dict(dist.distributed_c10d._world.pg_map, {dist.group.WORLD: object()}, clear=False): + comm = NPUCommunicator(cpu_group=dist.group.WORLD) output = comm.all_to_all(input_, scatter_sizes=scatter_sizes, @@ -80,11 +81,12 @@ def patched_all_to_all(output_tensor_list, torch.tensor([[50, 60]]) ]) - torch.distributed.all_to_all = patched_all_to_all + dist.all_to_all = patched_all_to_all input_ = torch.tensor([[10, 20], [30, 40]]) - comm = NPUCommunicator(cpu_group=dist.group.WORLD) - output = comm.all_to_all(input_, scatter_dim=0, gather_dim=0) + with patch.dict(dist.distributed_c10d._world.pg_map, {dist.group.WORLD: object()}, clear=False): + comm = NPUCommunicator(cpu_group=dist.group.WORLD) + output = comm.all_to_all(input_, scatter_dim=0, gather_dim=0) assert output.tolist() == [[10, 20], [50, 60]] From a78e2c3e3a4674db21bb6d00605a48aa0b399410 Mon Sep 17 00:00:00 2001 From: MrZ20 <2609716663@qq.com> Date: Tue, 3 Mar 2026 20:49:57 +0800 Subject: [PATCH 05/11] fix Signed-off-by: MrZ20 <2609716663@qq.com> --- .../multicard/2-cards/test_aclgraph_capture_replay.py | 1 - vllm_ascend/platform.py | 11 ----------- vllm_ascend/worker/worker.py | 2 -- 3 files changed, 14 deletions(-) diff --git a/tests/e2e/multicard/2-cards/test_aclgraph_capture_replay.py b/tests/e2e/multicard/2-cards/test_aclgraph_capture_replay.py index 38a78b65e96..36b01e7dadd 100644 --- a/tests/e2e/multicard/2-cards/test_aclgraph_capture_replay.py +++ b/tests/e2e/multicard/2-cards/test_aclgraph_capture_replay.py @@ -132,7 +132,6 @@ def _run_worker_process( torch.npu.reset_peak_memory_stats() -@pytest.mark.skip(reason="fix me") @pytest.mark.parametrize("model", MODELS) @pytest.mark.parametrize("max_tokens", [4, 36]) @patch.dict(os.environ, {"ASCEND_RT_VISIBLE_DEVICES": "0,1"}) diff --git a/vllm_ascend/platform.py b/vllm_ascend/platform.py index 117bcd68a26..2d161a42eb2 100644 --- a/vllm_ascend/platform.py +++ b/vllm_ascend/platform.py @@ -361,17 +361,6 @@ def check_and_update_config(cls, vllm_config: VllmConfig) -> None: if get_ascend_device_type() != AscendDeviceType._310P: compilation_config.custom_ops = ["all"] - # Upstream removed automatic +rms_norm forcing for SP (PR #35410), - # but Ascend needs it because torch.ops._C.rms_norm (CUDA kernel) is - # not available. Re-force it so the custom op wrapper routes to the - # OOT (NPU) implementation. - if ( - compilation_config.pass_config.enable_sp - and "+rms_norm" not in compilation_config.custom_ops - and "-rms_norm" not in compilation_config.custom_ops - ): - compilation_config.custom_ops.append("+rms_norm") - if ascend_config.recompute_scheduler_enable: from vllm_ascend.core.recompute_scheduler import RecomputeSchedulerConfig diff --git a/vllm_ascend/worker/worker.py b/vllm_ascend/worker/worker.py index 423b6509849..b8cbbdbd27a 100644 --- a/vllm_ascend/worker/worker.py +++ b/vllm_ascend/worker/worker.py @@ -464,8 +464,6 @@ def compile_or_warm_up_model(self) -> float: set_random_seed(self.model_config.seed) return self.vllm_config.compilation_config.compilation_time - return self.vllm_config.compilation_config.compilation_time - def _warm_up_atb(self): x = torch.rand((2, 4), dtype=torch.float16).npu() weight = torch.rand((2, 4), dtype=torch.float16).npu() From 9feb0b256cea4879ec44d23ea5f7a24e4b579ebe Mon Sep 17 00:00:00 2001 From: gcanlin Date: Tue, 3 Mar 2026 15:20:10 +0000 Subject: [PATCH 06/11] add apply_config_platform_defaults Signed-off-by: gcanlin --- docs/source/community/versioning_policy.md | 2 +- vllm_ascend/platform.py | 27 ++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/source/community/versioning_policy.md b/docs/source/community/versioning_policy.md index 94b3059b0d3..7c0f5f30d01 100644 --- a/docs/source/community/versioning_policy.md +++ b/docs/source/community/versioning_policy.md @@ -57,7 +57,7 @@ For main branch of vLLM Ascend, we usually make it compatible with the latest vL | vLLM Ascend | vLLM | Python | Stable CANN | PyTorch/torch_npu | |-------------|--------------|------------------|-------------|--------------------| -| main | 4572a06afe96d0a6d5d3efacf130c71505dd2bc9, v0.16.0 tag | >= 3.10, < 3.12 | 8.5.0 | 2.9.0 / 2.9.0 | +| main | 4034c3d32e30d01639459edd3ab486f56993876d, v0.16.0 tag | >= 3.10, < 3.12 | 8.5.0 | 2.9.0 / 2.9.0 | ## Release cadence diff --git a/vllm_ascend/platform.py b/vllm_ascend/platform.py index 2d161a42eb2..d64596f8f72 100644 --- a/vllm_ascend/platform.py +++ b/vllm_ascend/platform.py @@ -450,6 +450,33 @@ def check_and_update_config(cls, vllm_config: VllmConfig) -> None: ): speculative_config.enforce_eager = False + @classmethod + def apply_config_platform_defaults(cls, vllm_config: VllmConfig) -> None: + """ + Apply the platform-specific default values to the config. + + This function is called during the initialization of global VllmConfig, + after parsing cli arguments but BEFORE sequence parallelism imports. + + We must disable enable_sp and fuse_gemm_comms here because the vLLM + sequence parallelism code imports matcher_utils.py which accesses + CUDA-specific torch.ops._C ops at module level, causing import errors + on NPU platform. + """ + # Disable sequence parallelism to prevent import of CUDA-specific ops + # in vllm/compilation/passes/fusion/matcher_utils.py + if vllm_config.compilation_config is not None: + pass_config = vllm_config.compilation_config.pass_config + if pass_config.enable_sp: + logger.info( + "Sequence Parallelism (enable_sp) is not supported on NPU, " + "disabling. Use vllm-ascend's Flash Comm instead." + ) + pass_config.enable_sp = False + if pass_config.fuse_gemm_comms: + logger.info("fuse_gemm_comms is not supported on NPU, disabling.") + pass_config.fuse_gemm_comms = False + @classmethod def import_kernels(cls) -> None: # Directly importing vllm_ascend_C prevents ASCEND_RT_VISIBLE_DEVICES From 24e4aef4914ec7ea52daedeafd744cc85d928d49 Mon Sep 17 00:00:00 2001 From: gcanlin Date: Tue, 3 Mar 2026 15:20:42 +0000 Subject: [PATCH 07/11] dockerfile Signed-off-by: gcanlin --- .github/workflows/dockerfiles/Dockerfile.lint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerfiles/Dockerfile.lint b/.github/workflows/dockerfiles/Dockerfile.lint index a5aee23d769..64068c64939 100644 --- a/.github/workflows/dockerfiles/Dockerfile.lint +++ b/.github/workflows/dockerfiles/Dockerfile.lint @@ -27,7 +27,7 @@ RUN apt-get update -y && \ ARG VLLM_REPO=https://github.com/vllm-project/vllm.git # For lint purpose, actually we need make a main2main matching. -ARG VLLM_COMMIT=15d76f74e2fdb12a95ea00f0ca283acf6219a2b7 +ARG VLLM_COMMIT=4034c3d32e30d01639459edd3ab486f56993876d RUN git clone $VLLM_REPO /vllm-workspace/vllm && \ cd /vllm-workspace/vllm && \ git checkout $VLLM_COMMIT From 8763085c5f7006220eb0d114533ab734fff63b29 Mon Sep 17 00:00:00 2001 From: MrZ20 <2609716663@qq.com> Date: Wed, 4 Mar 2026 14:47:02 +0800 Subject: [PATCH 08/11] fix Signed-off-by: MrZ20 <2609716663@qq.com> --- .../2-cards/test_aclgraph_capture_replay.py | 1 + vllm_ascend/patch/__init__.py | 15 +++++++++++ vllm_ascend/patch/platform/__init__.py | 1 + .../patch_fusion_matcher_compat_ops.py | 0 vllm_ascend/platform.py | 27 ------------------- 5 files changed, 17 insertions(+), 27 deletions(-) create mode 100644 vllm_ascend/patch/platform/patch_fusion_matcher_compat_ops.py diff --git a/tests/e2e/multicard/2-cards/test_aclgraph_capture_replay.py b/tests/e2e/multicard/2-cards/test_aclgraph_capture_replay.py index 36b01e7dadd..38a78b65e96 100644 --- a/tests/e2e/multicard/2-cards/test_aclgraph_capture_replay.py +++ b/tests/e2e/multicard/2-cards/test_aclgraph_capture_replay.py @@ -132,6 +132,7 @@ def _run_worker_process( torch.npu.reset_peak_memory_stats() +@pytest.mark.skip(reason="fix me") @pytest.mark.parametrize("model", MODELS) @pytest.mark.parametrize("max_tokens", [4, 36]) @patch.dict(os.environ, {"ASCEND_RT_VISIBLE_DEVICES": "0,1"}) diff --git a/vllm_ascend/patch/__init__.py b/vllm_ascend/patch/__init__.py index 48cabca293f..4d1c69e86d9 100644 --- a/vllm_ascend/patch/__init__.py +++ b/vllm_ascend/patch/__init__.py @@ -94,6 +94,21 @@ # Future Plan: # Remove this patch when vLLM merge the PR. # +# ** 6. File: platform/patch_fusion_matcher_compat_ops.py** +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# 1. `torch.ops._C.rms_norm`, `torch.ops._C.fused_add_rms_norm`, +# `torch.ops._C.rotary_embedding`, `torch.ops.vllm.flashinfer_rotary_embedding` +# Why: +# upstream vLLM initializes fusion matcher global operators at import time. +# On Ascend environment these symbols may be absent and cause import failure. +# How： +# inject placeholders only when the symbols are missing so import can continue. +# Related PR (if no, explain why): +# temporary compatibility patch before upstream adjustment is merged. +# Future Plan: +# remove this patch once upstream no longer requires these global symbols or +# provides a backend-safe initialization path. +# # * Worker Patch: # =============== # diff --git a/vllm_ascend/patch/platform/__init__.py b/vllm_ascend/patch/platform/__init__.py index 52d9a74b156..7d54d6cf39c 100644 --- a/vllm_ascend/patch/platform/__init__.py +++ b/vllm_ascend/patch/platform/__init__.py @@ -16,6 +16,7 @@ import os +import vllm_ascend.patch.platform.patch_fusion_matcher_compat_ops # noqa import vllm_ascend.patch.platform.patch_distributed # noqa import vllm_ascend.patch.platform.patch_mamba_config # noqa import vllm_ascend.patch.platform.patch_sched_yield # noqa diff --git a/vllm_ascend/patch/platform/patch_fusion_matcher_compat_ops.py b/vllm_ascend/patch/platform/patch_fusion_matcher_compat_ops.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/vllm_ascend/platform.py b/vllm_ascend/platform.py index d64596f8f72..2d161a42eb2 100644 --- a/vllm_ascend/platform.py +++ b/vllm_ascend/platform.py @@ -450,33 +450,6 @@ def check_and_update_config(cls, vllm_config: VllmConfig) -> None: ): speculative_config.enforce_eager = False - @classmethod - def apply_config_platform_defaults(cls, vllm_config: VllmConfig) -> None: - """ - Apply the platform-specific default values to the config. - - This function is called during the initialization of global VllmConfig, - after parsing cli arguments but BEFORE sequence parallelism imports. - - We must disable enable_sp and fuse_gemm_comms here because the vLLM - sequence parallelism code imports matcher_utils.py which accesses - CUDA-specific torch.ops._C ops at module level, causing import errors - on NPU platform. - """ - # Disable sequence parallelism to prevent import of CUDA-specific ops - # in vllm/compilation/passes/fusion/matcher_utils.py - if vllm_config.compilation_config is not None: - pass_config = vllm_config.compilation_config.pass_config - if pass_config.enable_sp: - logger.info( - "Sequence Parallelism (enable_sp) is not supported on NPU, " - "disabling. Use vllm-ascend's Flash Comm instead." - ) - pass_config.enable_sp = False - if pass_config.fuse_gemm_comms: - logger.info("fuse_gemm_comms is not supported on NPU, disabling.") - pass_config.fuse_gemm_comms = False - @classmethod def import_kernels(cls) -> None: # Directly importing vllm_ascend_C prevents ASCEND_RT_VISIBLE_DEVICES From 0d2c3ca38af35bca25c02dd9ae85b33ad678e6a7 Mon Sep 17 00:00:00 2001 From: MrZ20 <2609716663@qq.com> Date: Wed, 4 Mar 2026 14:49:27 +0800 Subject: [PATCH 09/11] fix lint Signed-off-by: MrZ20 <2609716663@qq.com> --- vllm_ascend/patch/__init__.py | 1 - vllm_ascend/patch/platform/__init__.py | 2 +- .../patch_fusion_matcher_compat_ops.py | 24 +++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/vllm_ascend/patch/__init__.py b/vllm_ascend/patch/__init__.py index 4d1c69e86d9..e811c128882 100644 --- a/vllm_ascend/patch/__init__.py +++ b/vllm_ascend/patch/__init__.py @@ -97,7 +97,6 @@ # ** 6. File: platform/patch_fusion_matcher_compat_ops.py** # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # 1. `torch.ops._C.rms_norm`, `torch.ops._C.fused_add_rms_norm`, -# `torch.ops._C.rotary_embedding`, `torch.ops.vllm.flashinfer_rotary_embedding` # Why: # upstream vLLM initializes fusion matcher global operators at import time. # On Ascend environment these symbols may be absent and cause import failure. diff --git a/vllm_ascend/patch/platform/__init__.py b/vllm_ascend/patch/platform/__init__.py index 7d54d6cf39c..d6397eace9d 100644 --- a/vllm_ascend/patch/platform/__init__.py +++ b/vllm_ascend/patch/platform/__init__.py @@ -16,8 +16,8 @@ import os -import vllm_ascend.patch.platform.patch_fusion_matcher_compat_ops # noqa import vllm_ascend.patch.platform.patch_distributed # noqa +import vllm_ascend.patch.platform.patch_fusion_matcher_compat_ops # noqa import vllm_ascend.patch.platform.patch_mamba_config # noqa import vllm_ascend.patch.platform.patch_sched_yield # noqa diff --git a/vllm_ascend/patch/platform/patch_fusion_matcher_compat_ops.py b/vllm_ascend/patch/platform/patch_fusion_matcher_compat_ops.py index e69de29bb2d..6e6bcb75f9b 100644 --- a/vllm_ascend/patch/platform/patch_fusion_matcher_compat_ops.py +++ b/vllm_ascend/patch/platform/patch_fusion_matcher_compat_ops.py @@ -0,0 +1,24 @@ +import torch + + +class _MissingOp: + def __init__(self, op_name: str): + self.op_name = op_name + self.default = self + + def __call__(self, *args, **kwargs): + raise RuntimeError(f"Missing upstream op `{self.op_name}` was invoked.") + + +def _set_missing(namespace, op_name: str, full_name: str) -> None: + if not hasattr(namespace, op_name): + setattr(namespace, op_name, _MissingOp(full_name)) + + +_set_missing(torch.ops._C, "rms_norm", "torch.ops._C.rms_norm") +_set_missing(torch.ops._C, "fused_add_rms_norm", "torch.ops._C.fused_add_rms_norm") +_set_missing(torch.ops._C, "rotary_embedding", "torch.ops._C.rotary_embedding") +_set_missing(torch.ops._C, "static_scaled_fp8_quant", "torch.ops._C.static_scaled_fp8_quant") +_set_missing(torch.ops._C, "dynamic_scaled_fp8_quant", "torch.ops._C.dynamic_scaled_fp8_quant") +_set_missing(torch.ops._C, "dynamic_per_token_scaled_fp8_quant", "torch.ops._C.dynamic_per_token_scaled_fp8_quant") +_set_missing(torch.ops._C, "silu_and_mul", "torch.ops._C.silu_and_mul") From 6a09856767b7c45ca74d26bace3dea08f1dd149f Mon Sep 17 00:00:00 2001 From: MrZ20 <2609716663@qq.com> Date: Thu, 5 Mar 2026 12:02:15 +0800 Subject: [PATCH 10/11] update Signed-off-by: MrZ20 <2609716663@qq.com> --- tests/ut/distributed/test_communicator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/ut/distributed/test_communicator.py b/tests/ut/distributed/test_communicator.py index 33325e57803..f968d26ed2e 100644 --- a/tests/ut/distributed/test_communicator.py +++ b/tests/ut/distributed/test_communicator.py @@ -38,13 +38,13 @@ def patched_all_to_all(output_tensor_list, torch.tensor([50, 60]) ]) - dist.all_to_all = patched_all_to_all + torch.distributed.all_to_all = patched_all_to_all scatter_sizes = [2, 2] gather_sizes = [2, 2] input_ = torch.tensor([10, 20, 30, 40]) - with patch.dict(dist.distributed_c10d._world.pg_map, {dist.group.WORLD: object()}, clear=False): + with patch.dict(dist.distributed_c10d._world.pg_map, {dist.group.WORLD: MagicMock()}, clear=False): comm = NPUCommunicator(cpu_group=dist.group.WORLD) output = comm.all_to_all(input_, @@ -81,11 +81,11 @@ def patched_all_to_all(output_tensor_list, torch.tensor([[50, 60]]) ]) - dist.all_to_all = patched_all_to_all + torch.distributed.all_to_all = patched_all_to_all input_ = torch.tensor([[10, 20], [30, 40]]) - with patch.dict(dist.distributed_c10d._world.pg_map, {dist.group.WORLD: object()}, clear=False): + with patch.dict(dist.distributed_c10d._world.pg_map, {dist.group.WORLD: MagicMock()}, clear=False): comm = NPUCommunicator(cpu_group=dist.group.WORLD) output = comm.all_to_all(input_, scatter_dim=0, gather_dim=0) From c663504c1b87495bbe5573fd020fc9aabb6c4fed Mon Sep 17 00:00:00 2001 From: MrZ20 <2609716663@qq.com> Date: Thu, 5 Mar 2026 17:28:58 +0800 Subject: [PATCH 11/11] fix Signed-off-by: MrZ20 <2609716663@qq.com> --- vllm_ascend/worker/model_runner_v1.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/vllm_ascend/worker/model_runner_v1.py b/vllm_ascend/worker/model_runner_v1.py index e800868af74..923f4fd4db3 100644 --- a/vllm_ascend/worker/model_runner_v1.py +++ b/vllm_ascend/worker/model_runner_v1.py @@ -1827,22 +1827,18 @@ def _determine_batch_execution_and_padding( has_lora = len(self.input_batch.lora_id_to_lora_request) > 0 if force_has_lora is None else force_has_lora # ruff: noqa: E731 - if vllm_version_is("0.16.0"): + def dispatch_cudagraph(num_tokens, disable_full=False, valid_modes=None): + if force_eager: + return (CUDAGraphMode.NONE, BatchDescriptor(num_tokens_padded)) - def dispatch_cudagraph(num_tokens, disable_full=False, valid_modes=None): - if force_eager: - return (CUDAGraphMode.NONE, BatchDescriptor(num_tokens_padded)) + if vllm_version_is("0.16.0"): return self.cudagraph_dispatcher.dispatch( num_tokens=num_tokens, has_lora=has_lora, uniform_decode=uniform_decode, disable_full=disable_full, ) - else: - - def dispatch_cudagraph(num_tokens, disable_full=False, valid_modes=None): - if force_eager: - return (CUDAGraphMode.NONE, BatchDescriptor(num_tokens_padded)) + else: return self.cudagraph_dispatcher.dispatch( num_tokens=num_tokens, has_lora=has_lora,