From 51206f4697360ea33bf11a233cf20a07cca0ab7b Mon Sep 17 00:00:00 2001 From: ckhw Date: Sat, 25 Oct 2025 15:15:43 +0800 Subject: [PATCH 1/7] add qwen qwq testcase Signed-off-by: ckhw --- .../vllm_ascend_test_nightly_a3.yaml | 19 +++ .../e2e/nightly/models/test_qwen3_qwq_32b.py | 121 ++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 tests/e2e/nightly/models/test_qwen3_qwq_32b.py diff --git a/.github/workflows/vllm_ascend_test_nightly_a3.yaml b/.github/workflows/vllm_ascend_test_nightly_a3.yaml index c176db03ccf..08695d02c9c 100644 --- a/.github/workflows/vllm_ascend_test_nightly_a3.yaml +++ b/.github/workflows/vllm_ascend_test_nightly_a3.yaml @@ -68,6 +68,25 @@ jobs: - name: deepseek-r1-0528-w8a8-prefix-cache os: linux-aarch64-a3-16 tests: tests/e2e/nightly/features/test_prefix_cache_deepseek_r1_0528_w8a8.py + - name: qwen-qwq-32b-a3-4 + os: linux-aarch64-a3-4 + tests: tests/e2e/nightly/models/test_qwen3_qwq_32b.py + + uses: ./.github/workflows/_e2e_nightly_single_node.yaml + with: + vllm: v0.11.0 + runner: ${{ matrix.test_config.os }} + image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.2.rc1-a3-ubuntu22.04-py3.11 + tests: ${{ matrix.test_config.tests }} + single-node-tests1: + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + strategy: + fail-fast: false + matrix: + test_config: + - name: qwen-qwq-32b-a3-4 + os: linux-aarch64-a3-4 + tests: tests/e2e/nightly/models/test_qwen3_qwq_32b.py uses: ./.github/workflows/_e2e_nightly_single_node.yaml with: vllm: v0.11.0 diff --git a/tests/e2e/nightly/models/test_qwen3_qwq_32b.py b/tests/e2e/nightly/models/test_qwen3_qwq_32b.py new file mode 100644 index 00000000000..6ea92351b28 --- /dev/null +++ b/tests/e2e/nightly/models/test_qwen3_qwq_32b.py @@ -0,0 +1,121 @@ +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. +# Copyright 2023 The vLLM team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# This file is a part of the vllm-ascend project. +# +import os +from typing import Any + +import openai +import pytest +from vllm.utils import get_open_port + +from tests.e2e.conftest import RemoteOpenAIServer +from tools.aisbench import run_aisbench_cases + +MODELS = [ + "Qwen/QwQ-32B", +] + +MODES = [ + "aclgraph", + "single", +] + +TENSOR_PARALLELS = [4] + +prompts = [ + "San Francisco is a", +] + +api_keyword_args = { + "max_tokens": 10, +} + +aisbench_cases = [{ + "case_type": "accuracy", + "dataset_path": "vllm-ascend/gsm8k-lite", + "request_conf": "vllm_api_general_chat", + "dataset_conf": "gsm8k/gsm8k_gen_0_shot_cot_chat_prompt", + "max_out_len": 32768, + "batch_size": 32, + "baseline": 95, + "threshold": 5 +}, { + "case_type": "performance", + "dataset_path": "vllm-ascend/GSM8K-in3500-bs400", + "request_conf": "vllm_api_stream_chat", + "dataset_conf": "gsm8k/gsm8k_gen_0_shot_cot_str_perf", + "num_prompts": 176, + "max_out_len": 1500, + "batch_size": 44, + "baseline": 1, + "threshold": 0.97 +}] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", MODELS) +@pytest.mark.parametrize("mode", MODES) +@pytest.mark.parametrize("tp_size", TENSOR_PARALLELS) +async def test_models(model: str, mode: str, tp_size: int) -> None: + port = get_open_port() + env_dict = { + "TASK_QUEUE_ENABLE": "1", + "OMP_PROC_BIND": "false", + "VLLM_USE_V1": "1", + "HCCL_OP_EXPANSION_MODE": "AIV", + "VLLM_ASCEND_ENABLE_FLASHCOMM": "1", + "VLLM_ASCEND_ENABLE_TOPK_OPTIMIZE": "1", + "VLLM_ASCEND_ENABLE_DEBSE_OPTIMIZE": "1", + "VLLM_ASCEND_ENABLE_PREFETCH": "1" + } + server_args = [ + "--tensor-parallel-size", + str(tp_size), "--port", + str(port), "--max-model-len", "36864", "--max-num-batched-tokens", + "36864", "--block-size", "128", "--trust-remote-code", + "--gpu-memory-utilization", "0.9", "--compilation_config", + '{"cudagraph_mode":"FULL_DECODE_ONLY", "cudagraph_capture_sizes": [1, 8, 24, 48, 60]}', + "--reasoning-parser", "deepseek_r1", "--distributed_executor_backend", + "mp" + ] + if mode == "single": + server_args.remove("--compilation_config") + server_args.remove( + '{"cudagraph_mode":"FULL_DECODE_ONLY", "cudagraph_capture_sizes": [1, 8, 24, 48, 60]}' + ) + server_args.append("--additional-config") + server_args.append('{"ascend_scheduler_config":{"enabled":true}}') + server_args.append("--enforce-eager") + request_keyword_args: dict[str, Any] = { + **api_keyword_args, + } + with RemoteOpenAIServer(model, + server_args, + server_port=port, + env_dict=env_dict, + auto_port=False) as server: + client = server.get_async_client() + batch = await client.completions.create( + model=model, + prompt=prompts, + **request_keyword_args, + ) + choices: list[openai.types.CompletionChoice] = batch.choices + assert choices[0].text, "empty response" + if mode == "single": + return + # aisbench test + run_aisbench_cases(model, port, aisbench_cases) From ce52e70d4b1e1bec4d4808853387f6ab1a670e03 Mon Sep 17 00:00:00 2001 From: ckhw Date: Sat, 25 Oct 2025 15:17:46 +0800 Subject: [PATCH 2/7] add qwen qwq testcase Signed-off-by: ckhw --- .github/workflows/vllm_ascend_test_nightly_a3.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/vllm_ascend_test_nightly_a3.yaml b/.github/workflows/vllm_ascend_test_nightly_a3.yaml index 08695d02c9c..c8443410586 100644 --- a/.github/workflows/vllm_ascend_test_nightly_a3.yaml +++ b/.github/workflows/vllm_ascend_test_nightly_a3.yaml @@ -79,7 +79,6 @@ jobs: image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.2.rc1-a3-ubuntu22.04-py3.11 tests: ${{ matrix.test_config.tests }} single-node-tests1: - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' strategy: fail-fast: false matrix: From de716e544aac7316e41c6385e1785db16c0379ed Mon Sep 17 00:00:00 2001 From: ckhw Date: Sat, 25 Oct 2025 15:37:00 +0800 Subject: [PATCH 3/7] add qwen qwq testcase Signed-off-by: ckhw --- tests/e2e/nightly/models/test_qwen3_qwq_32b.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e/nightly/models/test_qwen3_qwq_32b.py b/tests/e2e/nightly/models/test_qwen3_qwq_32b.py index 6ea92351b28..a4f843a7738 100644 --- a/tests/e2e/nightly/models/test_qwen3_qwq_32b.py +++ b/tests/e2e/nightly/models/test_qwen3_qwq_32b.py @@ -14,7 +14,6 @@ # limitations under the License. # This file is a part of the vllm-ascend project. # -import os from typing import Any import openai From b564dd3f5798bf7c11f5533ab7b268073d25d390 Mon Sep 17 00:00:00 2001 From: ckhw Date: Sat, 25 Oct 2025 16:25:34 +0800 Subject: [PATCH 4/7] add qwen qwq testcase Signed-off-by: ckhw --- .github/workflows/vllm_ascend_test_nightly_a3.yaml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/vllm_ascend_test_nightly_a3.yaml b/.github/workflows/vllm_ascend_test_nightly_a3.yaml index c8443410586..5bb359fcab4 100644 --- a/.github/workflows/vllm_ascend_test_nightly_a3.yaml +++ b/.github/workflows/vllm_ascend_test_nightly_a3.yaml @@ -78,20 +78,6 @@ jobs: runner: ${{ matrix.test_config.os }} image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.2.rc1-a3-ubuntu22.04-py3.11 tests: ${{ matrix.test_config.tests }} - single-node-tests1: - strategy: - fail-fast: false - matrix: - test_config: - - name: qwen-qwq-32b-a3-4 - os: linux-aarch64-a3-4 - tests: tests/e2e/nightly/models/test_qwen3_qwq_32b.py - uses: ./.github/workflows/_e2e_nightly_single_node.yaml - with: - vllm: v0.11.0 - runner: ${{ matrix.test_config.os }} - image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.2.rc1-a3-ubuntu22.04-py3.11 - tests: ${{ matrix.test_config.tests }} multi-node-tests: needs: single-node-tests From dc691554b0587c4ee96681c6f225876e8a75f54d Mon Sep 17 00:00:00 2001 From: ckhw Date: Sat, 25 Oct 2025 16:39:13 +0800 Subject: [PATCH 5/7] add qwen qwq testcase Signed-off-by: ckhw --- .github/workflows/vllm_ascend_test_nightly_a3.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/vllm_ascend_test_nightly_a3.yaml b/.github/workflows/vllm_ascend_test_nightly_a3.yaml index 5bb359fcab4..a2ffb2b0ebb 100644 --- a/.github/workflows/vllm_ascend_test_nightly_a3.yaml +++ b/.github/workflows/vllm_ascend_test_nightly_a3.yaml @@ -71,7 +71,6 @@ jobs: - name: qwen-qwq-32b-a3-4 os: linux-aarch64-a3-4 tests: tests/e2e/nightly/models/test_qwen3_qwq_32b.py - uses: ./.github/workflows/_e2e_nightly_single_node.yaml with: vllm: v0.11.0 From 4ad7a0ff30ccb5f3f01a276be6e637b8eb95e529 Mon Sep 17 00:00:00 2001 From: ckhw Date: Sat, 25 Oct 2025 16:41:27 +0800 Subject: [PATCH 6/7] add qwq testcase Signed-off-by: ckhw --- .github/workflows/vllm_ascend_test_nightly_a3.yaml | 2 +- .../nightly/models/{test_qwen3_qwq_32b.py => test_qwq_32b.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/e2e/nightly/models/{test_qwen3_qwq_32b.py => test_qwq_32b.py} (100%) diff --git a/.github/workflows/vllm_ascend_test_nightly_a3.yaml b/.github/workflows/vllm_ascend_test_nightly_a3.yaml index a2ffb2b0ebb..4a520da0a36 100644 --- a/.github/workflows/vllm_ascend_test_nightly_a3.yaml +++ b/.github/workflows/vllm_ascend_test_nightly_a3.yaml @@ -68,7 +68,7 @@ jobs: - name: deepseek-r1-0528-w8a8-prefix-cache os: linux-aarch64-a3-16 tests: tests/e2e/nightly/features/test_prefix_cache_deepseek_r1_0528_w8a8.py - - name: qwen-qwq-32b-a3-4 + - name: qwq-32b-a3 os: linux-aarch64-a3-4 tests: tests/e2e/nightly/models/test_qwen3_qwq_32b.py uses: ./.github/workflows/_e2e_nightly_single_node.yaml diff --git a/tests/e2e/nightly/models/test_qwen3_qwq_32b.py b/tests/e2e/nightly/models/test_qwq_32b.py similarity index 100% rename from tests/e2e/nightly/models/test_qwen3_qwq_32b.py rename to tests/e2e/nightly/models/test_qwq_32b.py From abf627ce7660c40693c32666d949d784668661a2 Mon Sep 17 00:00:00 2001 From: ckhw Date: Sat, 25 Oct 2025 17:09:47 +0800 Subject: [PATCH 7/7] add qwen qwq testcase Signed-off-by: ckhw --- .github/workflows/vllm_ascend_test_nightly_a3.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vllm_ascend_test_nightly_a3.yaml b/.github/workflows/vllm_ascend_test_nightly_a3.yaml index 4a520da0a36..a5d899463f4 100644 --- a/.github/workflows/vllm_ascend_test_nightly_a3.yaml +++ b/.github/workflows/vllm_ascend_test_nightly_a3.yaml @@ -70,7 +70,7 @@ jobs: tests: tests/e2e/nightly/features/test_prefix_cache_deepseek_r1_0528_w8a8.py - name: qwq-32b-a3 os: linux-aarch64-a3-4 - tests: tests/e2e/nightly/models/test_qwen3_qwq_32b.py + tests: tests/e2e/nightly/models/test_qwq_32b.py uses: ./.github/workflows/_e2e_nightly_single_node.yaml with: vllm: v0.11.0