diff --git a/.buildkite/amd/test-amd-merge.yml b/.buildkite/amd/test-amd-merge.yml index 56083b7742b..b1b9e7ecc98 100644 --- a/.buildkite/amd/test-amd-merge.yml +++ b/.buildkite/amd/test-amd-merge.yml @@ -238,13 +238,10 @@ steps: grade: Blocking timeout_in_minutes: 50 commands: - - | - timeout 40m bash -c ' - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model' - ' + - export VLLM_LOGGING_LEVEL=DEBUG + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - timeout 40m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model and cuda" --run-level "advanced_model" # TODO: Bagel test on ROCm is very unstable. @tjtanaa # Need to debug before reneable numerical changes across large PRs diff --git a/tests/buildkite/test_amd_pipeline.py b/tests/buildkite/test_amd_pipeline.py new file mode 100644 index 00000000000..1e74b50a1e2 --- /dev/null +++ b/tests/buildkite/test_amd_pipeline.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +from pathlib import Path +from shlex import split + +import pytest +import yaml + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +AMD_MERGE_PIPELINE = Path(".buildkite/amd/test-amd-merge.yml") + + +def _find_step(label: str) -> dict: + pipeline = yaml.safe_load(AMD_MERGE_PIPELINE.read_text(encoding="utf-8")) + + def walk(steps: list[dict]) -> dict | None: + for step in steps: + if step.get("label") == label: + return step + if nested := walk(step.get("steps", [])): + return nested + return None + + step = walk(pipeline.get("steps", [])) + assert step is not None, f"missing AMD pipeline step: {label}" + return step + + +def test_qwen3_tts_base_preserves_advanced_model_arguments() -> None: + step = _find_step("Qwen3-TTS Base E2E Test") + commands = step["commands"] + + assert all("bash -c" not in command for command in commands) + pytest_command = next(command for command in commands if "pytest" in command) + argv = split(pytest_command) + + marker_index = argv.index("-m") + run_level_index = argv.index("--run-level") + assert argv[marker_index + 1] == "advanced_model and cuda" + assert argv[run_level_index + 1] == "advanced_model" diff --git a/tests/config/test_config_factory.py b/tests/config/test_config_factory.py index e3c6c585fdd..4cdb1b635b6 100644 --- a/tests/config/test_config_factory.py +++ b/tests/config/test_config_factory.py @@ -2399,6 +2399,21 @@ def test_ci_sampling_merge(self): # CI overrides max_tokens assert s0["max_tokens"] == 150 + def test_qwen3_omni_colocate_async_bounds_only_rocm_kv_cache(self): + ci_path = Path(get_deploy_config_path("ci/qwen3_omni_moe_colocate_async.yaml")) + pipeline = resolve_pipeline_config("qwen3_omni_moe_thinker_only") + assert isinstance(pipeline, PipelineConfig) + + cuda = _apply_platform_overrides(load_deploy_config(ci_path), platform="cuda") + cuda_stage = merge_pipeline_deploy(pipeline, cuda)[0] + assert cuda_stage.yaml_engine_args["gpu_memory_utilization"] == 0.9 + assert "kv_cache_memory_bytes" not in cuda_stage.yaml_engine_args + + rocm = _apply_platform_overrides(load_deploy_config(ci_path), platform="rocm") + rocm_stage = merge_pipeline_deploy(pipeline, rocm)[0] + assert "gpu_memory_utilization" not in rocm_stage.yaml_engine_args + assert rocm_stage.yaml_engine_args["kv_cache_memory_bytes"] == 2 * 1024**3 + def test_pure_inheritance_overlay(self, tmp_path): """An overlay with only ``base_config`` inherits everything.""" base = Path(get_deploy_config_path("qwen3_omni_moe.yaml")) diff --git a/tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py b/tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py index 54e9b8f6236..fbb5bbfe1d8 100644 --- a/tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py +++ b/tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py @@ -22,7 +22,7 @@ from tests.helpers.clean import wait_for_gpu_memory_to_clear from tests.helpers.mark import hardware_test -from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config +from tests.helpers.stage_config import get_deploy_config_path from vllm_omni.entrypoints.async_omni import AsyncOmni from vllm_omni.platforms import current_omni_platform @@ -52,26 +52,7 @@ def _colocate_async_deploy() -> str: CI yaml plus CuMem on every stage fails engine-core startup in the L3 merge job: Stage 1 resolves ``devices: "1"`` while only GPU 0 is visible. """ - return modify_stage_config( - get_deploy_config_path("qwen3_omni_moe_thinking.yaml"), - updates={ - "stages": { - 0: { - "enable_sleep_mode": True, - "enforce_eager": True, - "trust_remote_code": True, - "tensor_parallel_size": 1, - "devices": "0", - "gpu_memory_utilization": 0.9, - "max_num_seqs": 1, - "max_model_len": 8192, - "max_num_batched_tokens": 8192, - "enable_prefix_caching": False, - "skip_mm_profiling": True, - }, - }, - }, - ) + return get_deploy_config_path("ci/qwen3_omni_moe_colocate_async.yaml") @pytest.mark.advanced_model diff --git a/tests/helpers/stage_config.py b/tests/helpers/stage_config.py index 36d8c406a11..fed61937f36 100644 --- a/tests/helpers/stage_config.py +++ b/tests/helpers/stage_config.py @@ -381,6 +381,40 @@ def delete_by_path(config_dict: dict, path: str) -> None: }, }, }, + "qwen3_omni_moe_colocate_async": { + "base_config": "qwen3_omni_moe_thinking.yaml", + "stages": [ + { + "stage_id": 0, + "enable_sleep_mode": True, + "enforce_eager": True, + "trust_remote_code": True, + "tensor_parallel_size": 1, + "devices": "0", + "max_num_seqs": 1, + "max_model_len": 8192, + "max_num_batched_tokens": 8192, + "enable_prefix_caching": False, + "skip_mm_profiling": True, + }, + ], + "platforms": { + "rocm": { + "stages": [ + { + "stage_id": 0, + # This control-plane test needs only 8K tokens. On MI300, + # percentage sizing attempted a ~125 GiB KV allocation + # after the 64.5 GiB model load and then failed while + # allocating the cache. Keep CUDA's existing sizing and + # give only ROCm a bounded cache with ample headroom. + "gpu_memory_utilization": None, + "kv_cache_memory_bytes": 2 * 1024**3, + }, + ], + }, + }, + }, "qwen3_omni_moe_multi_replicas_4gpu": { "base_config": "qwen3_omni_moe.yaml", "async_chunk": True,