From 80276c501f145bd9e3a855b48da9ae2b4e55bb71 Mon Sep 17 00:00:00 2001 From: lm10-arg <347490427+lm10-arg@users.noreply.github.com> Date: Wed, 4 Nov 2026 23:18:42 +0900 Subject: [PATCH 1/5] [CI/Build] Fix vLLM 0.33 AMD regressions Signed-off-by: lm10-arg <347490427+lm10-arg@users.noreply.github.com> --- .buildkite/amd/scripts/aiter_mha_cache.py | 4 +++ tests/buildkite/test_aiter_mha_cache.py | 23 ++++++++++++ .../qwen3_tts/test_code_predictor_dtype.py | 4 +-- tests/tools/test_tools_package.py | 36 +++++++++++++++++++ tools/__init__.py | 9 +++++ 5 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 tests/tools/test_tools_package.py create mode 100644 tools/__init__.py diff --git a/.buildkite/amd/scripts/aiter_mha_cache.py b/.buildkite/amd/scripts/aiter_mha_cache.py index 23ff7327bdd..749265f9da7 100644 --- a/.buildkite/amd/scripts/aiter_mha_cache.py +++ b/.buildkite/amd/scripts/aiter_mha_cache.py @@ -234,6 +234,10 @@ def dynamic_library_paths() -> list[Path]: try: torch_distribution = importlib.metadata.distribution("torch") candidates.append(Path(torch_distribution.locate_file("torch/lib"))) + torch_site_packages = Path(torch_distribution.locate_file("")) + candidates.extend( + library.parent for library in torch_site_packages.glob("_rocm_sdk_*/lib/*/lib/librocm-openblas.so.*") + ) except importlib.metadata.PackageNotFoundError: pass candidates.extend((Path("/opt/rocm/lib"), Path("/opt/rocm/lib64"))) diff --git a/tests/buildkite/test_aiter_mha_cache.py b/tests/buildkite/test_aiter_mha_cache.py index b1144ceb84b..2aa87b19818 100644 --- a/tests/buildkite/test_aiter_mha_cache.py +++ b/tests/buildkite/test_aiter_mha_cache.py @@ -146,6 +146,29 @@ def missing_runner(command: list[str], **_kwargs) -> subprocess.CompletedProcess assert not cache.ldd_resolves(module, runner=missing_runner, library_paths=[]) +def test_dynamic_library_paths_include_therock_openblas_directory( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + site_packages = tmp_path / "site-packages" + torch_lib = site_packages / "torch/lib" + therock_lib = site_packages / "_rocm_sdk_libraries/lib/rocm_sysdeps/lib" + torch_lib.mkdir(parents=True) + therock_lib.mkdir(parents=True) + (therock_lib / "librocm-openblas.so.0").touch() + + class TorchDistribution: + def locate_file(self, path: str) -> Path: + return site_packages / path + + def distribution(name: str) -> TorchDistribution: + assert name == "torch" + return TorchDistribution() + + monkeypatch.setattr(cache.importlib.metadata, "distribution", distribution) + + assert cache.dynamic_library_paths()[:2] == [torch_lib, therock_lib] + + def test_image_dependency_identity_tracks_the_rocm_image_inputs(tmp_path: Path) -> None: root = tmp_path / "repo" for relative in cache._IMAGE_DEPENDENCY_PATHS: diff --git a/tests/model_executor/models/qwen3_tts/test_code_predictor_dtype.py b/tests/model_executor/models/qwen3_tts/test_code_predictor_dtype.py index 2a5e1f4a3ce..7cdeeca643b 100644 --- a/tests/model_executor/models/qwen3_tts/test_code_predictor_dtype.py +++ b/tests/model_executor/models/qwen3_tts/test_code_predictor_dtype.py @@ -974,8 +974,8 @@ def test_fused_gate_up_matches_separate(self, loaded_target_classes) -> None: ref_g, ref_u = g(x), u(x) out_g, out_u = fused(x).split([intermediate, intermediate], dim=-1) - assert torch.equal(ref_g, out_g) - assert torch.equal(ref_u, out_u) + torch.testing.assert_close(ref_g, out_g) + torch.testing.assert_close(ref_u, out_u) def test_load_weights_repacks_hf_shards(self, mocker: MockerFixture, loaded_target_classes) -> None: """``load_weights`` must transparently re-pack HF q/k/v and gate/up diff --git a/tests/tools/test_tools_package.py b/tests/tools/test_tools_package.py new file mode 100644 index 00000000000..782185d8d88 --- /dev/null +++ b/tests/tools/test_tools_package.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_local_tools_package_wins_over_later_regular_package(tmp_path: Path) -> None: + repo_root = Path(__file__).resolve().parents[2] + competing_root = tmp_path / "dependency" + competing_package = competing_root / "tools" + competing_package.mkdir(parents=True) + (competing_package / "__init__.py").write_text("SOURCE = 'dependency'\n", encoding="utf-8") + + environment = os.environ.copy() + environment["PYTHONPATH"] = os.pathsep.join((str(repo_root), str(competing_root))) + program = "from pathlib import Path; import tools; print(Path(tools.__file__).resolve())" + completed = subprocess.run( + [sys.executable, "-c", program], + cwd=tmp_path, + env=environment, + check=True, + capture_output=True, + text=True, + timeout=10, + ) + + assert Path(completed.stdout.strip()) == (repo_root / "tools/__init__.py").resolve() diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 00000000000..4a6c1d4f534 --- /dev/null +++ b/tools/__init__.py @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Repository-local developer and CI tools. + +Making this directory a regular package prevents a dependency's top-level +``tools`` package from shadowing these modules when the repository root appears +first on ``sys.path``. +""" From fe17e445a07ea0af64a9136aa57938f40d228201 Mon Sep 17 00:00:00 2001 From: andyluo7 <43718156+andyluo7@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:02:04 +0900 Subject: [PATCH 2/5] [CI/Build] Use portable Triton libdevice for MOSS Signed-off-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> --- vllm_omni/model_executor/models/moss_tts/local_kernels.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm_omni/model_executor/models/moss_tts/local_kernels.py b/vllm_omni/model_executor/models/moss_tts/local_kernels.py index 1e485bc8279..a9a37d3c84e 100644 --- a/vllm_omni/model_executor/models/moss_tts/local_kernels.py +++ b/vllm_omni/model_executor/models/moss_tts/local_kernels.py @@ -10,13 +10,13 @@ import numpy as np import torch from vllm.platforms import current_platform -from vllm.triton_utils import tl, triton +from vllm.triton_utils import tl, tldevice, triton from .triton_launch import cuda_pdl_launch_kwargs _gdc_launch = tl.extra.cuda.gdc_launch_dependents _gdc_wait = tl.extra.cuda.gdc_wait -libdevice = tl.extra.cuda.libdevice +libdevice = tldevice # Hopper+ supports programmatic dependent launch. The kernel waits for its # predecessor before any global access and lets its successor launch at once; results are unchanged. From 349fbf1571a7160e1b9646a2d261289a89fcd71f Mon Sep 17 00:00:00 2001 From: andyluo7 <43718156+andyluo7@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:34:51 +0900 Subject: [PATCH 3/5] [CI/Build] Gate MOSS round-to-nearest op to CUDA Signed-off-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> --- .../model_executor/models/moss_tts/local_kernels.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/vllm_omni/model_executor/models/moss_tts/local_kernels.py b/vllm_omni/model_executor/models/moss_tts/local_kernels.py index a9a37d3c84e..f07c81e0623 100644 --- a/vllm_omni/model_executor/models/moss_tts/local_kernels.py +++ b/vllm_omni/model_executor/models/moss_tts/local_kernels.py @@ -10,16 +10,17 @@ import numpy as np import torch from vllm.platforms import current_platform -from vllm.triton_utils import tl, tldevice, triton +from vllm.triton_utils import tl, triton +from vllm.triton_utils import tldevice as libdevice from .triton_launch import cuda_pdl_launch_kwargs _gdc_launch = tl.extra.cuda.gdc_launch_dependents _gdc_wait = tl.extra.cuda.gdc_wait -libdevice = tldevice # Hopper+ supports programmatic dependent launch. The kernel waits for its # predecessor before any global access and lets its successor launch at once; results are unchanged. +_CUDA_RN_OPS = current_platform.is_cuda() _USE_PDL = current_platform.is_cuda() and current_platform.has_device_capability(90) @@ -44,6 +45,7 @@ def _attend( BS: tl.constexpr, GROUP: tl.constexpr, LATE: tl.constexpr = False, + CUDA_RN_OPS: tl.constexpr = _CUDA_RN_OPS, ): """KV insertion and attention for GROUP heads of one row (shared body). @@ -82,7 +84,11 @@ def _attend( # Pin the accumulator's rounding. LLVM SLP can otherwise contract the # initial product with the first add differently when this body is # inlined into sampling on SM89, breaking fused/unfused bitwise parity. - result = libdevice.mul_rn(v, tl.sum(tl.where(s[None, :] == POSITION, probs, 0.0), axis=1)[:, None]) + current_probability = tl.sum(tl.where(s[None, :] == POSITION, probs, 0.0), axis=1)[:, None] + if CUDA_RN_OPS: + result = libdevice.mul_rn(v, current_probability) + else: + result = v * current_probability for p in tl.static_range(POSITION): vp = tl.load(V + base + p * D, dmask, 0, eviction_policy=CACHE_LOAD).to(tl.float32) result = tl.fma(vp, tl.sum(tl.where(s[None, :] == p, probs, 0.0), axis=1)[:, None], result) From 13a4c72c997037c42a1d3ced603e2d51444b862c Mon Sep 17 00:00:00 2001 From: andyluo7 <43718156+andyluo7@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:37:17 +0900 Subject: [PATCH 4/5] [CI/Build] Make TheRock library discovery deterministic Signed-off-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> --- .buildkite/amd/scripts/aiter_mha_cache.py | 4 +++- tests/buildkite/test_aiter_mha_cache.py | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.buildkite/amd/scripts/aiter_mha_cache.py b/.buildkite/amd/scripts/aiter_mha_cache.py index 749265f9da7..60cfb7f12a4 100644 --- a/.buildkite/amd/scripts/aiter_mha_cache.py +++ b/.buildkite/amd/scripts/aiter_mha_cache.py @@ -236,7 +236,9 @@ def dynamic_library_paths() -> list[Path]: candidates.append(Path(torch_distribution.locate_file("torch/lib"))) torch_site_packages = Path(torch_distribution.locate_file("")) candidates.extend( - library.parent for library in torch_site_packages.glob("_rocm_sdk_*/lib/*/lib/librocm-openblas.so.*") + sorted( + {library.parent for library in torch_site_packages.glob("_rocm_sdk_*/lib/*/lib/librocm-openblas.so.*")} + ) ) except importlib.metadata.PackageNotFoundError: pass diff --git a/tests/buildkite/test_aiter_mha_cache.py b/tests/buildkite/test_aiter_mha_cache.py index 2aa87b19818..1d642d0289f 100644 --- a/tests/buildkite/test_aiter_mha_cache.py +++ b/tests/buildkite/test_aiter_mha_cache.py @@ -152,9 +152,13 @@ def test_dynamic_library_paths_include_therock_openblas_directory( site_packages = tmp_path / "site-packages" torch_lib = site_packages / "torch/lib" therock_lib = site_packages / "_rocm_sdk_libraries/lib/rocm_sysdeps/lib" + therock_runtime_lib = site_packages / "_rocm_sdk_runtime/lib/rocm_sysdeps/lib" torch_lib.mkdir(parents=True) therock_lib.mkdir(parents=True) + therock_runtime_lib.mkdir(parents=True) (therock_lib / "librocm-openblas.so.0").touch() + (therock_lib / "librocm-openblas.so.0.3").touch() + (therock_runtime_lib / "librocm-openblas.so.0").touch() class TorchDistribution: def locate_file(self, path: str) -> Path: @@ -166,7 +170,7 @@ def distribution(name: str) -> TorchDistribution: monkeypatch.setattr(cache.importlib.metadata, "distribution", distribution) - assert cache.dynamic_library_paths()[:2] == [torch_lib, therock_lib] + assert cache.dynamic_library_paths()[:3] == [torch_lib, therock_lib, therock_runtime_lib] def test_image_dependency_identity_tracks_the_rocm_image_inputs(tmp_path: Path) -> None: From 4e7f3c9a3fa54ad0a4baedc59db4858ba8979e8d Mon Sep 17 00:00:00 2001 From: andyluo7 <43718156+andyluo7@users.noreply.github.com> Date: Thu, 5 Nov 2026 01:30:52 +0900 Subject: [PATCH 5/5] [CI/Build] Refresh ROCm R2 collection guard Signed-off-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> --- .buildkite/amd/test-amd-ready.yml | 4 ++-- tests/buildkite/test_amd_pipeline.py | 10 +++++----- tests/buildkite/test_amd_ready_r2_01.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.buildkite/amd/test-amd-ready.yml b/.buildkite/amd/test-amd-ready.yml index 20e4cd2ae91..66f4eeba771 100644 --- a/.buildkite/amd/test-amd-ready.yml +++ b/.buildkite/amd/test-amd-ready.yml @@ -652,11 +652,11 @@ steps: # do not duplicate its model startup in this time-bounded coverage lane. # The CUDA ready job also covers tests/worker/test_batched_omni_output.py. # Keep the initial ROCm burn-in scoped to tests/model_executor/. - # Shard 0 checks the full 1416-node set. The other shards skip that + # Shard 0 checks the full 1420-node set. The other shards skip that # collection and run their pytest slice immediately. - | if [[ "$${BUILDKITE_PARALLEL_JOB:-0}" == "0" ]]; then - bash .buildkite/amd/scripts/assert-pytest-collection.sh 1416 \ + bash .buildkite/amd/scripts/assert-pytest-collection.sh 1420 \ tests/model_executor/ \ -m 'core_model and rocm and MI325 and not nvidia_only and not (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)' \ --run-level "core_model" diff --git a/tests/buildkite/test_amd_pipeline.py b/tests/buildkite/test_amd_pipeline.py index 4f10954d349..492ba78d036 100644 --- a/tests/buildkite/test_amd_pipeline.py +++ b/tests/buildkite/test_amd_pipeline.py @@ -757,7 +757,7 @@ def test_joy_gpu_suite_remains_selected_by_cuda_model_lane() -> None: "ROCm · Model Executor GPU Coverage (R2-01) · Shard %N/%t", AMD_READY_PIPELINE, "NonBlocking", - 1416, + 1420, "core_model and rocm and MI325 and not nvidia_only and not " "(cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)", ), @@ -3601,7 +3601,7 @@ def test_worker_one_card_ready_stays_outside_r2_01() -> None: """READY blocks on the collected one-card worker nodes. Local collection of the three files with the MI325 marker returned these - 32 ids. R2-01 still collects 1416 model-executor nodes and does not run them. + 32 ids. R2-01 still collects 1420 model-executor nodes and does not run them. """ step = _find_step("ROCm · One-Card Worker GPU Test", AMD_READY_PIPELINE) command = "\n".join(step["commands"]) @@ -3647,7 +3647,7 @@ def test_worker_one_card_ready_stays_outside_r2_01() -> None: AMD_READY_PIPELINE, ) r2_01_command = "\n".join(r2_01["commands"]) - assert "assert-pytest-collection.sh 1416" in r2_01_command + assert "assert-pytest-collection.sh 1420" in r2_01_command assert "tests/model_executor/" in r2_01_command for path in _WORKER_ONE_CARD_FILES: assert path not in r2_01_command @@ -3858,7 +3858,7 @@ def test_worker_copy_ready_pins_remaining_one_card_nodes() -> None: The marker selects both intermediate-buffer residency rows and the three device-copy GPU tests. #9552's worker files stay on the other job. R2-01 - still collects 1416 model-executor nodes. + still collects 1420 model-executor nodes. """ step = _find_step("ROCm · One-Card Worker Copy Test", AMD_READY_PIPELINE) command = "\n".join(step["commands"]) @@ -3910,7 +3910,7 @@ def test_worker_copy_ready_pins_remaining_one_card_nodes() -> None: AMD_READY_PIPELINE, ) r2_01_command = "\n".join(r2_01["commands"]) - assert "assert-pytest-collection.sh 1416" in r2_01_command + assert "assert-pytest-collection.sh 1420" in r2_01_command for path in _WORKER_COPY_FILES: assert path not in r2_01_command diff --git a/tests/buildkite/test_amd_ready_r2_01.py b/tests/buildkite/test_amd_ready_r2_01.py index 7042f25a10e..7f7aeab7ea8 100644 --- a/tests/buildkite/test_amd_ready_r2_01.py +++ b/tests/buildkite/test_amd_ready_r2_01.py @@ -98,7 +98,7 @@ def test_r2_01_runs_once_fails_closed_and_publishes_results() -> None: assert "--shard-id=$$BUILDKITE_PARALLEL_JOB" in argv assert "--collect-only" not in joined assert '$${BUILDKITE_PARALLEL_JOB:-0}" == "0"' in joined - assert "assert-pytest-collection.sh 1416" in joined + assert "assert-pytest-collection.sh 1420" in joined assert "VLLM_CI_ALLOW_NO_TESTS" not in joined assert any(artifact_dir in command for command in commands) assert "-v" in argv