From 2e04e802b3c01f99e4cf9ffc059833bcc63eefef Mon Sep 17 00:00:00 2001 From: haic0 <149741444+haic0@users.noreply.github.com> Date: Mon, 5 Oct 2026 19:38:20 -0400 Subject: [PATCH 1/3] [CI][ROCm] Add HunyuanImage3 nightly coverage Assisted-by: Cursor Co-authored-by: Cursor Signed-off-by: haic0 <149741444+haic0@users.noreply.github.com> --- .buildkite/amd/test-amd-nightly.yml | 22 +++++++++++++++++++ .../test_hunyuan_image3_pixel_accuracy.py | 10 ++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/.buildkite/amd/test-amd-nightly.yml b/.buildkite/amd/test-amd-nightly.yml index 1d677543455..138fb120b23 100644 --- a/.buildkite/amd/test-amd-nightly.yml +++ b/.buildkite/amd/test-amd-nightly.yml @@ -132,3 +132,25 @@ steps: artifact_dir="$${BUILDKITE_BUILD_CHECKOUT_PATH:?}/artifacts/rocm-cosmos3-nightly" mkdir -p "$$artifact_dir" cp -v /tmp/cosmos3-pytest.xml "$$artifact_dir/" + + - group: ":full_moon: HunyuanImage3 MI300 Nightly" + steps: + - label: "HunyuanImage3 Offline Pixel Accuracy" + agent_pool: mi300_4 + mirror_hardwares: [amdproduction] + grade: NonBlocking + timeout_in_minutes: 180 + env: + HUNYUAN_IMAGE3_MODEL: "tencent/HunyuanImage-3.0-Instruct" + HUNYUAN_IMAGE3_DEVICES: "0,1,2,3" + DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA + artifact_paths: + - artifacts/rocm-hunyuanimage3/**/* + commands: + - pytest --collect-only -q tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py::test_hunyuan_image3_pixel_accuracy_offline -m "full_model and rocm and MI325 and cards_4" --run-level full_model + - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py::test_hunyuan_image3_pixel_accuracy_offline -m "full_model and rocm and MI325 and cards_4" --run-level full_model --junitxml=/tmp/hunyuanimage3-pytest.xml + - | + artifact_dir="$${BUILDKITE_BUILD_CHECKOUT_PATH:?}/artifacts/rocm-hunyuanimage3" + mkdir -p "$$artifact_dir" + cp -v tests/e2e/accuracy/artifacts/HunyuanImage-3_0-Instruct/*.png "$$artifact_dir/" + cp -v /tmp/hunyuanimage3-pytest.xml "$$artifact_dir/" diff --git a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py index 6399425e8ad..bd0acbedd5b 100644 --- a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py +++ b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py @@ -111,6 +111,14 @@ def _baseline_path() -> Path: }, ], "platforms": { + "rocm": { + "stages": [ + { + "stage_id": 0, + "moe_backend": "auto", + }, + ], + }, "npu": { "stages": [ { @@ -310,7 +318,7 @@ def test_hunyuan_image3_pixel_accuracy_online(accuracy_artifact_root: Path) -> N @pytest.mark.full_model -@hardware_test(res={"cuda": ["H100", "B200"], "npu": "A3"}, num_cards=4) +@hardware_test(res={"cuda": ["H100", "B200"], "rocm": "MI325", "npu": "A3"}, num_cards=4) def test_hunyuan_image3_pixel_accuracy_offline(accuracy_artifact_root: Path) -> None: model = _model_name() output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME) From b160d3667ef637899cda618a7e9f4f08334a7ba2 Mon Sep 17 00:00:00 2001 From: akshatvishu Date: Sun, 16 Aug 2026 02:19:36 +0530 Subject: [PATCH 2/3] [BugFix][ROCm] Keep Hunyuan GroupNorm on PyTorch Signed-off-by: akshatvishu --- tests/platforms/test_rocm_groupnorm_patch.py | 55 +++++++++++++++++++ .../rocm/patch/worker/patch_groupnorm.py | 8 +++ 2 files changed, 63 insertions(+) create mode 100644 tests/platforms/test_rocm_groupnorm_patch.py diff --git a/tests/platforms/test_rocm_groupnorm_patch.py b/tests/platforms/test_rocm_groupnorm_patch.py new file mode 100644 index 00000000000..c95157648f1 --- /dev/null +++ b/tests/platforms/test_rocm_groupnorm_patch.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +import importlib.util +import sys +import types +from pathlib import Path + +import pytest +import torch.nn as nn + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +_PATCH_PATH = Path(__file__).parents[2] / "vllm_omni" / "platforms" / "rocm" / "patch" / "worker" / "patch_groupnorm.py" +_SPEC = importlib.util.spec_from_file_location("test_patch_groupnorm", _PATCH_PATH) +assert _SPEC is not None and _SPEC.loader is not None +patch_groupnorm = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = patch_groupnorm +_SPEC.loader.exec_module(patch_groupnorm) +patch_groupnorm._registry_mod.initialize_model = patch_groupnorm._original_initialize_model + + +def test_hunyuan_image3_keeps_pytorch_groupnorm(monkeypatch): + model = types.SimpleNamespace(vae=nn.Sequential(nn.GroupNorm(4, 8))) + monkeypatch.setattr(patch_groupnorm, "_original_initialize_model", lambda _: model) + monkeypatch.setattr( + patch_groupnorm, + "_replace_groupnorm_with_aiter", + lambda _: pytest.fail("Hunyuan Image 3.0 must keep PyTorch GroupNorm"), + ) + + od_config = types.SimpleNamespace(model_class_name="HunyuanImage3ForCausalMM") + assert patch_groupnorm._patched_initialize_model(od_config) is model + assert type(model.vae[0]) is nn.GroupNorm + + +def test_other_models_keep_aiter_groupnorm_patch(monkeypatch): + model = types.SimpleNamespace(vae=nn.Sequential(nn.GroupNorm(4, 8))) + monkeypatch.setattr(patch_groupnorm, "_original_initialize_model", lambda _: model) + + replaced = False + + def replace_groupnorm(_vae): + nonlocal replaced + replaced = True + return True + + monkeypatch.setattr(patch_groupnorm, "_replace_groupnorm_with_aiter", replace_groupnorm) + aiter_ops = types.ModuleType("vllm._aiter_ops") + setattr(aiter_ops, "is_aiter_found_and_supported", lambda: True) + monkeypatch.setitem(sys.modules, "vllm._aiter_ops", aiter_ops) + + od_config = types.SimpleNamespace(model_class_name="OtherDiffusionModel") + assert patch_groupnorm._patched_initialize_model(od_config) is model + assert replaced diff --git a/vllm_omni/platforms/rocm/patch/worker/patch_groupnorm.py b/vllm_omni/platforms/rocm/patch/worker/patch_groupnorm.py index 7de866e73c2..94de56d3835 100644 --- a/vllm_omni/platforms/rocm/patch/worker/patch_groupnorm.py +++ b/vllm_omni/platforms/rocm/patch/worker/patch_groupnorm.py @@ -1,4 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Patch ``initialize_model`` to replace VAE GroupNorm with AITER GroupNorm on ROCm.""" @@ -42,6 +43,13 @@ def _patched_initialize_model(od_config): model = _original_initialize_model(od_config) if hasattr(model, "vae"): + # AITER GroupNorm does not preserve the PyTorch autocast behavior required + # by Hunyuan Image 3.0. Keep PyTorch GroupNorm until the fix is released: + # https://github.com/ROCm/aiter/issues/4780 + # https://github.com/ROCm/aiter/pull/4779 + if od_config.model_class_name == "HunyuanImage3ForCausalMM": + return model + try: from vllm._aiter_ops import is_aiter_found_and_supported From e52c180504c98126784d6b4f092cf17960c4ec20 Mon Sep 17 00:00:00 2001 From: andyluo7 Date: Tue, 6 Oct 2026 08:10:53 -0700 Subject: [PATCH 3/3] test(rocm): calibrate HunyuanImage3 PSNR threshold Signed-off-by: andyluo7 --- tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py index bd0acbedd5b..fe10b4c2a74 100644 --- a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py +++ b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py @@ -42,6 +42,7 @@ PROMPT = "A brown and white dog is running on the grass." MEAN_THRESHOLD = 3e-2 P99_THRESHOLD = 3e-1 +PSNR_THRESHOLD_ROCM = 29.0 PSNR_THRESHOLD_NPU = 26.0 # Per-device SSIM/PSNR for online/offline vs baseline. Unlisted devices use ``default``. @@ -56,6 +57,8 @@ def _psnr_threshold(thresholds: SimilarityThresholds | None = None) -> float: if current_omni_platform.is_npu(): return PSNR_THRESHOLD_NPU + if current_omni_platform.is_rocm(): + return PSNR_THRESHOLD_ROCM if thresholds is None: thresholds = resolve_similarity_thresholds(SIMILARITY_THRESHOLDS_BY_DEVICE) return thresholds.psnr