ci: repeat current ROCm stability integration

.buildkite/amd/test-amd-merge.yml

@@ -41,7 +41,52 @@ steps:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+        # Keep multi-GPU tests out of this one-GPU pool even when their module
+        # also carries the cpu marker for unit tests.
+        - "pytest -sv tests/diffusion -m 'core_model and cpu and not (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)' --ignore=tests/diffusion/cache/test_teacache_extractors.py --ignore=tests/diffusion/models/pi05/test_pi05_units.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+
+    # This module takes about 22 minutes on MI300 and was consistently assigned to shard 4,
+    # pushing that shard to the 45-minute timeout. Keep its module-scoped fixture in one lane.
+    - label: "Simple · Pi0.5 CPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 45
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/diffusion/models/pi05/test_pi05_units.py -m 'core_model and cpu'"
+
+    - label: "Diffusion · AuK VAE GPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 40
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - >-
+          timeout --signal=TERM --kill-after=2m 35m
+          pytest -sv tests/diffusion/models/auk/test_auk_vae_fastpath.py
+          -m 'core_model and rocm and cards_1' --run-level core_model
+
+    # AR paged attention is real GPU work, including cold AITER compilation.
+    # Keep its complete numerical matrix out of the CPU-marked Simple shards.
+    - label: "ROCm · AR Diffusion Paged Attention GPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 20
+      commands:
+        - export VLLM_ROCM_USE_AITER=1
+        - export AITER_JIT_DIR="/tmp/vllm-omni-ar-aiter-$$BUILDKITE_JOB_ID"
+        - export TORCH_EXTENSIONS_DIR="/tmp/vllm-omni-ar-torch-extensions-$$BUILDKITE_JOB_ID"
+        - mkdir -p "$$AITER_JIT_DIR" "$$TORCH_EXTENSIONS_DIR"
+        - >-
+          timeout --signal=TERM --kill-after=2m 15m
+          pytest -sv tests/diffusion/ar_diffusion/test_paged_attention.py
+          -m 'core_model and rocm and MI325 and cards_1' --run-level core_model
 
     - label: "Simple · Engine&Entrypoints Test"
       agent_pool: mi300_1
@@ -131,6 +176,11 @@ steps:
       grade: Blocking
       commands:
         - pytest -s -v tests/diffusion/distributed/test_tensor_parallel.py
+        - >-
+          timeout 20m pytest -s -v
+          tests/diffusion/attention/test_ulysses_uaa.py::test_ulysses_uaa_2d_mask_layout_matches_baseline
+          -m 'core_model and cards_2'
+          --run-level "core_model"
 
     - label: "Diffusion Model Test"
       agent_pool: mi300_1

.buildkite/amd/test-amd-nightly.yml

@@ -51,6 +51,8 @@ steps:
         mirror_hardwares: [amdproduction]
         grade: NonBlocking
         timeout_in_minutes: 120
+        artifact_paths:
+          - qwen3-omni-doc-artifacts/**/*
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
           - pytest -s -v tests/examples/online_serving/test_qwen3_omni.py -m "full_model and rocm and MI325 and cards_2" --run-level "full_model"

.buildkite/amd/test-amd-ready.yml

@@ -41,7 +41,52 @@ steps:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+        # Keep multi-GPU tests out of this one-GPU pool even when their module
+        # also carries the cpu marker for unit tests.
+        - "pytest -sv tests/diffusion -m 'core_model and cpu and not (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)' --ignore=tests/diffusion/cache/test_teacache_extractors.py --ignore=tests/diffusion/models/pi05/test_pi05_units.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+
+    # This module takes about 22 minutes on MI300 and was consistently assigned to shard 4,
+    # pushing that shard to the 45-minute timeout. Keep its module-scoped fixture in one lane.
+    - label: "Simple · Pi0.5 CPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 45
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/diffusion/models/pi05/test_pi05_units.py -m 'core_model and cpu'"
+
+    - label: "Diffusion · AuK VAE GPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 40
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - >-
+          timeout --signal=TERM --kill-after=2m 35m
+          pytest -sv tests/diffusion/models/auk/test_auk_vae_fastpath.py
+          -m 'core_model and rocm and cards_1' --run-level core_model
+
+    # AR paged attention is real GPU work, including cold AITER compilation.
+    # Keep its complete numerical matrix out of the CPU-marked Simple shards.
+    - label: "ROCm · AR Diffusion Paged Attention GPU Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 20
+      commands:
+        - export VLLM_ROCM_USE_AITER=1
+        - export AITER_JIT_DIR="/tmp/vllm-omni-ar-aiter-$$BUILDKITE_JOB_ID"
+        - export TORCH_EXTENSIONS_DIR="/tmp/vllm-omni-ar-torch-extensions-$$BUILDKITE_JOB_ID"
+        - mkdir -p "$$AITER_JIT_DIR" "$$TORCH_EXTENSIONS_DIR"
+        - >-
+          timeout --signal=TERM --kill-after=2m 15m
+          pytest -sv tests/diffusion/ar_diffusion/test_paged_attention.py
+          -m 'core_model and rocm and MI325 and cards_1' --run-level core_model
 
     - label: "Simple · Engine&Entrypoints Test"
       agent_pool: mi300_1
@@ -141,6 +186,11 @@ steps:
         - export VLLM_LOGGING_LEVEL=DEBUG
         - export VLLM_WORKER_MULTIPROC_METHOD=spawn
         - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
+        - >-
+          timeout 20m pytest -s -v
+          tests/diffusion/attention/test_ulysses_uaa.py::test_ulysses_uaa_2d_mask_layout_matches_baseline
+          -m 'core_model and cards_2'
+          --run-level "core_model"
         - >-
           timeout 20m pytest -s -v
           tests/diffusion/models/ltx2/test_ltx2_transformer_ulysses.py
@@ -165,6 +215,7 @@ steps:
         # registered multi-GPU marker from this one-GPU job.
         - >-
           timeout 15m pytest -sv tests/diffusion/models/
+          --ignore=tests/diffusion/models/auk/test_auk_vae_fastpath.py
           -m 'core_model and cuda and not
           (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
           --run-level "core_model"

.buildkite/amd/test-template-amd-omni.j2

@@ -52,6 +52,10 @@
 {{ indent }}                value: "0"
 {{ indent }}              - name: VLLM_ROCM_USE_AITER
 {{ indent }}                value: "0"
+{{ indent }}              - name: GPU_ARCHS
+{{ indent }}                value: "gfx942"
+{{ indent }}              - name: CLANG_TOOLCHAIN_PROGRAM_TIMEOUT
+{{ indent }}                value: "300"
 {{ indent }}              - name: PYTORCH_ROCM_ARCH
 {{ indent }}                value: ""
 {{ indent }}        volumes:
@@ -127,6 +131,8 @@ steps:
           MIOPEN_DEBUG_CONV_DIRECT: "0"
           MIOPEN_DEBUG_CONV_GEMM: "0"
           VLLM_ROCM_USE_AITER: "0"
+          GPU_ARCHS: "gfx942"
+          CLANG_TOOLCHAIN_PROGRAM_TIMEOUT: "300"
           PYTORCH_ROCM_ARCH: ""
 {% if step.env %}
 {% for name in step.env %}
@@ -203,6 +209,8 @@ steps:
       MIOPEN_DEBUG_CONV_DIRECT: "0"
       MIOPEN_DEBUG_CONV_GEMM: "0"
       VLLM_ROCM_USE_AITER: "0"
+      GPU_ARCHS: "gfx942"
+      CLANG_TOOLCHAIN_PROGRAM_TIMEOUT: "300"
       PYTORCH_ROCM_ARCH: ""
 {% if step.env %}
 {% for name in step.env %}

.buildkite/cuda/test-nightly.yml

@@ -48,6 +48,8 @@ steps:
       - label: ":full_moon: Omni · Doc Test with H100 · 2-GPU"
         source_file_dependencies: omni_qwen3_omni_doc
         timeout_in_minutes: 90
+        artifact_paths:
+          - qwen3-omni-doc-artifacts/**/*
         commands:
           - pytest -s -v tests/examples/ -m "full_model and omni and H100 and B200 and cards_2" --run-level "full_model"
 

tests/buildkite/test_amd_pipeline.py

@@ -14,6 +14,11 @@
 AMD_READY_PIPELINE = Path(".buildkite/amd/test-amd-ready.yml")
 AMD_TEMPLATE = Path(".buildkite/amd/test-template-amd-omni.j2")
 
+MULTI_GPU_MARKER_EXCLUSION = (
+    "core_model and cpu and not (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)"
+)
+ULYSSES_UAA_2D_NODE = "tests/diffusion/attention/test_ulysses_uaa.py::test_ulysses_uaa_2d_mask_layout_matches_baseline"
+
 
 def _find_step(label: str, pipeline_path: Path = AMD_MERGE_PIPELINE) -> dict:
     pipeline = yaml.safe_load(pipeline_path.read_text(encoding="utf-8"))
@@ -57,15 +62,24 @@ def test_qwen3_accuracy_defers_artifact_path_expansion() -> None:
     assert step["artifact_paths"] == ["tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json"]
 
 
-def test_ready_diffusion_cpu_suite_is_sharded() -> None:
-    step = _find_step("Simple · Diffusion Test · Shard %N/%t", AMD_READY_PIPELINE)
+@pytest.mark.parametrize("pipeline_path", [AMD_READY_PIPELINE, AMD_MERGE_PIPELINE], ids=["ready", "merge"])
+def test_diffusion_cpu_suite_is_sharded_and_pi05_isolated(pipeline_path: Path) -> None:
+    step = _find_step("Simple · Diffusion Test · Shard %N/%t", pipeline_path)
     pytest_command = next(command for command in step["commands"] if "pytest" in command)
 
     assert step["parallelism"] == 4
     assert step["timeout_in_minutes"] == 45
+    assert "--ignore=tests/diffusion/models/pi05/test_pi05_units.py" in pytest_command
     assert "--num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT" in pytest_command
     assert "--shard-id=$$BUILDKITE_PARALLEL_JOB" in pytest_command
 
+    pi05_step = _find_step("Simple · Pi0.5 CPU Test", pipeline_path)
+    assert pi05_step["grade"] == "Blocking"
+    assert pi05_step["timeout_in_minutes"] == 45
+    pi05_command = next(command for command in pi05_step["commands"] if "pytest" in command)
+    assert "tests/diffusion/models/pi05/test_pi05_units.py" in pi05_command
+    assert "--num-shards" not in pi05_command
+
 
 def test_z_image_merge_timeout_covers_cold_aiter_compile() -> None:
     step = _find_step("Diffusion Model Test")
@@ -74,6 +88,37 @@ def test_z_image_merge_timeout_covers_cold_aiter_compile() -> None:
     assert split(pytest_command)[:2] == ["timeout", "55m"]
 
 
+@pytest.mark.parametrize("pipeline_path", [AMD_READY_PIPELINE, AMD_MERGE_PIPELINE])
+def test_diffusion_cpu_suite_excludes_multi_gpu_tests(pipeline_path: Path) -> None:
+    step = _find_step("Simple · Diffusion Test · Shard %N/%t", pipeline_path)
+    pytest_command = next(command for command in step["commands"] if "pytest" in command)
+    argv = split(pytest_command)
+
+    marker_index = argv.index("-m")
+    assert argv[marker_index + 1] == MULTI_GPU_MARKER_EXCLUSION
+
+
+@pytest.mark.parametrize(
+    ("pipeline_path", "step_label"),
+    [
+        (AMD_READY_PIPELINE, "Diffusion Sequence Parallelism Test"),
+        (AMD_MERGE_PIPELINE, "Diffusion Tensor Parallelism Test"),
+    ],
+)
+def test_ulysses_uaa_2d_mask_runs_on_two_gpu_lane(
+    pipeline_path: Path,
+    step_label: str,
+) -> None:
+    step = _find_step(step_label, pipeline_path)
+    pytest_command = next(command for command in step["commands"] if ULYSSES_UAA_2D_NODE in command)
+    argv = split(pytest_command)
+
+    assert step["agent_pool"] == "mi300_2"
+    assert ULYSSES_UAA_2D_NODE in argv
+    marker_index = argv.index("-m")
+    assert argv[marker_index + 1] == "core_model and cards_2"
+
+
 def test_cosyvoice_ready_smoke_uses_sdpa() -> None:
     step = _find_step("CosyVoice3-TTS E2E Smoke (SDPA)", AMD_READY_PIPELINE)
 
@@ -104,3 +149,19 @@ def test_amd_template_preserves_step_retry_policy() -> None:
     # policy when the source suite is rendered into the uploaded pipeline.
     assert template.count("{% if step.retry %}") == 2
     assert template.count("{% for retry_rule in step.retry.automatic %}") == 2
+
+
+def test_amd_template_pins_aiter_jit_architecture() -> None:
+    template = AMD_TEMPLATE.read_text(encoding="utf-8")
+
+    # AITER otherwise emits --offload-arch=native for every Ninja compile
+    # process. Under cold, highly parallel JIT builds, the repeated ROCm
+    # architecture probes can time out before compilation starts.
+    assert template.count("GPU_ARCHS") == 3
+    assert template.count('GPU_ARCHS: "gfx942"') == 2
+    assert template.count("- name: GPU_ARCHS") == 1
+
+    # Keep a bounded fallback for any toolchain child probes that remain.
+    assert template.count("CLANG_TOOLCHAIN_PROGRAM_TIMEOUT") == 3
+    assert template.count('CLANG_TOOLCHAIN_PROGRAM_TIMEOUT: "300"') == 2
+    assert template.count("- name: CLANG_TOOLCHAIN_PROGRAM_TIMEOUT") == 1