[CI] Solidify speculative decoding E2E coverage

.buildkite/test_areas/spec_decode.yaml

@@ -29,21 +29,47 @@ steps:
       depends_on:
       - image-build-amd
 
-- label: Spec Decode Eagle
-  key: spec-decode-eagle
+- label: "Spec Decode Eagle 1: DeepSeek + Qwen"
+  key: spec-decode-eagle-1-deepseek-qwen
   timeout_in_minutes: 25
   device: h200_35gb
   source_file_dependencies:
     - vllm/v1/spec_decode/
     - vllm/v1/worker/gpu/spec_decode/
     - tests/v1/e2e/spec_decode/
   commands:
-    - pytest -v -s v1/e2e/spec_decode/eagle/
+    - pytest -v -s v1/e2e/spec_decode/eagle/ -k "deepseek_eagle or qwen3_eagle3"
   mirror:
     amd:
       dind: false
       device: mi300_1
-      timeout_in_minutes: 55
+      timeout_in_minutes: 25
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - vllm/v1/spec_decode/
+      - vllm/v1/worker/gpu/spec_decode/
+      - vllm/model_executor/model_loader/
+      - vllm/v1/sample/
+      - vllm/model_executor/layers/
+      - tests/v1/e2e/spec_decode/
+      - vllm/platforms/rocm.py
+
+- label: "Spec Decode Eagle 2: Llama 3 + Qwen VL + Other"
+  key: spec-decode-eagle-2-llama3-qwen-vl-other
+  timeout_in_minutes: 25
+  device: h200_35gb
+  source_file_dependencies:
+    - vllm/v1/spec_decode/
+    - vllm/v1/worker/gpu/spec_decode/
+    - tests/v1/e2e/spec_decode/
+  commands:
+    - pytest -v -s v1/e2e/spec_decode/eagle/ -k "not deepseek_eagle and not qwen3_eagle3"
+  mirror:
+    amd:
+      dind: false
+      device: mi300_1
+      timeout_in_minutes: 25
       depends_on:
       - image-build-amd
       source_file_dependencies:
@@ -192,8 +218,8 @@ steps:
     - export VLLM_ALLOW_INSECURE_SERIALIZATION=1
     - pytest -v -s v1/spec_decode/test_speculators_correctness.py -m slow_test
 
-- label: Spec Decode DeepSeek MTP Parallel Load (B200)
-  key: spec-decode-deepseek-mtp-parallel-load-b200
+- label: Spec Decode DeepSeek MTP Parallel Load (2xB200-2xMI300)
+  key: spec-decode-deepseek-mtp-parallel-load-2xb200-2xmi300
   timeout_in_minutes: 30
   device: b200-k8s
   optional: true
@@ -207,6 +233,26 @@ steps:
     - tests/v1/e2e/spec_decode/test_mtp_parallel_load.py
   commands:
     - pytest -v -s v1/e2e/spec_decode/test_mtp_parallel_load.py
+  mirror:
+    amd:
+      dind: false
+      device: mi300_2
+      timeout_in_minutes: 45
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - vllm/distributed/
+      - vllm/model_executor/model_loader/
+      - vllm/model_executor/layers/
+      - vllm/model_executor/models/deepseek_mtp.py
+      - vllm/model_executor/models/deepseek_v2.py
+      - vllm/v1/attention/
+      - vllm/v1/engine/
+      - vllm/v1/spec_decode/
+      - vllm/v1/worker/gpu/spec_decode/
+      - vllm/v1/worker/gpu_model_runner.py
+      - tests/v1/e2e/spec_decode/test_mtp_parallel_load.py
+      - vllm/platforms/rocm.py
 
 - label: Spec Decode AL DFlash Nightly
   key: spec-decode-dflash-nightly
@@ -221,6 +267,27 @@ steps:
     - tests/v1/e2e/spec_decode/
   commands:
     - pytest -v -s v1/e2e/spec_decode/acceptance_rates/dflash/
+  mirror:
+    amd:
+      dind: false
+      device: mi300_1
+      timeout_in_minutes: 120
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - vllm/v1/spec_decode/
+      - vllm/v1/worker/gpu/spec_decode/
+      - vllm/v1/attention/
+      - vllm/v1/sample/
+      - vllm/model_executor/model_loader/
+      - vllm/model_executor/layers/
+      - vllm/model_executor/models/qwen2.py
+      - vllm/model_executor/models/qwen3.py
+      - vllm/model_executor/models/qwen3_dflash.py
+      - vllm/model_executor/models/laguna.py
+      - vllm/model_executor/models/laguna_dflash.py
+      - tests/v1/e2e/spec_decode/
+      - vllm/platforms/rocm.py
 
 - label: Spec Decode AL DSpark Nightly
   key: spec-decode-dspark-nightly
@@ -235,6 +302,31 @@ steps:
     - tests/v1/e2e/spec_decode/
   commands:
     - pytest -v -s v1/e2e/spec_decode/acceptance_rates/dspark/
+  mirror:
+    amd:
+      dind: false
+      device: mi300_1
+      timeout_in_minutes: 90
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - vllm/v1/spec_decode/
+      - vllm/v1/worker/gpu/spec_decode/
+      - vllm/v1/attention/
+      - vllm/v1/sample/
+      - vllm/model_executor/model_loader/
+      - vllm/model_executor/layers/
+      - vllm/model_executor/models/qwen2.py
+      - vllm/model_executor/models/qwen3.py
+      - vllm/model_executor/models/qwen3_dflash.py
+      - vllm/model_executor/models/qwen3_dspark.py
+      - vllm/model_executor/models/gemma4.py
+      - vllm/model_executor/models/gemma4_mm.py
+      - vllm/model_executor/models/gemma4_unified.py
+      - vllm/model_executor/models/gemma4_mtp.py
+      - vllm/model_executor/models/gemma4_dspark.py
+      - tests/v1/e2e/spec_decode/
+      - vllm/platforms/rocm.py
 
 - label: Spec Decode AL MTP + Other Acceptance Nightly
   key: spec-decode-mtp-other-acceptance-nightly

tests/v1/e2e/spec_decode/acceptance_rates/dflash/test_dflash.py

@@ -4,12 +4,10 @@
 from dataclasses import dataclass
 
 import pytest
-import torch
 
 from tests.evals.gsm8k.gsm8k_eval import evaluate_gsm8k_offline
 from tests.utils import single_gpu_only
-from vllm import LLM
-from vllm.distributed import cleanup_dist_env_and_memory
+from vllm.config import CompilationConfig
 
 from ...utils import compute_acceptance_len
 from ..utils import run_acceptance_length_eval
@@ -56,9 +54,11 @@ class DFlashCorrectnessConfig:
 def test_dflash_reference_acceptance_lengths(
     monkeypatch: pytest.MonkeyPatch,
     use_mrv2: bool,
+    vllm_runner,
 ):
     run_acceptance_length_eval(
         monkeypatch,
+        vllm_runner,
         spec_config={
             "model": QWEN3_DFLASH.model,
             "trust_remote_code": True,
@@ -110,14 +110,16 @@ def test_dflash_correctness(
     monkeypatch: pytest.MonkeyPatch,
     config: DFlashCorrectnessConfig,
     use_mrv2: bool,
+    vllm_runner,
 ):
     """Guard GSM8K accuracy and batched acceptance length for DFlash models."""
     monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_mrv2 else "0")
     if config.disable_flashinfer_sampler:
         monkeypatch.setenv("VLLM_USE_FLASHINFER_SAMPLER", "0")
 
-    spec_llm = LLM(
-        model=config.model,
+    with vllm_runner(
+        config.model,
+        block_size=None,
         trust_remote_code=True,
         speculative_config={
             "method": "dflash",
@@ -130,23 +132,30 @@ def test_dflash_correctness(
         gpu_memory_utilization=0.85,
         enforce_eager=config.enforce_eager,
         disable_log_stats=False,
-    )
-
-    results = evaluate_gsm8k_offline(
-        spec_llm,
-        num_questions=config.num_questions,
-        use_chat_completions=config.use_chat_completions,
-    )
-    accuracy = results["accuracy"]
-    acceptance_len = compute_acceptance_len(spec_llm.get_metrics())
-    print(
-        f"{config.model}: GSM8K accuracy={accuracy:.3f}, "
-        f"acceptance_len={acceptance_len:.2f}"
-    )
-
-    assert accuracy >= config.expected_accuracy
-    assert acceptance_len >= config.expected_acceptance_len
-
-    del spec_llm
-    torch.accelerator.empty_cache()
-    cleanup_dist_env_and_memory()
+        enable_chunked_prefill=None,
+        compilation_config=CompilationConfig(),
+    ) as spec_runner:
+        spec_llm = spec_runner.llm
+        results = evaluate_gsm8k_offline(
+            spec_llm,
+            num_questions=config.num_questions,
+            use_chat_completions=config.use_chat_completions,
+        )
+        accuracy = results["accuracy"]
+        acceptance_len = compute_acceptance_len(spec_llm.get_metrics())
+        context = (
+            f"DFlash target={config.model}, draft={config.draft_model}, MRV2={use_mrv2}"
+        )
+        print(
+            f"{context}: GSM8K accuracy={accuracy:.3f}, "
+            f"acceptance_len={acceptance_len:.2f}"
+        )
+
+        assert accuracy >= config.expected_accuracy, (
+            f"{context}: GSM8K accuracy {accuracy:.3f} is below "
+            f"{config.expected_accuracy:.3f}; acceptance_len={acceptance_len:.3f}"
+        )
+        assert acceptance_len >= config.expected_acceptance_len, (
+            f"{context}: acceptance_len {acceptance_len:.3f} is below "
+            f"{config.expected_acceptance_len:.3f}; accuracy={accuracy:.3f}"
+        )