[NVFP4] NVFP4 MOE emulation fallback for H100/MI300/MI350, standardize TritonExperts usage for OCP MX emulation

tests/models/quantization/test_nvfp4.py

@@ -90,19 +90,22 @@ def test_models(example_prompts, model_name) -> None:
 EAGER = [True, False]
 
 
-@pytest.mark.skipif(
-    not current_platform.has_device_capability(100),
-    reason="modelopt_fp4 is not supported on this GPU type.",
-)
+if not current_platform.has_device_capability(100):
+    TEST_NVFP4_BACKENDS = ["emulation"]
+else:
+    TEST_NVFP4_BACKENDS = [
+        "emulation",
+        "flashinfer-cudnn",
+        "flashinfer-trtllm",  # the small seq_len ensures trtllm_8x4_layout backend is used
+        "flashinfer-cutlass",
+    ]
+
+
 @pytest.mark.parametrize("model", ["nvidia/Llama-3.1-8B-Instruct-NVFP4"])
 @pytest.mark.parametrize("eager", EAGER)
 @pytest.mark.parametrize(
     "backend",
-    [
-        "flashinfer-cudnn",
-        "flashinfer-trtllm",  # the small seq_len ensures trtllm_8x4_layout backend is used
-        "flashinfer-cutlass",
-    ],
+    TEST_NVFP4_BACKENDS,
 )
 def test_nvfp4(vllm_runner, model, eager, backend, monkeypatch):
     monkeypatch.setenv("VLLM_NVFP4_GEMM_BACKEND", backend)

tests/quantization/test_blackwell_moe.py

@@ -10,11 +10,6 @@
 from tests.utils import RemoteOpenAIServer
 from vllm.platforms import current_platform
 
-if not current_platform.is_device_capability_family(100):
-    pytest.skip(
-        "This test only runs on Blackwell GPUs (SM10x).", allow_module_level=True
-    )
-
 
 @pytest.fixture(scope="module", autouse=True)
 def set_test_environment():
@@ -84,6 +79,10 @@ def can_initialize(
         "'list'."
     )
 )
+@pytest.mark.skipif(
+    not current_platform.is_device_capability_family(100),
+    reason="This test only runs on Blackwell GPUs (SM10x).",
+)
 def test_llama4_fp8_tensor_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
     can_initialize(
         "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8",
@@ -92,6 +91,10 @@ def test_llama4_fp8_tensor_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatc
     )
 
 
+@pytest.mark.skipif(
+    not current_platform.is_device_capability_family(100),
+    reason="This test only runs on Blackwell GPUs (SM10x).",
+)
 def test_llama4_fp8_tensor_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
     can_initialize(
         "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8",
@@ -100,6 +103,10 @@ def test_llama4_fp8_tensor_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch
     )
 
 
+@pytest.mark.skipif(
+    not current_platform.is_device_capability_family(100),
+    reason="This test only runs on Blackwell GPUs (SM10x).",
+)
 def test_llama4_nvfp4_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
     can_initialize(
         "nvidia/Llama-4-Scout-17B-16E-Instruct-FP4",
@@ -108,6 +115,10 @@ def test_llama4_nvfp4_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
     )
 
 
+@pytest.mark.skipif(
+    not current_platform.is_device_capability_family(100),
+    reason="This test only runs on Blackwell GPUs (SM10x).",
+)
 def test_llama4_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
     can_initialize(
         "nvidia/Llama-4-Scout-17B-16E-Instruct-FP4",
@@ -116,9 +127,21 @@ def test_llama4_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
     )
 
 
+def test_llama4_nvfp4_moe_emulation(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "nvidia/Llama-4-Scout-17B-16E-Instruct-FP4",
+        hf_overrides=HF_OVERRIDE_MM,
+        extra_args=["--moe-backend=emulation", "--gpu-memory-utilization", "0.8"],
+    )
+
+
 ## DeepSeekV3 ##
 
 
+@pytest.mark.skipif(
+    not current_platform.is_device_capability_family(100),
+    reason="This test only runs on Blackwell GPUs (SM10x).",
+)
 def test_deepseek_fp8_block_moe_deep_gemm(monkeypatch: pytest.MonkeyPatch):
     can_initialize(
         "deepseek-ai/DeepSeek-V3.1",
@@ -141,6 +164,10 @@ def test_deepseek_fp8_block_moe_vllm_triton(monkeypatch: pytest.MonkeyPatch):
         "Expected failure: assert self.block_quant is None"
     )
 )
+@pytest.mark.skipif(
+    not current_platform.is_device_capability_family(100),
+    reason="This test only runs on Blackwell GPUs (SM10x).",
+)
 def test_deepseek_fp8_block_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
     can_initialize(
         "deepseek-ai/DeepSeek-V3.1",
@@ -149,6 +176,10 @@ def test_deepseek_fp8_block_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPat
     )
 
 
+@pytest.mark.skipif(
+    not current_platform.is_device_capability_family(100),
+    reason="This test only runs on Blackwell GPUs (SM10x).",
+)
 def test_deepseek_fp8_block_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
     can_initialize(
         "deepseek-ai/DeepSeek-V3.1",
@@ -157,6 +188,10 @@ def test_deepseek_fp8_block_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatc
     )
 
 
+@pytest.mark.skipif(
+    not current_platform.is_device_capability_family(100),
+    reason="This test only runs on Blackwell GPUs (SM10x).",
+)
 def test_deepseek_nvfp4_moe_flashinfer_vllm(monkeypatch: pytest.MonkeyPatch):
     can_initialize(
         "nvidia/DeepSeek-R1-0528-FP4-v2",
@@ -165,6 +200,10 @@ def test_deepseek_nvfp4_moe_flashinfer_vllm(monkeypatch: pytest.MonkeyPatch):
     )
 
 
+@pytest.mark.skipif(
+    not current_platform.is_device_capability_family(100),
+    reason="This test only runs on Blackwell GPUs (SM10x).",
+)
 def test_deepseek_nvfp4_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
     can_initialize(
         "nvidia/DeepSeek-R1-0528-FP4-v2",
@@ -173,6 +212,10 @@ def test_deepseek_nvfp4_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
     )
 
 
+@pytest.mark.skipif(
+    not current_platform.is_device_capability_family(100),
+    reason="This test only runs on Blackwell GPUs (SM10x).",
+)
 def test_deepseek_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
     can_initialize(
         "nvidia/DeepSeek-R1-0528-FP4-v2",
@@ -184,16 +227,28 @@ def test_deepseek_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
 ## GPT-OSS ##
 
 
+@pytest.mark.skipif(
+    not current_platform.is_device_capability_family(100),
+    reason="This test only runs on Blackwell GPUs (SM10x).",
+)
 def test_gptoss_mxfp4bf16_moe_flashinfer(monkeypatch: pytest.MonkeyPatch):
     monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16", "1")
     can_initialize("openai/gpt-oss-20b", hf_overrides=HF_OVERRIDE_TEXT)
 
 
+@pytest.mark.skipif(
+    not current_platform.is_device_capability_family(100),
+    reason="This test only runs on Blackwell GPUs (SM10x).",
+)
 def test_gptoss_mxfp4mxfp8_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
     monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS", "1")
     can_initialize("openai/gpt-oss-20b", hf_overrides=HF_OVERRIDE_TEXT)
 
 
+@pytest.mark.skipif(
+    not current_platform.is_device_capability_family(100),
+    reason="This test only runs on Blackwell GPUs (SM10x).",
+)
 def test_gptoss_mxfp4mxfp8_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
     monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", "1")
     can_initialize("openai/gpt-oss-20b", hf_overrides=HF_OVERRIDE_TEXT)
@@ -210,6 +265,10 @@ def test_gptoss_eager(monkeypatch: pytest.MonkeyPatch):
 ## Qwen3 Next ##
 
 
+@pytest.mark.skipif(
+    not current_platform.is_device_capability_family(100),
+    reason="This test only runs on Blackwell GPUs (SM10x).",
+)
 def test_qwen3_next_bf16_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
     can_initialize(
         "Qwen/Qwen3-Next-80B-A3B-Instruct",
@@ -221,6 +280,10 @@ def test_qwen3_next_bf16_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
 ## NemoTron ##
 
 
+@pytest.mark.skipif(
+    not current_platform.is_device_capability_family(100),
+    reason="This test only runs on Blackwell GPUs (SM10x).",
+)
 def test_nemotron_fp8_moe_flashinfer_throughput(monkeypatch: pytest.MonkeyPatch):
     can_initialize(
         "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8",
@@ -236,6 +299,10 @@ def test_nemotron_fp8_moe_flashinfer_throughput(monkeypatch: pytest.MonkeyPatch)
         "no act_and_mul MLP layer."
     )
 )
+@pytest.mark.skipif(
+    not current_platform.is_device_capability_family(100),
+    reason="This test only runs on Blackwell GPUs (SM10x).",
+)
 def test_nemotron_fp8_moe_flashinfer_latency(monkeypatch: pytest.MonkeyPatch):
     can_initialize(
         "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8",
@@ -251,6 +318,10 @@ def test_nemotron_fp8_moe_flashinfer_latency(monkeypatch: pytest.MonkeyPatch):
         "no act_and_mul MLP layer."
     )
 )
+@pytest.mark.skipif(
+    not current_platform.is_device_capability_family(100),
+    reason="This test only runs on Blackwell GPUs (SM10x).",
+)
 def test_nemotron_fp8_moe_vllm_triton(monkeypatch: pytest.MonkeyPatch):
     can_initialize(
         "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8",
@@ -259,6 +330,10 @@ def test_nemotron_fp8_moe_vllm_triton(monkeypatch: pytest.MonkeyPatch):
     )
 
 
+@pytest.mark.skipif(
+    not current_platform.is_device_capability_family(100),
+    reason="This test only runs on Blackwell GPUs (SM10x).",
+)
 def test_nemotron_fp4_moe_flashinfer_throughput(monkeypatch: pytest.MonkeyPatch):
     can_initialize(
         "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4",
@@ -274,6 +349,10 @@ def test_nemotron_fp4_moe_flashinfer_throughput(monkeypatch: pytest.MonkeyPatch)
         "hidden_dim % 512 != 0."
     )
 )
+@pytest.mark.skipif(
+    not current_platform.is_device_capability_family(100),
+    reason="This test only runs on Blackwell GPUs (SM10x).",
+)
 def test_nemotron_fp4_moe_flashinfer_latency(monkeypatch: pytest.MonkeyPatch):
     can_initialize(
         "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4",

tests/quantization/test_compressed_tensors.py

@@ -639,9 +639,6 @@ def check_model(model):
         assert output
 
 
-@pytest.mark.skipif(
-    not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
-)
 @pytest.mark.parametrize(
     "args",
     [
@@ -671,7 +668,7 @@ def check_model(model):
             assert qkv_proj.scheme.group_size == 16
 
         llm.apply_model(check_model)
-        output = llm.generate_greedy("Hello my name is", max_tokens=4)
+        output = llm.generate_greedy(["Hello my name is"], max_tokens=4)
         print(output)
         assert output
 

vllm/config/kernel.py

@@ -19,6 +19,7 @@
     "flashinfer_cutedsl",
     "marlin",
     "aiter",
+    "emulation",
 ]
 
 
@@ -40,7 +41,10 @@ class KernelConfig:
     - "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels\n
     - "flashinfer_cutedsl": Use FlashInfer with CuteDSL kernels (FP4 only)\n
     - "marlin": Use Marlin kernels (weight-only quantization)\n
-    - "aiter": Use AMD AITer kernels (ROCm only)"""
+    - "aiter": Use AMD AITer kernels (ROCm only)
+    - "emulation": use BF16/FP16 GEMM, dequantizing weights and
+                   running QDQ on activations.
+    """
 
     @field_validator("moe_backend", mode="before")
     @classmethod

vllm/envs.py

@@ -1405,6 +1405,8 @@ def _get_or_set_default() -> str:
     # - "flashinfer-trtllm": use flashinfer trtllm GEMM backend
     # - "flashinfer-cutlass": use flashinfer cutlass GEMM backend
     # - "marlin": use marlin GEMM backend (for GPUs without native FP4 support)
+    # - "emulation": use BF16/FP16 GEMM,
+    #                dequantizing weights and running QDQ on activations.
     # - <none>: automatically pick an available backend
     "VLLM_NVFP4_GEMM_BACKEND": env_with_choices(
         "VLLM_NVFP4_GEMM_BACKEND",
@@ -1415,6 +1417,7 @@ def _get_or_set_default() -> str:
             "flashinfer-cutlass",
             "cutlass",
             "marlin",
+            "emulation",
         ],
     ),
     # Controls garbage collection during CUDA graph capture.

vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
-import flashinfer
+
 import torch
 
 import vllm.model_executor.layers.fused_moe.modular_kernel as mk
@@ -157,6 +157,8 @@ def apply(
         expert_tokens_meta: mk.ExpertTokensMetadata | None,
         apply_router_weight_on_input: bool,
     ):
+        import flashinfer
+
         assert activation in [MoEActivation.SILU, MoEActivation.RELU2_NO_MUL]
         assert a1q_scale is not None
         assert self.quant_config.w1_scale is not None
@@ -271,6 +273,8 @@ def apply(
         routed_scaling_factor: float | None = None,
         topk_group: int | None = None,
     ) -> torch.Tensor:
+        import flashinfer
+
         assert activation in [MoEActivation.SILU, MoEActivation.RELU2_NO_MUL]
         assert a1q_scale is not None
         assert self.quant_config.w1_scale is not None

vllm/model_executor/layers/fused_moe/fused_moe.py

@@ -1932,6 +1932,7 @@ def __init__(
         moe_config: FusedMoEConfig,
         quant_config: FusedMoEQuantConfig,
     ):
+        self.emulation = False
         super().__init__(moe_config, quant_config)
 
     @staticmethod
@@ -2130,6 +2131,8 @@ def apply(
             self.quant_dtype,
             self.per_act_token_quant,
             self.block_shape,
+            emulation=self.emulation,
+            topk_ids=topk_ids,
         )
 
         invoke_fused_moe_triton_kernel(