From 765a3f7e47561876a9d906433e0caebe3a7c7ffe Mon Sep 17 00:00:00 2001 From: yancy Date: Tue, 13 Oct 2026 15:48:38 +0000 Subject: [PATCH 1/2] python/sglang/srt/layers/quantization/fp4_utils.py: Skip marlin backend for fp4 on SM100 (Blackwell) FP4 Marlin is weight-only (W4A16) and does not support W4A4 GEMM. FlashInfer's mm_fp4 rejects backend='marlin' on SM100+ with: BackendSupportedError: mm_fp4 does not support backend 'marlin' with capability 100 When the user explicitly passes --fp4-gemm-backend marlin on Blackwell GPUs, or auto-selection unexpectedly chooses marlin, fall back to flashinfer_cutedsl (SM100) or flashinfer_cutlass (other SM12x) with a clear warning. Fixes: #43988 Signed-off-by: yancy --- python/sglang/srt/layers/quantization/fp4_utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python/sglang/srt/layers/quantization/fp4_utils.py b/python/sglang/srt/layers/quantization/fp4_utils.py index 0cdafd64bb0c..743227517861 100644 --- a/python/sglang/srt/layers/quantization/fp4_utils.py +++ b/python/sglang/srt/layers/quantization/fp4_utils.py @@ -159,6 +159,19 @@ def initialize_fp4_gemm_config() -> None: else: backend = "flashinfer_cutlass" + # FP4 Marlin is weight-only (W4A16) and does not support W4A4 GEMM. + # FlashInfer's mm_fp4 rejects backend='marlin' on SM100+, so force a + # fallback whenever the user (or auto-selection) picks marlin on + # Blackwell GPUs. See issue #43988. + if backend == "marlin" and get_platform().is_blackwell: + fallback = "flashinfer_cutedsl" if get_platform().is_sm100 else "flashinfer_cutlass" + logger.warning( + "FP4 Marlin does not support SM100+ (Blackwell); falling back to %s. " + "FlashInfer's mm_fp4 has no 'marlin' backend for capability 100.", + fallback, + ) + backend = fallback + FP4_GEMM_RUNNER_BACKEND = Fp4GemmRunnerBackend(backend) From 061e04027ff1d14857caf1fe7b903360f0775e01 Mon Sep 17 00:00:00 2001 From: yancy Date: Tue, 13 Oct 2026 18:15:34 +0000 Subject: [PATCH 2/2] style: apply pre-commit lint fixes Signed-off-by: yancy --- python/sglang/srt/layers/quantization/fp4_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/layers/quantization/fp4_utils.py b/python/sglang/srt/layers/quantization/fp4_utils.py index 743227517861..26da51c5f609 100644 --- a/python/sglang/srt/layers/quantization/fp4_utils.py +++ b/python/sglang/srt/layers/quantization/fp4_utils.py @@ -164,7 +164,9 @@ def initialize_fp4_gemm_config() -> None: # fallback whenever the user (or auto-selection) picks marlin on # Blackwell GPUs. See issue #43988. if backend == "marlin" and get_platform().is_blackwell: - fallback = "flashinfer_cutedsl" if get_platform().is_sm100 else "flashinfer_cutlass" + fallback = ( + "flashinfer_cutedsl" if get_platform().is_sm100 else "flashinfer_cutlass" + ) logger.warning( "FP4 Marlin does not support SM100+ (Blackwell); falling back to %s. " "FlashInfer's mm_fp4 has no 'marlin' backend for capability 100.",