From 7477b5290c3181eb4494a30db21ec372f606d1c2 Mon Sep 17 00:00:00 2001 From: jbernloehr Date: Tue, 29 Sep 2026 11:09:37 +0200 Subject: [PATCH 1/2] fix: route SM110 FP8 and NVFP4 auto backends --- python/sglang/srt/arg_groups/overrides.py | 4 +- .../srt/layers/quantization/fp8_utils.py | 3 + python/sglang/srt/runtime_context.py | 1 + python/sglang/srt/utils/common.py | 5 + .../unit/test_sm110_backend_routing.py | 95 +++++++++++++++++++ 5 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 test/registered/unit/test_sm110_backend_routing.py diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index 597c19f098a1..04402f21aba3 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -1446,11 +1446,11 @@ def _moe_runner_backend_quant_constraints(view: Any) -> dict: if ( moe_runner_backend == "auto" and view.quantization == "modelopt_fp4" - and get_platform().is_sm120 + and (get_platform().is_sm120 or get_platform().is_sm110) ): moe_runner_backend = "flashinfer_cutlass" logger.info( - "Use flashinfer_cutlass as MoE runner backend on SM120 for " + "Use flashinfer_cutlass as MoE runner backend on SM110/SM120 for " "modelopt_fp4 (trtllm-gen MoE kernels are SM100-only)" ) if moe_runner_backend != view.moe_runner_backend: diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index 2c4a70be9fff..883af39a4c31 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -1009,6 +1009,9 @@ def initialize_fp8_gemm_config() -> None: backend = get_exec().kernel.fp8_gemm_runner_backend if backend == "auto" and get_platform().is_sm120: backend = "cutlass" + elif backend == "auto" and get_platform().is_sm110: + # DeepGEMM's block-FP8 kernel does not support SM110. + backend = "triton" backend = Fp8GemmRunnerBackend(backend) diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index a361146e76fa..a1920a115624 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -1776,6 +1776,7 @@ def pre_capture_activation_reserve_mb(gpu_mem: float | None) -> float: "is_sm90": "is_sm90_supported", "is_sm100": "is_sm100_supported", "is_sm100_or_sm110": "is_sm100_or_sm110_supported", + "is_sm110": "is_sm110_supported", "is_sm120": "is_sm120_supported", "is_blackwell": "is_blackwell_supported", "is_hopper_with_cuda_12_3": "is_hopper_with_cuda_12_3", diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index a05df0388836..b6ba5788d6cd 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -311,6 +311,11 @@ def _check_cuda_device_version( cuda_version=(12, 8), ) ) +is_sm110_supported = lru_cache(maxsize=1)( + partial( + _check_cuda_device_version, device_capability_majors=[11], cuda_version=(12, 8) + ) +) is_sm80_supported = lru_cache(maxsize=1)( partial( _check_cuda_device_version, device_capability_majors=[8], cuda_version=(11, 0) diff --git a/test/registered/unit/test_sm110_backend_routing.py b/test/registered/unit/test_sm110_backend_routing.py new file mode 100644 index 000000000000..c083a8e8acaf --- /dev/null +++ b/test/registered/unit/test_sm110_backend_routing.py @@ -0,0 +1,95 @@ +"""CPU regression tests for Thor's SM110 automatic backend selection.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from sglang.srt.arg_groups.overrides import ( + ResolvedView, + _moe_runner_backend_quant_constraints, +) +from sglang.srt.layers.quantization import fp8_utils +from sglang.srt.layers.quantization.fp8_utils import Fp8GemmRunnerBackend +from sglang.srt.runtime_context import get_platform, override_platform +from sglang.srt.utils import common +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestSm110BackendRouting(CustomTestCase): + def test_sm110_probe_requires_major_11_and_cuda_12_8(self): + try: + for device_sm, cuda_version, expected in ( + (100, "13.0", False), + (110, "12.7", False), + (110, "12.8", True), + (110, "13.0", True), + (120, "13.0", False), + ): + with self.subTest(device_sm=device_sm, cuda=cuda_version): + with ( + patch.object(common, "is_cuda", return_value=True), + patch.object(common, "get_device_sm", return_value=device_sm), + patch.object(common.torch.version, "cuda", cuda_version), + ): + common.is_sm110_supported.cache_clear() + self.assertEqual(get_platform().is_sm110, expected) + finally: + common.is_sm110_supported.cache_clear() + + def test_fp8_auto_routes_sm110_to_triton_only(self): + for is_sm110, is_sm120, requested, expected in ( + (True, False, "auto", Fp8GemmRunnerBackend.TRITON), + (False, True, "auto", Fp8GemmRunnerBackend.CUTLASS), + (False, False, "auto", Fp8GemmRunnerBackend.AUTO), + (True, False, "deep_gemm", Fp8GemmRunnerBackend.DEEP_GEMM), + ): + with self.subTest(sm110=is_sm110, sm120=is_sm120, requested=requested): + exec_config = SimpleNamespace( + kernel=SimpleNamespace(fp8_gemm_runner_backend=requested) + ) + with ( + override_platform(is_sm110=is_sm110, is_sm120=is_sm120), + patch.object(fp8_utils, "get_exec", return_value=exec_config), + patch.object(fp8_utils, "FP8_GEMM_RUNNER_BACKEND", None), + ): + fp8_utils.initialize_fp8_gemm_config() + self.assertEqual(fp8_utils.get_fp8_gemm_runner_backend(), expected) + + def test_modelopt_fp4_auto_routes_sm110_to_flashinfer_cutlass_only(self): + def view(quantization="modelopt_fp4", backend="auto"): + return ResolvedView( + SimpleNamespace( + quantization=quantization, + moe_runner_backend=backend, + moe_a2a_backend="none", + ) + ) + + for is_sm110, is_sm120, quantization, backend, expected in ( + (True, False, "modelopt_fp4", "auto", "flashinfer_cutlass"), + (False, True, "modelopt_fp4", "auto", "flashinfer_cutlass"), + (False, False, "modelopt_fp4", "auto", None), + (True, False, "modelopt_fp4", "triton", None), + (True, False, "modelopt_fp8", "auto", None), + ): + with self.subTest( + sm110=is_sm110, + sm120=is_sm120, + quantization=quantization, + backend=backend, + ): + with override_platform(is_sm110=is_sm110, is_sm120=is_sm120): + result = _moe_runner_backend_quant_constraints( + view(quantization, backend) + ) + self.assertEqual( + result, + {"moe_runner_backend": expected} if expected else {}, + ) + + +if __name__ == "__main__": + unittest.main() From 60eec0b7654563670e7b64e5eccbab3f3ecb8b09 Mon Sep 17 00:00:00 2001 From: jbernloehr Date: Sat, 3 Oct 2026 21:22:57 +0200 Subject: [PATCH 2/2] test: remove mocked SM110 backend routing coverage --- .../unit/test_sm110_backend_routing.py | 95 ------------------- 1 file changed, 95 deletions(-) delete mode 100644 test/registered/unit/test_sm110_backend_routing.py diff --git a/test/registered/unit/test_sm110_backend_routing.py b/test/registered/unit/test_sm110_backend_routing.py deleted file mode 100644 index c083a8e8acaf..000000000000 --- a/test/registered/unit/test_sm110_backend_routing.py +++ /dev/null @@ -1,95 +0,0 @@ -"""CPU regression tests for Thor's SM110 automatic backend selection.""" - -import unittest -from types import SimpleNamespace -from unittest.mock import patch - -from sglang.srt.arg_groups.overrides import ( - ResolvedView, - _moe_runner_backend_quant_constraints, -) -from sglang.srt.layers.quantization import fp8_utils -from sglang.srt.layers.quantization.fp8_utils import Fp8GemmRunnerBackend -from sglang.srt.runtime_context import get_platform, override_platform -from sglang.srt.utils import common -from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import CustomTestCase - -register_cpu_ci(est_time=5, suite="base-a-test-cpu") - - -class TestSm110BackendRouting(CustomTestCase): - def test_sm110_probe_requires_major_11_and_cuda_12_8(self): - try: - for device_sm, cuda_version, expected in ( - (100, "13.0", False), - (110, "12.7", False), - (110, "12.8", True), - (110, "13.0", True), - (120, "13.0", False), - ): - with self.subTest(device_sm=device_sm, cuda=cuda_version): - with ( - patch.object(common, "is_cuda", return_value=True), - patch.object(common, "get_device_sm", return_value=device_sm), - patch.object(common.torch.version, "cuda", cuda_version), - ): - common.is_sm110_supported.cache_clear() - self.assertEqual(get_platform().is_sm110, expected) - finally: - common.is_sm110_supported.cache_clear() - - def test_fp8_auto_routes_sm110_to_triton_only(self): - for is_sm110, is_sm120, requested, expected in ( - (True, False, "auto", Fp8GemmRunnerBackend.TRITON), - (False, True, "auto", Fp8GemmRunnerBackend.CUTLASS), - (False, False, "auto", Fp8GemmRunnerBackend.AUTO), - (True, False, "deep_gemm", Fp8GemmRunnerBackend.DEEP_GEMM), - ): - with self.subTest(sm110=is_sm110, sm120=is_sm120, requested=requested): - exec_config = SimpleNamespace( - kernel=SimpleNamespace(fp8_gemm_runner_backend=requested) - ) - with ( - override_platform(is_sm110=is_sm110, is_sm120=is_sm120), - patch.object(fp8_utils, "get_exec", return_value=exec_config), - patch.object(fp8_utils, "FP8_GEMM_RUNNER_BACKEND", None), - ): - fp8_utils.initialize_fp8_gemm_config() - self.assertEqual(fp8_utils.get_fp8_gemm_runner_backend(), expected) - - def test_modelopt_fp4_auto_routes_sm110_to_flashinfer_cutlass_only(self): - def view(quantization="modelopt_fp4", backend="auto"): - return ResolvedView( - SimpleNamespace( - quantization=quantization, - moe_runner_backend=backend, - moe_a2a_backend="none", - ) - ) - - for is_sm110, is_sm120, quantization, backend, expected in ( - (True, False, "modelopt_fp4", "auto", "flashinfer_cutlass"), - (False, True, "modelopt_fp4", "auto", "flashinfer_cutlass"), - (False, False, "modelopt_fp4", "auto", None), - (True, False, "modelopt_fp4", "triton", None), - (True, False, "modelopt_fp8", "auto", None), - ): - with self.subTest( - sm110=is_sm110, - sm120=is_sm120, - quantization=quantization, - backend=backend, - ): - with override_platform(is_sm110=is_sm110, is_sm120=is_sm120): - result = _moe_runner_backend_quant_constraints( - view(quantization, backend) - ) - self.assertEqual( - result, - {"moe_runner_backend": expected} if expected else {}, - ) - - -if __name__ == "__main__": - unittest.main()