fix(serving): pin cuBLAS/cuDNN to deterministic modes for batch-shape invariance

python/sglang/srt/arg_groups/attention_hook.py

@@ -596,6 +596,39 @@ def handle_deterministic_inference(server_args: Any):
         envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.set(True)
 
     if cfg.enable_deterministic_inference:
+        # --- cuBLAS / cuDNN / PyTorch deterministic knobs ---
+        # These settings must be applied before any cuBLAS/cuDNN lazy
+        # initialisation so the GPU libraries pick their deterministic code
+        # paths from the first call.  The cuBLAS workspace config pins the
+        # GEMM algorithm selection across different M/N/K shapes, which is
+        # what caused the BF16 batch-shape drift that crossed rounding
+        # boundaries (see issue #43956).  Setting them again after cuBLAS
+        # has already picked its workspace is a no-op, so the env vars go
+        # first and torch.use_deterministic_algorithms is called below as a
+        # soft fallback.
+        os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8")
+        os.environ.setdefault("CUDNN_DETERMINISTIC", "1")
+        os.environ.setdefault("CUDNN_BENCHMARK", "0")
+        try:
+            import torch
+
+            torch.use_deterministic_algorithms(True, warn_only=True)
+            if hasattr(torch.backends, "cudnn"):
+                torch.backends.cudnn.deterministic = True
+                torch.backends.cudnn.benchmark = False
+            if hasattr(
+                torch.backends.cuda.matmul, "allow_bf16_reduced_precision_reduction"
+            ):
+                torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = (
+                    False
+                )
+            if hasattr(torch.backends.cuda, "enable_flash_sdp"):
+                torch.backends.cuda.enable_flash_sdp(False)
+            if hasattr(torch.backends.cuda, "enable_mem_efficient_sdp"):
+                torch.backends.cuda.enable_mem_efficient_sdp(False)
+        except Exception as _e:  # pragma: no cover - torch may not be present yet
+            logger.debug("Could not apply PyTorch deterministic settings: %s", _e)
+
         if cfg.enable_aiter_allreduce_fusion:
             logger.warning(
                 "Disable --enable-aiter-allreduce-fusion because deterministic inference is enabled."

python/sglang/srt/arg_groups/serving_hook.py

@@ -410,6 +410,14 @@ def handle_environment_variables(server_args: Any):
     )
     if cfg.enable_deterministic_inference:
         envs.SGLANG_FLASHINFER_MOE_FUSED_FINALIZE.set("0")
+        # Double-guess cuBLAS / cuDNN determinism here too, for code paths
+        # that might initialise cuBLAS between handle_deterministic_inference
+        # (pipeline step 228) and handle_environment_variables (step 318) —
+        # e.g. model loading or backend auto-detection.  Using setdefault
+        # means handle_deterministic_inference still wins when both run.
+        os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8")
+        os.environ.setdefault("CUDNN_DETERMINISTIC", "1")
+        os.environ.setdefault("CUDNN_BENCHMARK", "0")
     if cfg.debug_cuda_graph:
         if not (get_platform().is_cuda or get_platform().is_hip):
             logger.warning(

python/sglang/srt/layers/quantization/fp4_utils.py

@@ -159,6 +159,21 @@ def initialize_fp4_gemm_config() -> None:
         else:
             backend = "flashinfer_cutlass"
 
+    # FP4 Marlin is weight-only (W4A16) and does not support W4A4 GEMM.
+    # FlashInfer's mm_fp4 rejects backend='marlin' on SM100+, so force a
+    # fallback whenever the user (or auto-selection) picks marlin on
+    # Blackwell GPUs. See issue #43988.
+    if backend == "marlin" and get_platform().is_blackwell:
+        fallback = (
+            "flashinfer_cutedsl" if get_platform().is_sm100 else "flashinfer_cutlass"
+        )
+        logger.warning(
+            "FP4 Marlin does not support SM100+ (Blackwell); falling back to %s. "
+            "FlashInfer's mm_fp4 has no 'marlin' backend for capability 100.",
+            fallback,
+        )
+        backend = fallback
+
     FP4_GEMM_RUNNER_BACKEND = Fp4GemmRunnerBackend(backend)