diff --git a/docker/Dockerfile b/docker/Dockerfile index 17e1fae627f9..5f82fb862f2e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,7 +19,7 @@ ARG PIP_DEFAULT_INDEX ARG UBUNTU_MIRROR ARG GITHUB_ARTIFACTORY=github.com ARG INSTALL_FLASHINFER_JIT_CACHE=0 -ARG FLASHINFER_VERSION=0.6.14 +ARG FLASHINFER_VERSION=0.6.15 ARG MOONCAKE_VERSION=0.3.11.post1 ARG MSCCLPP_VERSION=sglang-v0.9.1 #if need other arg please add in MOONCAKE_COMPILE_ARG diff --git a/python/pyproject.toml b/python/pyproject.toml index e0f52fc332ab..192b2007865e 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "einops", "fastapi", "flash-attn-4==4.0.0b15", - "flashinfer_python[cu13]==0.6.14", # keep it aligned with jit-cache version in Dockerfile + "flashinfer_python[cu13]==0.6.15", # keep it aligned with jit-cache version in Dockerfile "gguf", "humming-kernels[cu13]==0.1.10", "interegular", diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py index 7fd810c4708b..9e8b2e581177 100644 --- a/python/sglang/srt/entrypoints/engine.py +++ b/python/sglang/srt/entrypoints/engine.py @@ -1304,7 +1304,7 @@ def _set_envs_and_config(server_args: ServerArgs): if server_args.attention_backend == "flashinfer": assert_pkg_version( "flashinfer_python", - "0.6.14", + "0.6.15", "Please uninstall the old version and " "reinstall the latest version by following the instructions " "at https://docs.flashinfer.ai/installation.html.", diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py index daa6ba2f903f..02366f886df5 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py @@ -222,6 +222,17 @@ def _resolve_w2_alpha_from_scalar_fc2_input_scale( return w1_alpha, fc2_input_scale, w2_alpha, used_input_scale +def _cutedsl_wrapper_activation_type(activation: str, activation_type_cls: Any) -> Any: + if activation == "silu": + return activation_type_cls.Swiglu + if activation == "relu2": + return activation_type_cls.Relu2 + raise ValueError( + f"CuteDSL MoE wrapper supports 'silu' (gated) or 'relu2' (non-gated) " + f"activation, got {activation!r}." + ) + + def ensure_cutedsl_wrapper(layer: torch.nn.Module) -> None: """Lazily create CuteDslMoEWrapper and resolve scales on first forward. @@ -237,7 +248,7 @@ def ensure_cutedsl_wrapper(layer: torch.nn.Module) -> None: return try: - from flashinfer import CuteDslMoEWrapper + from flashinfer import ActivationType, CuteDslMoEWrapper except ImportError as e: raise ImportError( "flashinfer_cutedsl backend requires FlashInfer with CuteDSL support. " @@ -284,7 +295,9 @@ def ensure_cutedsl_wrapper(layer: torch.nn.Module) -> None: local_expert_offset=layer.moe_ep_rank * layer.num_local_experts, output_dtype=layer.moe_runner_config.params_dtype, device=str(layer.w13_weight.device), - activation=layer.moe_runner_config.activation, + activation_type=_cutedsl_wrapper_activation_type( + layer.moe_runner_config.activation, ActivationType + ), ) w1_alpha, fc2_input_scale, w2_alpha, used_input_scale = ( diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index a086013beed6..7428ce1833c7 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -459,14 +459,6 @@ def __init__( "quark", ): correction_bias_dtype = torch.bfloat16 - # NOTE(kpham-sgl): flashinfer trtllm routing requires a bf16 - # routing_bias; an fp32 bias yields NaN routing on exact ties. - # Mirror the fp8 path's cast. - if ( - quant_config.get_name() == "modelopt_fp4" - and get_moe_runner_backend().is_flashinfer_trtllm() - ): - correction_bias_dtype = torch.bfloat16 self.e_score_correction_bias = nn.Parameter( torch.empty((config.n_routed_experts), dtype=correction_bias_dtype) ) diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 578d55007eb3..608492653b69 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -1907,7 +1907,7 @@ def check_pkg_version_at_least(pkg: str, min_version: str) -> bool: Args: pkg: Package name (distribution name, e.g., "flashinfer-python") - min_version: Minimum version required (e.g., "0.6.14") + min_version: Minimum version required (e.g., "0.6.15") Returns: True if package is installed and version >= min_version, False otherwise diff --git a/test/registered/models_e2e/test_dsa_glm52_nvfp4_tp_mtp.py b/test/registered/models_e2e/test_dsa_glm52_nvfp4_tp_mtp.py index 210112a5aa29..8f1c3d84d93c 100644 --- a/test/registered/models_e2e/test_dsa_glm52_nvfp4_tp_mtp.py +++ b/test/registered/models_e2e/test_dsa_glm52_nvfp4_tp_mtp.py @@ -21,7 +21,7 @@ class TestGLM52NVFP4TPMTP( model = "nvidia/GLM-5.2-NVFP4" tp_size = 4 mem_fraction_static = 0.8 - bs_1_speed_thres = 250 + bs_1_speed_thres = 280 extra_server_args = [ "--moe-runner-backend", "flashinfer_trtllm",