diff --git a/vllm/model_executor/layers/fused_moe/activation.py b/vllm/model_executor/layers/fused_moe/activation.py index 46a9bd1b4c58..bcee30e6e5c7 100644 --- a/vllm/model_executor/layers/fused_moe/activation.py +++ b/vllm/model_executor/layers/fused_moe/activation.py @@ -272,7 +272,7 @@ def _apply_moe_activation_masked( "SWIGLUOAI_UNINTERLEAVE requires clamp_limit" ) elif activation == MoEActivation.SWIGLUSTEP: - masked_clamp_limit = 7.0 + masked_clamp_limit = 7.0 if config.clamp_limit is None else config.clamp_limit torch.ops._C.masked_moe_activation( output, @@ -390,7 +390,8 @@ def apply_moe_activation( elif activation == MoEActivation.SWIGLUSTEP: from vllm.model_executor.layers.activation import swiglustep_and_mul_triton - swiglustep_and_mul_triton(output, input) + limit = 7.0 if config.clamp_limit is None else config.clamp_limit + swiglustep_and_mul_triton(output, input, limit=limit) # Activations without gated multiplication elif activation == MoEActivation.SILU_NO_MUL: diff --git a/vllm/model_executor/models/bailing_moe_v3.py b/vllm/model_executor/models/bailing_moe_v3.py index cbb1e8bc8f3f..1e55bc8b7e57 100644 --- a/vllm/model_executor/models/bailing_moe_v3.py +++ b/vllm/model_executor/models/bailing_moe_v3.py @@ -1203,7 +1203,9 @@ def __init__( intermediate_size=config.moe_intermediate_size, renormalize=True, quant_config=quant_config, - activation="silu", + activation=( + "swiglustep" if self.expert_swiglu_limit is not None else "silu" + ), swiglu_limit=self.expert_swiglu_limit, prefix=f"{prefix}.experts", scoring_func="sigmoid",