diff --git a/benchmarks/bench_cute_dsl_moe_distributed.py b/benchmarks/bench_cute_dsl_moe_distributed.py index b3e02a0e51d..b86e96f8f13 100644 --- a/benchmarks/bench_cute_dsl_moe_distributed.py +++ b/benchmarks/bench_cute_dsl_moe_distributed.py @@ -116,6 +116,8 @@ def _profile_worker_arguments(args, num_tokens): ] if args.use_per_token_activation: arguments.append("--use-per-token-activation") + if args.apply_router_weight_on_input: + arguments.append("--apply-router-weight-on-input") if not args.use_fused_finalize: arguments.append("--no-fused-finalize") if not args.enable_pdl: @@ -451,6 +453,7 @@ def _distributed_moe_config( enable_pdl=args.enable_pdl, tune_max_num_tokens=tune_max_num_tokens, use_fused_finalize=args.use_fused_finalize, + apply_router_weight_on_input=args.apply_router_weight_on_input, ), ) @@ -1267,6 +1270,10 @@ def _run_distributed_benchmark(args, token_counts): ) if rank == 0: print("\nDeepSeek-V3 distributed CuTe DSL MoE benchmark") + print( + "CuTe DSL router weight placement: " + f"{'input' if args.apply_router_weight_on_input else 'output'}" + ) selected_mode = None selected_variant_name = None @@ -1337,6 +1344,13 @@ def main(): dest="use_fused_finalize", help="Use deterministic two-stage finalize.", ) + parser.add_argument( + "--apply-router-weight-on-input", + action="store_true", + help=( + "Apply routing weights to the CuTe DSL FC2 input instead of after GEMM2." + ), + ) parser.add_argument( "--no-pdl", action="store_false", diff --git a/benchmarks/bench_moe_deepseek.py b/benchmarks/bench_moe_deepseek.py index b4ce4b0ee83..b8bd8264a25 100644 --- a/benchmarks/bench_moe_deepseek.py +++ b/benchmarks/bench_moe_deepseek.py @@ -42,6 +42,9 @@ python bench_moe_deepseek.py --functional-api # atomic fused (default) python bench_moe_deepseek.py --functional-api --no-fused-finalize # deterministic + # Apply routing weights to the FC2 input + python bench_moe_deepseek.py --apply-router-weight-on-input + Metrics: - ms: Latency in milliseconds - TFLOPS: Computational throughput @@ -279,6 +282,7 @@ def bench_cute_dsl( use_fused_finalize=True, profile_cuda=False, profile_iters=10, + apply_router_weight_on_input=False, ): """Benchmark CuteDSL MoE. @@ -300,6 +304,8 @@ def bench_cute_dsl( profile_cuda: Capture steady-state CUDA graph replays between cudaProfilerStart/Stop instead of benchmarking. profile_iters: Number of cold-L2 graph replays to capture. + apply_router_weight_on_input: Apply routing weights after the expert + activation, on the FC2 input, instead of after GEMM2. """ import contextlib @@ -417,6 +423,7 @@ def prepare_activation(x, x_sf): local_expert_offset=local_expert_offset, use_fused_finalize=use_fused_finalize, quant_mode=quant_mode, + apply_router_weight_on_input=apply_router_weight_on_input, ) def run(x, x_sf, router_logits, routing_bias, topk_values, topk_indices): @@ -480,6 +487,7 @@ def run(x, x_sf, router_logits, routing_bias, topk_values, topk_indices): quant_mode=quant_mode, per_token_scale=per_token_scale, use_fused_finalize=use_fused_finalize, + apply_router_weight_on_input=apply_router_weight_on_input, ) # Pass input tensors via input_kwargs for cold L2 cache rotation @@ -856,6 +864,7 @@ def run_benchmark( profile_cuda=False, profile_iters=10, profile_backend=None, + apply_router_weight_on_input=False, ): """ Unified benchmark for DeepSeek-V3 MoE backends. @@ -894,6 +903,8 @@ def run_benchmark( profile_cuda: Capture one backend for an external CUDA profiler. profile_iters: Number of cold-L2 graph replays to capture. profile_backend: Backend to run when profile_cuda is enabled. + apply_router_weight_on_input: Apply routing weights after the expert + activation, on the FC2 input, instead of after GEMM2. Returns: List of BenchResult objects @@ -932,6 +943,7 @@ def run_benchmark( profile_cuda=profile_cuda, profile_iters=profile_iters, profile_backend=profile_backend, + apply_router_weight_on_input=apply_router_weight_on_input, ) results.extend(row) rows_and_histograms.append((row, histogram_record)) @@ -951,6 +963,7 @@ def run_benchmark( use_per_token_activation=use_per_token_activation, include_activation_quant=include_activation_quant, use_fused_finalize=use_fused_finalize, + apply_router_weight_on_input=apply_router_weight_on_input, ) for row, histogram_record in rows_and_histograms: _print_row(row, histogram_record) @@ -976,6 +989,7 @@ def _benchmark_single( profile_cuda=False, profile_iters=10, profile_backend=None, + apply_router_weight_on_input=False, ): """Benchmark all backends for a single token count. @@ -1009,6 +1023,7 @@ def _benchmark_single( use_fused_finalize=use_fused_finalize, profile_cuda=profile_cuda, profile_iters=profile_iters, + apply_router_weight_on_input=apply_router_weight_on_input, ) if run_cute_dsl_w4a16: lat["CuteDSL W4A16"] = bench_cute_dsl( @@ -1027,6 +1042,7 @@ def _benchmark_single( use_fused_finalize=use_fused_finalize, profile_cuda=profile_cuda, profile_iters=profile_iters, + apply_router_weight_on_input=apply_router_weight_on_input, ) if run_cutlass and not use_per_token_activation: lat["CUTLASS"] = bench_cutlass( @@ -1082,6 +1098,7 @@ def _print_header( use_per_token_activation=False, include_activation_quant=False, use_fused_finalize=True, + apply_router_weight_on_input=False, ): """Print benchmark header.""" table_width = 120 if use_per_token_activation else 159 @@ -1124,6 +1141,10 @@ def _print_header( "CuteDSL finalize: " f"{'atomic fused' if use_fused_finalize else 'deterministic two-stage'}" ) + print( + "CuteDSL router weight placement: " + f"{'input' if apply_router_weight_on_input else 'output'}" + ) if use_per_token_activation: print("CUTLASS omitted: it does not consume the per-token activation scale.") print("-" * table_width) @@ -1357,6 +1378,11 @@ def main(): dest="use_fused_finalize", help="Use deterministic two-stage CuTe DSL finalize instead of atomic fused finalize.", ) + parser.add_argument( + "--apply-router-weight-on-input", + action="store_true", + help="Apply routing weights to the CuTe DSL FC2 input instead of after GEMM2.", + ) parser.add_argument( "--profile-cuda", action="store_true", @@ -1418,6 +1444,10 @@ def main(): "CuteDSL finalize: " f"{'atomic fused' if args.use_fused_finalize else 'deterministic two-stage'}" ) + print( + "CuteDSL router weight placement: " + f"{'input' if args.apply_router_weight_on_input else 'output'}" + ) run_benchmark( token_counts=tokens, @@ -1437,6 +1467,7 @@ def main(): profile_cuda=args.profile_cuda, profile_iters=args.profile_iters, profile_backend=args.profile_backend, + apply_router_weight_on_input=args.apply_router_weight_on_input, ) return 0 diff --git a/csrc/moe_utils_binding.cu b/csrc/moe_utils_binding.cu index 66fef4b938a..545668c28da 100644 --- a/csrc/moe_utils_binding.cu +++ b/csrc/moe_utils_binding.cu @@ -126,28 +126,29 @@ void moe_unpermute_fp16_float_scale(int64_t permuted_input_ptr, int64_t output_p int64_t expanded_idx_to_permuted_idx_ptr, int64_t topk_scales_ptr, int32_t num_tokens, int32_t hidden_size, int32_t top_k, bool input_is_expanded, - bool enable_pdl, int64_t cuda_stream_ptr) { + bool apply_topk_scales, bool enable_pdl, + int64_t cuda_stream_ptr) { cudaStream_t stream = cuda_stream_ptr != 0 ? reinterpret_cast(cuda_stream_ptr) : get_current_stream(); - moeUnpermute(reinterpret_cast(permuted_input_ptr), - reinterpret_cast(output_ptr), - reinterpret_cast(expanded_idx_to_permuted_idx_ptr), - reinterpret_cast(topk_scales_ptr), num_tokens, - hidden_size, top_k, input_is_expanded, enable_pdl, stream); + moeUnpermute( + reinterpret_cast(permuted_input_ptr), reinterpret_cast(output_ptr), + reinterpret_cast(expanded_idx_to_permuted_idx_ptr), + reinterpret_cast(topk_scales_ptr), num_tokens, hidden_size, top_k, + input_is_expanded, apply_topk_scales, enable_pdl, stream); } void moe_unpermute_fp16_half_scale(int64_t permuted_input_ptr, int64_t output_ptr, int64_t expanded_idx_to_permuted_idx_ptr, int64_t topk_scales_ptr, int32_t num_tokens, int32_t hidden_size, - int32_t top_k, bool input_is_expanded, bool enable_pdl, - int64_t cuda_stream_ptr) { + int32_t top_k, bool input_is_expanded, bool apply_topk_scales, + bool enable_pdl, int64_t cuda_stream_ptr) { cudaStream_t stream = cuda_stream_ptr != 0 ? reinterpret_cast(cuda_stream_ptr) : get_current_stream(); moeUnpermute(reinterpret_cast(permuted_input_ptr), reinterpret_cast(output_ptr), reinterpret_cast(expanded_idx_to_permuted_idx_ptr), reinterpret_cast(topk_scales_ptr), num_tokens, hidden_size, - top_k, input_is_expanded, enable_pdl, stream); + top_k, input_is_expanded, apply_topk_scales, enable_pdl, stream); } #ifdef ENABLE_BF16 @@ -155,7 +156,8 @@ void moe_unpermute_bf16_float_scale(int64_t permuted_input_ptr, int64_t output_p int64_t expanded_idx_to_permuted_idx_ptr, int64_t topk_scales_ptr, int32_t num_tokens, int32_t hidden_size, int32_t top_k, bool input_is_expanded, - bool enable_pdl, int64_t cuda_stream_ptr) { + bool apply_topk_scales, bool enable_pdl, + int64_t cuda_stream_ptr) { cudaStream_t stream = cuda_stream_ptr != 0 ? reinterpret_cast(cuda_stream_ptr) : get_current_stream(); moeUnpermute<__nv_bfloat16, float>( @@ -163,14 +165,14 @@ void moe_unpermute_bf16_float_scale(int64_t permuted_input_ptr, int64_t output_p reinterpret_cast<__nv_bfloat16*>(output_ptr), reinterpret_cast(expanded_idx_to_permuted_idx_ptr), reinterpret_cast(topk_scales_ptr), num_tokens, hidden_size, top_k, - input_is_expanded, enable_pdl, stream); + input_is_expanded, apply_topk_scales, enable_pdl, stream); } void moe_unpermute_bf16_bf16_scale(int64_t permuted_input_ptr, int64_t output_ptr, int64_t expanded_idx_to_permuted_idx_ptr, int64_t topk_scales_ptr, int32_t num_tokens, int32_t hidden_size, - int32_t top_k, bool input_is_expanded, bool enable_pdl, - int64_t cuda_stream_ptr) { + int32_t top_k, bool input_is_expanded, bool apply_topk_scales, + bool enable_pdl, int64_t cuda_stream_ptr) { cudaStream_t stream = cuda_stream_ptr != 0 ? reinterpret_cast(cuda_stream_ptr) : get_current_stream(); moeUnpermute<__nv_bfloat16, __nv_bfloat16>( @@ -178,7 +180,7 @@ void moe_unpermute_bf16_bf16_scale(int64_t permuted_input_ptr, int64_t output_pt reinterpret_cast<__nv_bfloat16*>(output_ptr), reinterpret_cast(expanded_idx_to_permuted_idx_ptr), reinterpret_cast<__nv_bfloat16 const*>(topk_scales_ptr), num_tokens, hidden_size, top_k, - input_is_expanded, enable_pdl, stream); + input_is_expanded, apply_topk_scales, enable_pdl, stream); } #endif diff --git a/csrc/nv_internal/tensorrt_llm/kernels/cuteDslKernels/moeUtils.cu b/csrc/nv_internal/tensorrt_llm/kernels/cuteDslKernels/moeUtils.cu index e77684e374e..1a3fe9e0373 100644 --- a/csrc/nv_internal/tensorrt_llm/kernels/cuteDslKernels/moeUtils.cu +++ b/csrc/nv_internal/tensorrt_llm/kernels/cuteDslKernels/moeUtils.cu @@ -156,7 +156,8 @@ INSTANTIATE_MOE_PERMUTE(__nv_fp4_e2m1, uint8_t); #endif #undef INSTANTIATE_MOE_PERMUTE -template +template __global__ void moeUnpermuteKernel(InputType const* permuted_input, InputType* output, int32_t const* expanded_idx_to_permuted_idx, TopKScaleType const* topk_scales, int32_t const hidden_size, @@ -190,11 +191,18 @@ __global__ void moeUnpermuteKernel(InputType const* permuted_input, InputType* o auto const* src_ptr = reinterpret_cast(permuted_input) + input_idx * kCopyPerToken; *reinterpret_cast(rmem) = src_ptr[i]; - TopKScaleType const scale = topk_scales[expanded_idx]; + if constexpr (ApplyTopKScales) { + TopKScaleType const scale = topk_scales[expanded_idx]; #pragma unroll - for (int32_t j = 0; j < kElemPerCopy; j++) { - rmemAccum[j] += static_cast(rmem[j]) * static_cast(scale); + for (int32_t j = 0; j < kElemPerCopy; j++) { + rmemAccum[j] += static_cast(rmem[j]) * static_cast(scale); + } + } else { +#pragma unroll + for (int32_t j = 0; j < kElemPerCopy; j++) { + rmemAccum[j] += static_cast(rmem[j]); + } } } #pragma unroll @@ -213,7 +221,8 @@ template void moeUnpermute(InputType const* permuted_input, InputType* output, int32_t const* expanded_idx_to_permuted_idx, TopKScaleType const* topk_scales, int32_t const num_tokens, int32_t const hidden_size, int32_t const top_k, - bool input_is_expanded, bool enable_pdl, cudaStream_t stream) { + bool input_is_expanded, bool apply_topk_scales, bool enable_pdl, + cudaStream_t stream) { int32_t constexpr kThreadsPerBlock = 256; int32_t constexpr kElemPerCopy = elemPerCopy(); TLLM_CHECK_WITH_INFO(hidden_size % kElemPerCopy == 0, "hidden_size must be divisible by %d.", @@ -222,8 +231,6 @@ void moeUnpermute(InputType const* permuted_input, InputType* output, int32_t const blocks = num_tokens; int32_t const threads = kThreadsPerBlock; - auto kernel = &moeUnpermuteKernel; - cudaLaunchConfig_t config; config.gridDim = blocks; config.blockDim = threads; @@ -234,8 +241,15 @@ void moeUnpermute(InputType const* permuted_input, InputType* output, attrs[0].val.programmaticStreamSerializationAllowed = enable_pdl; config.numAttrs = 1; config.attrs = attrs; - cudaLaunchKernelEx(&config, kernel, permuted_input, output, expanded_idx_to_permuted_idx, - topk_scales, hidden_size, top_k, input_is_expanded); + if (apply_topk_scales) { + auto kernel = &moeUnpermuteKernel; + cudaLaunchKernelEx(&config, kernel, permuted_input, output, expanded_idx_to_permuted_idx, + topk_scales, hidden_size, top_k, input_is_expanded); + } else { + auto kernel = &moeUnpermuteKernel; + cudaLaunchKernelEx(&config, kernel, permuted_input, output, expanded_idx_to_permuted_idx, + topk_scales, hidden_size, top_k, input_is_expanded); + } } #define INSTANTIATE_MOE_UNPERMUTE(InputType, TopKScaleType) \ @@ -243,7 +257,7 @@ void moeUnpermute(InputType const* permuted_input, InputType* output, InputType const* permuted_input, InputType* output, \ int32_t const* expanded_idx_to_permuted_idx, TopKScaleType const* topk_scales, \ int32_t const num_tokens, int32_t const hidden_size, int32_t const top_k, \ - bool input_is_expanded, bool enable_pdl, cudaStream_t stream) + bool input_is_expanded, bool apply_topk_scales, bool enable_pdl, cudaStream_t stream) INSTANTIATE_MOE_UNPERMUTE(half, float); INSTANTIATE_MOE_UNPERMUTE(half, half); diff --git a/csrc/nv_internal/tensorrt_llm/kernels/cuteDslKernels/moeUtils.h b/csrc/nv_internal/tensorrt_llm/kernels/cuteDslKernels/moeUtils.h index 1b841a73f57..9d618f71a87 100644 --- a/csrc/nv_internal/tensorrt_llm/kernels/cuteDslKernels/moeUtils.h +++ b/csrc/nv_internal/tensorrt_llm/kernels/cuteDslKernels/moeUtils.h @@ -47,7 +47,8 @@ template void moeUnpermute(InputType const* permuted_input, InputType* output, int32_t const* expanded_idx_to_permuted_idx, TopKScaleType const* topk_scales, int32_t const num_tokens, int32_t const hidden_size, int32_t const top_k, - bool input_is_expanded, bool enable_pdl, cudaStream_t stream); + bool input_is_expanded, bool apply_topk_scales, bool enable_pdl, + cudaStream_t stream); template void moeOutputMemset(InputType* input, int32_t const* tile_idx_to_mn_limit, diff --git a/flashinfer/fused_moe/api.py b/flashinfer/fused_moe/api.py index d61947c9c03..99c1142eefe 100644 --- a/flashinfer/fused_moe/api.py +++ b/flashinfer/fused_moe/api.py @@ -204,12 +204,16 @@ class ExecutionConfig: Token budget hint for autotuner / CUDA graph capture. use_fused_finalize : bool Whether supported backends reduce routed outputs in the GEMM2 epilogue. + apply_router_weight_on_input : bool + Whether supported backends apply routing weights after the expert + activation, on the FC2 input, instead of after GEMM2. """ do_finalize: bool = True enable_pdl: Optional[bool] = None tune_max_num_tokens: int = 8192 use_fused_finalize: bool = True + apply_router_weight_on_input: bool = False def __repr__(self) -> str: parts = [] @@ -221,6 +225,10 @@ def __repr__(self) -> str: parts.append(f"tune_max_num_tokens={self.tune_max_num_tokens!r}") if not self.use_fused_finalize: parts.append(f"use_fused_finalize={self.use_fused_finalize!r}") + if self.apply_router_weight_on_input: + parts.append( + f"apply_router_weight_on_input={self.apply_router_weight_on_input!r}" + ) return f"ExecutionConfig({', '.join(parts)})" diff --git a/flashinfer/fused_moe/cute_dsl/blackwell/blockscaled_contiguous_gather_grouped_gemm_act_fusion.py b/flashinfer/fused_moe/cute_dsl/blackwell/blockscaled_contiguous_gather_grouped_gemm_act_fusion.py index 1a7701eb5d6..6c074455a1c 100644 --- a/flashinfer/fused_moe/cute_dsl/blackwell/blockscaled_contiguous_gather_grouped_gemm_act_fusion.py +++ b/flashinfer/fused_moe/cute_dsl/blackwell/blockscaled_contiguous_gather_grouped_gemm_act_fusion.py @@ -424,6 +424,7 @@ def __init__( situ_linear_beta: Optional[float] = None, gated: bool = True, use_a_per_token_scale: bool = False, + apply_router_weight_on_input: bool = False, ): """Initializes the configuration for a Blackwell blockscaled dense GEMM kernel with gather operation and FC1 activation fusion. @@ -482,11 +483,15 @@ def __init__( :param gated: Whether GEMM1 output is split into up/gate halves. If False, the epilogue computes non-gated ReLU^2. :type gated: bool + :param apply_router_weight_on_input: Whether to multiply the expert + activation output by each route's weight before the FC2-input store. + :type apply_router_weight_on_input: bool """ self.sf_vec_size = sf_vec_size self.enable_pdl = enable_pdl self.use_a_per_token_scale = use_a_per_token_scale + self.apply_router_weight_on_input = apply_router_weight_on_input self.topk = topk self.gated = gated self.out_n_factor = 2 if gated else 1 @@ -796,6 +801,7 @@ def __call__( num_non_exiting_tiles: cute.Tensor, alpha: cute.Tensor, a_per_token_scale: Optional[cute.Tensor], + token_final_scales: Optional[cute.Tensor], max_active_clusters: cutlass.Constexpr, stream: cuda.CUstream, epilogue_op: cutlass.Constexpr = lambda x: x, @@ -858,6 +864,9 @@ def __call__( shape (orig_m,). Indexed by the original token ID decoded from token_id_mapping_tensor. Only used when use_a_per_token_scale is true. :type a_per_token_scale: Optional[cute.Tensor] + :param token_final_scales: Optional routing weights, flattened from + shape (orig_m, topk). Used when apply_router_weight_on_input is true. + :type token_final_scales: Optional[cute.Tensor] :param max_active_clusters: Maximum number of active clusters :type max_active_clusters: cutlass.Constexpr :param stream: CUDA stream for asynchronous execution @@ -1133,6 +1142,7 @@ class SharedStorage2cta: num_non_exiting_tiles, alpha, a_per_token_scale, + token_final_scales, self.cluster_layout_vmnk, self.cluster_layout_sfb_vmnk, self.a_smem_layout_staged, @@ -1220,6 +1230,7 @@ def kernel( num_non_exiting_tiles: cute.Tensor, alpha: cute.Tensor, a_per_token_scale: Optional[cute.Tensor], + token_final_scales: Optional[cute.Tensor], cluster_layout_vmnk: cute.Layout, cluster_layout_sfb_vmnk: cute.Layout, a_smem_layout_staged: cute.ComposedLayout, @@ -2560,7 +2571,9 @@ def kernel( tile_info[1] = sInfo[(1, tile_info_consumer_state.index)] tile_info[2] = sInfo[(2, tile_info_consumer_state.index)] tile_info[3] = sInfo[(3, tile_info_consumer_state.index)] - if cutlass.const_expr(self.use_a_per_token_scale): + if cutlass.const_expr( + self.use_a_per_token_scale or self.apply_router_weight_on_input + ): tile_info[4] = sInfo[(4, tile_info_consumer_state.index)] is_valid_tile = tile_info[3] == 1 cute.arch.fence_proxy( @@ -2583,13 +2596,21 @@ def kernel( expert_idx = mma_tile_coord_mnl[2] alpha_val = alpha[expert_idx] - if cutlass.const_expr(self.use_a_per_token_scale): + router_weight = cutlass.Float32(1.0) + if cutlass.const_expr( + self.use_a_per_token_scale or self.apply_router_weight_on_input + ): tile_m_start = tile_info[0] * self.cta_tile_shape_mnk[0] permuted_row = tile_m_start + epi_tidx if permuted_row < tile_info[4]: expanded_idx = token_id_mapping_tensor[permuted_row] - token_idx = expanded_idx // self.topk - alpha_val = alpha_val * a_per_token_scale[token_idx] + if cutlass.const_expr(self.use_a_per_token_scale): + token_idx = expanded_idx // self.topk + alpha_val = alpha_val * a_per_token_scale[token_idx] + if cutlass.const_expr(self.apply_router_weight_on_input): + router_weight = cutlass.Float32( + token_final_scales[expanded_idx] + ) # # Slice to per mma tile index @@ -2976,6 +2997,22 @@ def kernel( * (up_clamped + swiglu_beta) ) + if cutlass.const_expr(self.apply_router_weight_on_input): + # Apply the route in FP32 after the expert activation + # and before the FC2-input store or quantization. + if cutlass.const_expr(self.vectorized_f32): + for i in cutlass.range_constexpr(0, cute.size(tCompute), 2): + ( + tCompute[i], + tCompute[i + 1], + ) = cute.arch.mul_packed_f32x2( + (tCompute[i], tCompute[i + 1]), + (router_weight, router_weight), + ) + else: + for i in cutlass.range_constexpr(cute.size(tCompute)): + tCompute[i] = tCompute[i] * router_weight + if cutlass.const_expr(self.generate_sfc): # # Quantization path for Float4E2M1FN output: @@ -3155,7 +3192,9 @@ def kernel( tile_info[1] = sInfo[(1, tile_info_consumer_state.index)] tile_info[2] = sInfo[(2, tile_info_consumer_state.index)] tile_info[3] = sInfo[(3, tile_info_consumer_state.index)] - if cutlass.const_expr(self.use_a_per_token_scale): + if cutlass.const_expr( + self.use_a_per_token_scale or self.apply_router_weight_on_input + ): tile_info[4] = sInfo[(4, tile_info_consumer_state.index)] is_valid_tile = tile_info[3] == 1 cute.arch.fence_proxy( @@ -3833,6 +3872,7 @@ def wrapper( num_non_exiting_tiles_ptr: cute.Pointer, global_sf_ptr: Optional[cute.Pointer], a_per_token_scale_ptr: Optional[cute.Pointer], + token_final_scales_ptr: Optional[cute.Pointer], orig_m: cutlass.Int64, m: cutlass.Int64, n: cutlass.Int64, @@ -3883,6 +3923,13 @@ def wrapper( if cutlass.const_expr(a_per_token_scale_ptr is not None) else None ) + token_final_scales = ( + cute.make_tensor( + token_final_scales_ptr, layout=cute.make_layout((orig_m * self.topk,)) + ) + if cutlass.const_expr(token_final_scales_ptr is not None) + else None + ) tile_idx_to_group_idx = cute.make_tensor( tile_idx_to_group_idx_ptr, layout=cute.make_layout((num_tiles,)) @@ -3916,6 +3963,7 @@ def wrapper( num_non_exiting_tiles, alpha, a_per_token_scale, + token_final_scales, max_active_clusters=max_active_clusters, stream=stream, epilogue_op=epilogue_op, diff --git a/flashinfer/fused_moe/cute_dsl/blackwell/blockscaled_contiguous_grouped_gemm_finalize_fusion.py b/flashinfer/fused_moe/cute_dsl/blackwell/blockscaled_contiguous_grouped_gemm_finalize_fusion.py index 229bd7a4105..bf5877f8b78 100644 --- a/flashinfer/fused_moe/cute_dsl/blackwell/blockscaled_contiguous_grouped_gemm_finalize_fusion.py +++ b/flashinfer/fused_moe/cute_dsl/blackwell/blockscaled_contiguous_grouped_gemm_finalize_fusion.py @@ -364,6 +364,7 @@ def __init__( enable_pdl: bool = True, use_a_per_token_scale: bool = False, use_fused_finalize: bool = True, + apply_router_weight_on_input: bool = False, ): """Initializes the configuration for a Blackwell blockscaled dense GEMM kernel. @@ -387,6 +388,7 @@ def __init__( self.enable_pdl = enable_pdl self.use_a_per_token_scale = use_a_per_token_scale self.use_fused_finalize = use_fused_finalize + self.apply_router_weight_on_input = apply_router_weight_on_input self.acc_dtype = cutlass.Float32 self.use_2cta_instrs = mma_tiler_mn[0] == 256 self.cluster_shape_mn = cluster_shape_mn @@ -1964,11 +1966,12 @@ def kernel( topk_idx = safe_idx % topK is_valid_row = cutlass.Int32(permuted_row < tile_info[4]) gather_tok = token_idx * is_valid_row - token_scale = token_final_scales[(gather_tok, topk_idx)] - output_idx = token_idx - if cutlass.const_expr(not self.use_fused_finalize): - token_scale = self.final_scale_dtype(1.0) - output_idx = safe_idx + token_scale = self.final_scale_dtype(1.0) + output_idx = safe_idx + if cutlass.const_expr(self.use_fused_finalize): + output_idx = token_idx + if cutlass.const_expr(not self.apply_router_weight_on_input): + token_scale = token_final_scales[(gather_tok, topk_idx)] if cutlass.const_expr(self.use_a_per_token_scale): token_scale = cutlass.Float32(token_scale) * cutlass.Float32( a_per_token_scale[permuted_row] diff --git a/flashinfer/fused_moe/cute_dsl/blackwell/moe_w4a16.py b/flashinfer/fused_moe/cute_dsl/blackwell/moe_w4a16.py index 0e09e09c417..d355e6e75e9 100644 --- a/flashinfer/fused_moe/cute_dsl/blackwell/moe_w4a16.py +++ b/flashinfer/fused_moe/cute_dsl/blackwell/moe_w4a16.py @@ -129,6 +129,7 @@ def _get_compiled_kernel( swiglu_beta: float, swiglu_limit: float, use_fused_finalize: bool, + apply_router_weight_on_input: bool, enable_pdl: bool, use_clc_scheduler: bool, mma_tiler_mnk: Tuple[int, int, int], @@ -147,6 +148,7 @@ def _get_compiled_kernel( swiglu_beta, swiglu_limit, use_fused_finalize, + apply_router_weight_on_input, enable_pdl, use_clc_scheduler, mma_tiler_m, @@ -171,6 +173,7 @@ def _get_compiled_kernel( swiglu_beta=swiglu_beta, swiglu_limit=swiglu_limit, use_fused_finalize=use_fused_finalize, + apply_router_weight_on_input=apply_router_weight_on_input, enable_pdl=enable_pdl, use_clc_scheduler=use_clc_scheduler, raster_along_m=raster_along_m, @@ -215,6 +218,7 @@ def _run_grouped_gemm( swiglu_beta: float, swiglu_limit: float, use_fused_finalize: bool, + apply_router_weight_on_input: bool, permuted_idx_to_expanded_idx: Optional[torch.Tensor], token_final_scales: Optional[torch.Tensor], enable_pdl: bool, @@ -307,7 +311,9 @@ def _run_grouped_gemm( if token_final_scales is not None else None ) - num_tokens = int(output.size(0)) if use_fused_finalize else 0 + num_tokens = ( + int(token_final_scales.size(0)) if token_final_scales is not None else 0 + ) top_k = int(token_final_scales.size(1)) if token_final_scales is not None else 0 compiled = _get_compiled_kernel( num_local_experts, @@ -333,6 +339,7 @@ def _run_grouped_gemm( swiglu_beta, swiglu_limit, use_fused_finalize, + apply_router_weight_on_input, enable_pdl, use_clc_scheduler, mma_tiler_mnk, @@ -381,6 +388,7 @@ def launch_w4a16_moe( swiglu_limit: float = DEFAULT_SWIGLU_LIMIT, tactic: Optional[W4A16MoeTactic] = None, workspace_cache: Optional[Dict[Tuple, _W4A16Workspace]] = None, + apply_router_weight_on_input: bool = False, ) -> torch.Tensor: """Run BF16 activations against online-decoded NVFP4 expert weights.""" top_k = int(token_selected_experts.size(1)) @@ -463,8 +471,13 @@ def launch_w4a16_moe( swiglu_beta=swiglu_beta, swiglu_limit=swiglu_limit, use_fused_finalize=False, - permuted_idx_to_expanded_idx=None, - token_final_scales=None, + apply_router_weight_on_input=apply_router_weight_on_input, + permuted_idx_to_expanded_idx=( + permuted_idx_to_expanded_idx if apply_router_weight_on_input else None + ), + token_final_scales=( + token_final_scales if apply_router_weight_on_input else None + ), enable_pdl=enable_pdl, tactic=gemm1_tactic, ) @@ -486,6 +499,7 @@ def launch_w4a16_moe( swiglu_beta=DEFAULT_SWIGLU_BETA, swiglu_limit=DEFAULT_SWIGLU_LIMIT, use_fused_finalize=use_fused_finalize, + apply_router_weight_on_input=apply_router_weight_on_input, permuted_idx_to_expanded_idx=( permuted_idx_to_expanded_idx if use_fused_finalize else None ), @@ -501,6 +515,7 @@ def launch_w4a16_moe( topk_scales=token_final_scales, num_tokens=int(x.size(0)), top_k=top_k, + apply_topk_scales=not apply_router_weight_on_input, enable_pdl=enable_pdl, ) return moe_output diff --git a/flashinfer/fused_moe/cute_dsl/blackwell/moe_w4a16_kernel.py b/flashinfer/fused_moe/cute_dsl/blackwell/moe_w4a16_kernel.py index 5b9b588df95..4a0da582585 100644 --- a/flashinfer/fused_moe/cute_dsl/blackwell/moe_w4a16_kernel.py +++ b/flashinfer/fused_moe/cute_dsl/blackwell/moe_w4a16_kernel.py @@ -85,6 +85,7 @@ def __init__( swiglu_beta: float, swiglu_limit: float, use_fused_finalize: bool, + apply_router_weight_on_input: bool, enable_pdl: bool, use_clc_scheduler: bool, raster_along_m: bool, @@ -123,6 +124,8 @@ def __init__( "W4A16 fused finalize is only implemented for the non-gated epilogue" ) self.use_fused_finalize = use_fused_finalize + self.apply_router_weight_on_input = apply_router_weight_on_input + self.load_router_weight = self.fuse_activation and apply_router_weight_on_input self.enable_pdl = enable_pdl self.cta_group = ( tcgen05.CtaGroup.TWO if self.use_2cta_instrs else tcgen05.CtaGroup.ONE @@ -232,6 +235,7 @@ def _setup_attributes(self): self.transform_a_source, self.smem_buffer_align_bytes, self.use_fused_finalize, + self.load_router_weight, self.use_clc_scheduler, ) @@ -592,7 +596,7 @@ def wrapper( alpha = cute.make_tensor(alpha_ptr, cute.make_layout((self.group_count,))) permuted_idx_to_expanded_idx = ( cute.make_tensor(permuted_idx_to_expanded_idx_ptr, cute.make_layout((n,))) - if cutlass.const_expr(self.use_fused_finalize) + if cutlass.const_expr(self.use_fused_finalize or self.load_router_weight) else None ) token_final_scales = ( @@ -600,7 +604,7 @@ def wrapper( token_final_scales_ptr, cute.make_ordered_layout((num_tokens, top_k), order=(1, 0)), ) - if cutlass.const_expr(self.use_fused_finalize) + if cutlass.const_expr(self.use_fused_finalize or self.load_router_weight) else None ) return self( @@ -1031,13 +1035,13 @@ def kernel( byte_alignment=self.smem_buffer_align_bytes, swizzle=a_smem_layout_transform.inner, ) - sFinalizeScale = ( + sRouteScale = ( smem.allocate_tensor( element_type=cutlass.Float32, layout=cute.make_layout((self.epi_tile_n,)), byte_alignment=16, ) - if cutlass.const_expr(self.use_fused_finalize) + if cutlass.const_expr(self.use_fused_finalize or self.load_router_weight) else None ) sTile_info = storage.tile_info.get_tensor( @@ -1944,6 +1948,30 @@ def kernel( gated_identity_mn = gated_tTR_identity_base[ (None, None, None, 0, gated_subtile_idx) ] + if cutlass.const_expr(self.load_router_weight): + top_k = token_final_scales.shape[1] + route = epi_tidx + route_in_tile = gated_subtile_idx * self.epi_tile_n + route + if route < self.epi_tile_n: + permuted_row = work_tile.coord_n + route_in_tile + expanded_idx = permuted_idx_to_expanded_idx[ + permuted_row + ] + safe_idx = cutlass.max(expanded_idx, cutlass.Int32(0)) + token_idx = safe_idx // top_k + topk_idx = safe_idx % top_k + is_valid = cutlass.Int32( + route_in_tile < work_tile.distance_to_boundary + ) + token_scale = token_final_scales[ + (token_idx * is_valid, topk_idx) + ] + sRouteScale[route] = cutlass.Float32( + token_scale + ) * cutlass.Float32(is_valid) + + cute.arch.fence_proxy("async.shared", space="cta") + self.epilog_sync_barrier.arrive_and_wait() up = cute.coalesce( cute.flatten(gated_tTR_rAcc[(None, 0, None)]) ).load() @@ -1955,6 +1983,8 @@ def kernel( ) up_size = cute.size(gated_tTR_rAcc[(None, 0, None)]) for i in cutlass.range_constexpr(0, up_size, 2): + coord_0 = up_coords[i] + coord_1 = up_coords[i + 1] up_pair = cute.arch.mul_packed_f32x2( (up[i], up[i + 1]), (gated_alpha_f32, gated_alpha_f32), @@ -2004,9 +2034,17 @@ def kernel( up_pair, (swiglu_beta, swiglu_beta) ) result = cute.arch.mul_packed_f32x2(up_pair, silu) + if cutlass.const_expr(self.load_router_weight): + # Route in FP32 after activation and before the + # BF16 FC2-input store. + result = cute.arch.mul_packed_f32x2( + result, + ( + sRouteScale[coord_0[1] % self.epi_tile_n], + sRouteScale[coord_1[1] % self.epi_tile_n], + ), + ) - coord_0 = up_coords[i] - coord_1 = up_coords[i + 1] # Fold paired TMEM rows back into the reduced-M # output tile and keep the staged N coordinate local. output_m_0 = coord_0[0] // 32 * 16 + coord_0[0] % 16 @@ -2060,6 +2098,33 @@ def kernel( if cutlass.const_expr(self.fuse_activation): alpha_f32 = cutlass.Float32(alpha_val) + activation_thr_slice = m_thr_offset[ + (None, None, None, subtile_idx) + ] + if cutlass.const_expr(self.load_router_weight): + top_k = token_final_scales.shape[1] + route = epi_tidx + route_in_tile = subtile_idx * self.epi_tile_n + route + if route < self.epi_tile_n: + permuted_row = work_tile.coord_n + route_in_tile + expanded_idx = permuted_idx_to_expanded_idx[ + permuted_row + ] + safe_idx = cutlass.max(expanded_idx, cutlass.Int32(0)) + token_idx = safe_idx // top_k + topk_idx = safe_idx % top_k + is_valid = cutlass.Int32( + route_in_tile < work_tile.distance_to_boundary + ) + token_scale = token_final_scales[ + (token_idx * is_valid, topk_idx) + ] + sRouteScale[route] = cutlass.Float32( + token_scale + ) * cutlass.Float32(is_valid) + + cute.arch.fence_proxy("async.shared", space="cta") + self.epilog_sync_barrier.arrive_and_wait() acc_up = tTR_rAcc.load() for i in cutlass.range_constexpr(0, cute.size(tTR_rAcc), 2): value = cute.arch.mul_packed_f32x2( @@ -2070,9 +2135,22 @@ def kernel( cute.arch.fmax(value[0], cutlass.Float32(0.0)), cute.arch.fmax(value[1], cutlass.Float32(0.0)), ) - tTR_rAcc[i], tTR_rAcc[i + 1] = cute.arch.mul_packed_f32x2( - relu, relu - ) + activated = cute.arch.mul_packed_f32x2(relu, relu) + if cutlass.const_expr(self.load_router_weight): + coord_0 = activation_thr_slice[(i)] + coord_1 = activation_thr_slice[(i + 1)] + # Route in FP32 after activation and before the + # BF16 FC2-input store. + activated = cute.arch.mul_packed_f32x2( + activated, + ( + sRouteScale[coord_0[1] % self.epi_tile_n], + sRouteScale[coord_1[1] % self.epi_tile_n], + ), + ) + tTR_rAcc[i], tTR_rAcc[i + 1] = activated + if cutlass.const_expr(self.load_router_weight): + self.epilog_sync_barrier.arrive_and_wait() if cutlass.const_expr(self.use_fused_finalize): finalize_thr_slice = m_thr_offset[ (None, None, None, subtile_idx) @@ -2089,32 +2167,44 @@ def kernel( finalize_scale_expanded_idx = permuted_idx_to_expanded_idx[ finalize_scale_permuted_row ] - finalize_scale_safe_idx = cutlass.max( - finalize_scale_expanded_idx, cutlass.Int32(0) - ) - finalize_scale_token_idx = finalize_scale_safe_idx // top_k - finalize_scale_topk_idx = finalize_scale_safe_idx % top_k finalize_scale_is_valid = cutlass.Int32( finalize_scale_route_in_tile < work_tile.distance_to_boundary ) - finalize_token_scale = token_final_scales[ - ( - finalize_scale_token_idx * finalize_scale_is_valid, - finalize_scale_topk_idx, + finalize_scale = cutlass.Float32( + alpha_val + ) * cutlass.Float32(finalize_scale_is_valid) + # Input-side mode already routed the activation before + # its BF16 FC2-input store; output-side folds it in here. + if cutlass.const_expr( + not self.apply_router_weight_on_input + ): + finalize_scale_safe_idx = cutlass.max( + finalize_scale_expanded_idx, cutlass.Int32(0) ) - ] - sFinalizeScale[finalize_scale_route] = ( - cutlass.Float32(alpha_val) - * cutlass.Float32(finalize_token_scale) - * cutlass.Float32(finalize_scale_is_valid) - ) + finalize_scale_token_idx = ( + finalize_scale_safe_idx // top_k + ) + finalize_scale_topk_idx = ( + finalize_scale_safe_idx % top_k + ) + finalize_token_scale = token_final_scales[ + ( + finalize_scale_token_idx + * finalize_scale_is_valid, + finalize_scale_topk_idx, + ) + ] + finalize_scale = finalize_scale * cutlass.Float32( + finalize_token_scale + ) + sRouteScale[finalize_scale_route] = finalize_scale cute.arch.fence_proxy("async.shared", space="cta") self.epilog_sync_barrier.arrive_and_wait() for i in cutlass.range(cute.size(tTR_rC), unroll_full=True): finalize_route = finalize_thr_slice[(i)][1] - finalize_value = sFinalizeScale[ + finalize_value = sRouteScale[ finalize_route % self.epi_tile_n ] * cutlass.Float32(tTR_rAcc[i]) sFinalize[ @@ -2261,6 +2351,7 @@ def _compute_stages_and_tmem_cols( transform_a_source: tcgen05.OperandSource, smem_buffer_align_bytes: int, use_fused_finalize: bool, + load_router_weight: bool, use_clc_scheduler: bool, ) -> tuple[int, int, int, int, int, int, int]: """Fit the load, transform, accumulator, and epilogue pipelines.""" @@ -2304,12 +2395,12 @@ def _compute_stages_and_tmem_cols( ) c_bytes_per_stage = cute.size_in_bytes(c_dtype, c_smem_layout_staged_one) c_bytes = c_bytes_per_stage * c_stage_count - finalize_metadata_bytes = ( + route_scale_bytes = ( cute.size_in_bytes( cutlass.Float32, cute.make_layout((cute.size(epi_tile[1]),)), ) - if use_fused_finalize + if use_fused_finalize or load_router_weight else 0 ) @@ -2326,7 +2417,7 @@ def _compute_stages_and_tmem_cols( carveout_smem_bytes = ( bytes_per_pipeline_stage * accumulator_stage_count + c_bytes - + finalize_metadata_bytes + + route_scale_bytes + tile_info_bytes ) diff --git a/flashinfer/fused_moe/cute_dsl/blockscaled_contiguous_gather_grouped_gemm_act_fusion.py b/flashinfer/fused_moe/cute_dsl/blockscaled_contiguous_gather_grouped_gemm_act_fusion.py index b90fb38bca7..32796c738e9 100644 --- a/flashinfer/fused_moe/cute_dsl/blockscaled_contiguous_gather_grouped_gemm_act_fusion.py +++ b/flashinfer/fused_moe/cute_dsl/blockscaled_contiguous_gather_grouped_gemm_act_fusion.py @@ -216,6 +216,7 @@ def _get_compiled_gather_kernel( num_tiles_ptr, norm_const_ptr, a_per_token_scale_ptr, + token_final_scales_ptr, max_active_clusters: int, stream, # Dtype parameters (compile-time - IN cache key) @@ -240,6 +241,7 @@ def _get_compiled_gather_kernel( situ_linear_beta: Optional[float] = None, gated: bool = True, use_a_per_token_scale: bool = False, + apply_router_weight_on_input: bool = False, ): """Get or compile the gather grouped GEMM with FC1 activation fusion. @@ -288,6 +290,7 @@ def _get_compiled_gather_kernel( situ_linear_beta, gated, use_a_per_token_scale, + apply_router_weight_on_input, ) if cache_key not in _gather_kernel_cache: @@ -308,6 +311,7 @@ def _get_compiled_gather_kernel( situ_linear_beta=situ_linear_beta, gated=gated, use_a_per_token_scale=use_a_per_token_scale, + apply_router_weight_on_input=apply_router_weight_on_input, ) # Compile with runtime parameters - they can vary across calls @@ -315,6 +319,7 @@ def _get_compiled_gather_kernel( # (a_ptr, b_ptr, a_sf_ptr, b_sf_ptr, c_ptr, c_sf_ptr, alpha_ptr, # tile_idx_to_group_idx_ptr, tile_idx_to_mn_limit_ptr, token_id_mapping_ptr, # num_non_exiting_tiles_ptr, global_sf_ptr, a_per_token_scale_ptr, + # token_final_scales_ptr, # orig_m, m, n, k, l, tile_size, scaling_vector_size, # max_active_clusters, stream) compiled_gemm = cute.compile( @@ -332,6 +337,7 @@ def _get_compiled_gather_kernel( num_tiles_ptr, norm_const_ptr, a_per_token_scale_ptr, + token_final_scales_ptr, orig_m, permuted_m, n, @@ -363,6 +369,8 @@ def blockscaled_contiguous_gather_grouped_gemm_act_fusion_nvfp4( global_scale: Optional[torch.Tensor] = None, *, a_per_token_scale: Optional[torch.Tensor] = None, + token_final_scales: Optional[torch.Tensor] = None, + apply_router_weight_on_input: bool = False, topk: int = 8, ab_dtype: str = "float4_e2m1fn", sf_dtype: str = "float8_e4m3fn", @@ -416,6 +424,10 @@ def blockscaled_contiguous_gather_grouped_gemm_act_fusion_nvfp4( a_per_token_scale: Optional per-token row scale for operand A, shape (seq_len,), float32. Indexed by the original token ID and applied before the fused activation. + token_final_scales: Routing weights of shape (seq_len, topk). Required + when ``apply_router_weight_on_input`` is true. + apply_router_weight_on_input: Apply each route's weight to the expert + activation output before the FC2-input store. Defaults to False. topk: Number of experts per token. Default: 8 ab_dtype: Data type for A and B matrices. Default: "float4_e2m1fn" sf_dtype: Data type for scale factors. Default: "float8_e4m3fn" @@ -519,6 +531,25 @@ def blockscaled_contiguous_gather_grouped_gemm_act_fusion_nvfp4( f"got {tuple(a_per_token_scale.shape)}" ) + if apply_router_weight_on_input: + if token_final_scales is None: + raise ValueError( + "token_final_scales is required when apply_router_weight_on_input=True" + ) + if token_final_scales.device.type != "cuda": + raise ValueError("token_final_scales must be on CUDA device") + if token_final_scales.device != a.device: + raise ValueError("token_final_scales must be on the same device as a") + if token_final_scales.dtype != torch.float32: + raise ValueError("token_final_scales must have dtype torch.float32") + if not token_final_scales.is_contiguous(): + raise ValueError("token_final_scales must be contiguous") + if token_final_scales.shape != (seq_len, topk): + raise ValueError( + f"token_final_scales must have shape ({seq_len}, {topk}), " + f"got {tuple(token_final_scales.shape)}" + ) + if n % 128 != 0: raise ValueError(f"GEMM1 output dim n={n} must be a multiple of 128.") @@ -646,6 +677,15 @@ def blockscaled_contiguous_gather_grouped_gemm_act_fusion_nvfp4( ) else: a_per_token_scale_ptr = None + token_final_scales_ptr = ( + make_ptr( + cutlass.Float32, + token_final_scales.data_ptr(), + cute.AddressSpace.gmem, + ) + if apply_router_weight_on_input + else None + ) tile_idx_ptr = make_ptr( cutlass.Int32, tile_idx_to_expert_idx.data_ptr(), cute.AddressSpace.gmem ) @@ -685,6 +725,7 @@ def blockscaled_contiguous_gather_grouped_gemm_act_fusion_nvfp4( num_tiles_ptr=num_tiles_ptr, norm_const_ptr=norm_const_ptr, a_per_token_scale_ptr=a_per_token_scale_ptr, + token_final_scales_ptr=token_final_scales_ptr, max_active_clusters=max_active_clusters, stream=stream, # Dtype parameters (compile-time, in cache key) @@ -708,13 +749,15 @@ def blockscaled_contiguous_gather_grouped_gemm_act_fusion_nvfp4( situ_linear_beta=situ_linear_beta, gated=gated, use_a_per_token_scale=use_a_per_token_scale, + apply_router_weight_on_input=apply_router_weight_on_input, ) # Execute kernel with runtime parameters # Order must match wrapper signature: # (a_ptr, b_ptr, a_sf_ptr, b_sf_ptr, c_ptr, c_sf_ptr, alpha_ptr, # tile_idx_ptr, mn_limit_ptr, token_id_ptr, num_tiles_ptr, global_sf_ptr, - # a_per_token_scale_ptr, orig_m, m, n, k, l, stream) + # a_per_token_scale_ptr, token_final_scales_ptr, orig_m, m, n, k, l, + # stream) compiled_gemm( a_ptr, b_ptr, @@ -729,6 +772,7 @@ def blockscaled_contiguous_gather_grouped_gemm_act_fusion_nvfp4( num_tiles_ptr, norm_const_ptr, a_per_token_scale_ptr, + token_final_scales_ptr, seq_len, # orig_m permuted_m, n, diff --git a/flashinfer/fused_moe/cute_dsl/blockscaled_contiguous_grouped_gemm_finalize_fusion.py b/flashinfer/fused_moe/cute_dsl/blockscaled_contiguous_grouped_gemm_finalize_fusion.py index aa962a49c41..dc2f309605e 100644 --- a/flashinfer/fused_moe/cute_dsl/blockscaled_contiguous_grouped_gemm_finalize_fusion.py +++ b/flashinfer/fused_moe/cute_dsl/blockscaled_contiguous_grouped_gemm_finalize_fusion.py @@ -190,6 +190,7 @@ def _get_compiled_finalize_kernel( enable_pdl: bool = True, use_a_per_token_scale: bool = False, use_fused_finalize: bool = True, + apply_router_weight_on_input: bool = False, ): """Get or compile the grouped GEMM with finalize fusion kernel. @@ -212,6 +213,7 @@ def _get_compiled_finalize_kernel( enable_pdl, use_a_per_token_scale, use_fused_finalize, + apply_router_weight_on_input, ) if cache_key not in _finalize_kernel_cache: @@ -224,6 +226,7 @@ def _get_compiled_finalize_kernel( enable_pdl=enable_pdl, use_a_per_token_scale=use_a_per_token_scale, use_fused_finalize=use_fused_finalize, + apply_router_weight_on_input=apply_router_weight_on_input, ) # Compile with runtime parameters - they can vary across calls @@ -289,6 +292,7 @@ def blockscaled_contiguous_grouped_gemm_finalize_fusion_nvfp4( sm_count: Optional[int] = None, enable_pdl: bool = True, use_fused_finalize: bool = True, + apply_router_weight_on_input: bool = False, ) -> torch.Tensor: """Blockscaled contiguous grouped GEMM for MoE GEMM2 workloads. @@ -324,6 +328,8 @@ def blockscaled_contiguous_grouped_gemm_finalize_fusion_nvfp4( sm_count: Number of SMs to use. Default: max available. use_fused_finalize: Use atomic fused finalize; otherwise write expanded rows for deterministic reduction. Default: True. + apply_router_weight_on_input: Routing weights were applied to the FC2 + input, so GEMM2 must not apply them again. Default: False. Returns: out: Output tensor with dtype out_dtype. The shape is ``(seq_len, n)`` @@ -549,6 +555,7 @@ def blockscaled_contiguous_grouped_gemm_finalize_fusion_nvfp4( enable_pdl=enable_pdl, use_fused_finalize=use_fused_finalize, use_a_per_token_scale=use_a_per_token_scale, + apply_router_weight_on_input=apply_router_weight_on_input, ) # Execute kernel with runtime parameters diff --git a/flashinfer/fused_moe/cute_dsl/fused_moe.py b/flashinfer/fused_moe/cute_dsl/fused_moe.py index efca0dd77e5..2f214bb057c 100644 --- a/flashinfer/fused_moe/cute_dsl/fused_moe.py +++ b/flashinfer/fused_moe/cute_dsl/fused_moe.py @@ -179,6 +179,7 @@ def _moe_core_impl( swiglu_limit: float = DEFAULT_SWIGLU_LIMIT, situ_beta: Optional[float] = None, situ_linear_beta: Optional[float] = None, + apply_router_weight_on_input: bool = False, ) -> torch.Tensor: """Core MoE implementation shared by functional and wrapper APIs. @@ -233,6 +234,8 @@ def _moe_core_impl( swiglu_limit: SwiGLU clamp limit. situ_beta: When set with ActivationType.Swiglu, use the SiTU gate. situ_linear_beta: Optional SiTU tanh clamp for the up branch. + apply_router_weight_on_input: Apply routing weights to the expert + activation output, on the FC2 input, instead of after FC2. Returns: Output tensor [num_tokens, hidden_size]. @@ -326,6 +329,8 @@ def _moe_core_impl( num_non_exiting_tiles=num_non_exiting_tiles, out=gemm1_out, **output_kwargs, + token_final_scales=token_final_scales, + apply_router_weight_on_input=apply_router_weight_on_input, topk=top_k, mma_tiler_mn=gemm1_mma_tiler_mn, cluster_shape_mn=gemm1_cluster_shape_mn, @@ -375,6 +380,8 @@ def _moe_core_impl( device=x.device, ) + # Input-side mode scales the activated FC2 input in GEMM1; otherwise + # finalize/unpermute applies the routing weight after FC2, exactly once. # Step 3: GEMM2 with optional atomic finalize blockscaled_contiguous_grouped_gemm_finalize_fusion_nvfp4( a=intermediate, @@ -393,6 +400,7 @@ def _moe_core_impl( cluster_shape_mn=gemm2_cluster_shape_mn, enable_pdl=enable_pdl, use_fused_finalize=use_fused_finalize, + apply_router_weight_on_input=apply_router_weight_on_input, ) # Step 4: Deterministic routing-weight reduction @@ -405,6 +413,7 @@ def _moe_core_impl( num_tokens=num_tokens, top_k=top_k, input_is_expanded=True, + apply_topk_scales=not apply_router_weight_on_input, enable_pdl=enable_pdl, ) @@ -435,6 +444,8 @@ class CuteDslMoEWrapper: resources for CUDA graph capture. use_fused_finalize: Use atomic fused finalize; otherwise use the deterministic two-stage finalize. + apply_router_weight_on_input: Whether this wrapper applies routing + weights to the expert activation output before FC2. quant_mode: Selected W4A4 or W4A16 compute mode. max_num_tokens: Deprecated; accepted for backwards compatibility but ignored. @@ -487,6 +498,7 @@ def __init__( situ_linear_beta: Optional[float] = None, use_fused_finalize: bool = True, quant_mode: str = "w4a4", + apply_router_weight_on_input: bool = False, ): r"""Configure the CuTe-DSL NVFP4 fused-MoE wrapper. @@ -541,6 +553,9 @@ def __init__( quant_mode : str Compute mode: ``"w4a4"`` / ``"nvfp4"`` or ``"w4a16"``. Defaults to ``"w4a4"``. + apply_router_weight_on_input : bool + Apply routing weights after the expert activation, on the FC2 + input, instead of after FC2. Defaults to ``False``. """ activation, gated = normalize_cute_dsl_moe_activation_type(activation_type) quant_mode = quant_mode.lower() @@ -566,6 +581,7 @@ def __init__( self.situ_linear_beta = situ_linear_beta self.use_fused_finalize = use_fused_finalize self.quant_mode = quant_mode + self.apply_router_weight_on_input = apply_router_weight_on_input # Persistent CUDA resources for async-memset / GEMM1 overlap. These # are created outside graph capture (so they can be reused inside it) @@ -608,6 +624,7 @@ def _forward_with_tactic_weak(*args, **kwargs): situ_beta=situ_beta, situ_linear_beta=situ_linear_beta, use_per_token_activation=False, + apply_router_weight_on_input=apply_router_weight_on_input, ) self._per_token_runner = CuteDslFusedMoENvfp4Runner( forward_impl=_forward_with_tactic_weak, @@ -625,6 +642,7 @@ def _forward_with_tactic_weak(*args, **kwargs): situ_beta=situ_beta, situ_linear_beta=situ_linear_beta, use_per_token_activation=True, + apply_router_weight_on_input=apply_router_weight_on_input, ) if use_cuda_graph: @@ -646,6 +664,7 @@ def _forward_with_tactic_weak(*args, **kwargs): swiglu_alpha=swiglu_alpha, swiglu_beta=swiglu_beta, swiglu_limit=swiglu_limit, + apply_router_weight_on_input=apply_router_weight_on_input, ) else: raise ValueError( @@ -680,6 +699,7 @@ def _forward_with_tactic( per_token_scale: Optional[torch.Tensor] = None, enable_pdl: bool = True, use_async_memset: bool = True, + apply_router_weight_on_input: bool = False, **kwargs, ) -> torch.Tensor: """Forward implementation called by auto-tuner.""" @@ -722,6 +742,7 @@ def _forward_with_tactic( swiglu_limit=self.swiglu_limit, situ_beta=self.situ_beta, situ_linear_beta=self.situ_linear_beta, + apply_router_weight_on_input=apply_router_weight_on_input, ) @flashinfer_api(trace=cute_dsl_moe_wrapper_run_trace) @@ -913,6 +934,7 @@ def _cute_dsl_fused_moe_nvfp4_impl( swiglu_limit: float = DEFAULT_SWIGLU_LIMIT, situ_beta: Optional[float] = None, situ_linear_beta: Optional[float] = None, + apply_router_weight_on_input: bool = False, ) -> torch.Tensor: """Internal implementation called by auto-tuner for functional API.""" return _moe_core_impl( @@ -949,6 +971,7 @@ def _cute_dsl_fused_moe_nvfp4_impl( swiglu_limit=swiglu_limit, situ_beta=situ_beta, situ_linear_beta=situ_linear_beta, + apply_router_weight_on_input=apply_router_weight_on_input, ) @@ -984,6 +1007,7 @@ def cute_dsl_fused_moe_nvfp4( *, quant_mode: str = "w4a4", per_token_scale: Optional[torch.Tensor] = None, + apply_router_weight_on_input: bool = False, ) -> torch.Tensor: r"""Run a fused MoE forward pass using the CuTe-DSL NVFP4 kernels. @@ -1061,6 +1085,9 @@ def cute_dsl_fused_moe_nvfp4( Optional SiTU tanh clamp for the up branch. per_token_scale : Optional[torch.Tensor] Optional W4A4 per-token input row scale for GEMM1. + apply_router_weight_on_input : bool + Apply routing weights after the expert activation, on the FC2 input, + instead of after FC2. Defaults to ``False``. Returns ------- @@ -1106,6 +1133,7 @@ def cute_dsl_fused_moe_nvfp4( situ_beta=situ_beta, situ_linear_beta=situ_linear_beta, use_per_token_activation=use_per_token_activation, + apply_router_weight_on_input=apply_router_weight_on_input, ) inputs = [ @@ -1151,6 +1179,7 @@ def cute_dsl_fused_moe_nvfp4( swiglu_alpha=swiglu_alpha, swiglu_beta=swiglu_beta, swiglu_limit=swiglu_limit, + apply_router_weight_on_input=apply_router_weight_on_input, ) inputs = [ x, diff --git a/flashinfer/fused_moe/cute_dsl/moe_utils.py b/flashinfer/fused_moe/cute_dsl/moe_utils.py index 9a0f47bb468..b007a800df0 100644 --- a/flashinfer/fused_moe/cute_dsl/moe_utils.py +++ b/flashinfer/fused_moe/cute_dsl/moe_utils.py @@ -261,6 +261,7 @@ def moe_unpermute( top_k: int, enable_pdl: bool = False, input_is_expanded: bool = False, + apply_topk_scales: bool = True, ) -> None: """ Unpermute and scale outputs after expert computation. @@ -284,10 +285,12 @@ def moe_unpermute( Default is False. input_is_expanded: Whether input rows use expanded (token, top-k slot) order instead of expert-permuted order. + apply_topk_scales: Whether to multiply each expert contribution by its + top-k scale before reduction. Defaults to True. Note: - Output is the weighted sum of expert contributions: - output[i] = sum(topk_scales[i, k] * expert_output[i, k] for k in range(top_k)) + When ``apply_topk_scales`` is True, output is the weighted sum of expert + contributions. Otherwise, output is their unweighted sum. """ module = _get_moe_utils_module() input_dtype_suffix = _get_dtype_suffix(permuted_input.dtype) @@ -316,6 +319,7 @@ def moe_unpermute( hidden_size, top_k, input_is_expanded, + apply_topk_scales, enable_pdl, _get_cuda_stream_ptr(), ) diff --git a/flashinfer/fused_moe/cute_dsl/tuner.py b/flashinfer/fused_moe/cute_dsl/tuner.py index e1e3a9c0141..449fa3e1599 100644 --- a/flashinfer/fused_moe/cute_dsl/tuner.py +++ b/flashinfer/fused_moe/cute_dsl/tuner.py @@ -270,6 +270,8 @@ class CuteDslFusedMoENvfp4Runner(TunableRunner): output_dtype: Output data type (default: torch.bfloat16). use_per_token_activation: Whether inputs include per-token row scales for GEMM1. + apply_router_weight_on_input: Whether routing weights are applied to + the expert activation output before FC2. situ_beta: When set with ActivationType.Swiglu, use the SiTU gate. situ_linear_beta: Optional SiTU tanh clamp for the up branch. """ @@ -291,6 +293,7 @@ def __init__( situ_beta: Optional[float] = None, situ_linear_beta: Optional[float] = None, use_per_token_activation: bool = False, + apply_router_weight_on_input: bool = False, ): activation_type, gated = normalize_cute_dsl_moe_activation_type(activation_type) validate_cute_dsl_moe_situ_config(activation_type, situ_beta, situ_linear_beta) @@ -310,6 +313,7 @@ def __init__( self.situ_beta = situ_beta self.situ_linear_beta = situ_linear_beta self.use_per_token_activation = use_per_token_activation + self.apply_router_weight_on_input = apply_router_weight_on_input # Helper that builds a deterministic balanced approx-max-load # assignment for token_selected_experts during autotune profiling. @@ -446,6 +450,7 @@ def __hash__(self): self.situ_beta, self.situ_linear_beta, self.use_per_token_activation, + self.apply_router_weight_on_input, ) ) @@ -457,6 +462,7 @@ def get_cache_key_extras(self, inputs: List[torch.Tensor]) -> tuple: self.swiglu_limit, self.situ_beta, self.situ_linear_beta, + self.apply_router_weight_on_input, ) def get_valid_tactics( # type: ignore[override] @@ -685,6 +691,7 @@ def forward( # type: ignore[override] swiglu_limit=self.swiglu_limit, situ_beta=self.situ_beta, situ_linear_beta=self.situ_linear_beta, + apply_router_weight_on_input=self.apply_router_weight_on_input, **kwargs, ) @@ -757,6 +764,7 @@ def __init__( swiglu_alpha: float = DEFAULT_SWIGLU_ALPHA, swiglu_beta: float = DEFAULT_SWIGLU_BETA, swiglu_limit: float = DEFAULT_SWIGLU_LIMIT, + apply_router_weight_on_input: bool = False, ): activation_type, _ = normalize_cute_dsl_moe_activation_type(activation_type) if output_dtype != torch.bfloat16: @@ -772,6 +780,7 @@ def __init__( self.swiglu_alpha = swiglu_alpha self.swiglu_beta = swiglu_beta self.swiglu_limit = swiglu_limit + self.apply_router_weight_on_input = apply_router_weight_on_input self._workspace_cache: Dict[Tuple, Any] = {} # Match production EP routing density while retaining seeded load @@ -841,6 +850,7 @@ def __hash__(self): self.swiglu_alpha, self.swiglu_beta, self.swiglu_limit, + self.apply_router_weight_on_input, ) ) @@ -857,6 +867,7 @@ def get_cache_key_extras(self, inputs: List[torch.Tensor]) -> tuple: self.swiglu_alpha, self.swiglu_beta, self.swiglu_limit, + self.apply_router_weight_on_input, ) def get_valid_tactics( # type: ignore[override] @@ -962,6 +973,7 @@ def forward( # type: ignore[override] num_local_experts=self.num_local_experts, local_expert_offset=self.local_expert_offset, use_fused_finalize=self.use_fused_finalize, + apply_router_weight_on_input=self.apply_router_weight_on_input, moe_output=moe_output, enable_pdl=self.enable_pdl, activation_type=self.activation_type, diff --git a/flashinfer/fused_moe/runners.py b/flashinfer/fused_moe/runners.py index 553491bd8eb..0a80efddd1c 100644 --- a/flashinfer/fused_moe/runners.py +++ b/flashinfer/fused_moe/runners.py @@ -179,6 +179,7 @@ class MoERunner(TunableRunner): backend_key: ClassVar[str] = "" supported_routing_modes: tuple[RoutingInputMode, ...] = () supported_quant_variants: ClassVar[tuple[QuantVariant, ...]] = () + supports_apply_router_weight_on_input: ClassVar[bool] = False config: MoEConfig @@ -189,6 +190,14 @@ def check_support(self) -> None: raise NotImplementedError( f"{type(self).__name__} does not support QuantVariant.{variant.name}." ) + if ( + self.config.execution.apply_router_weight_on_input + and not self.supports_apply_router_weight_on_input + ): + raise NotImplementedError( + f"{type(self).__name__} does not support " + "apply_router_weight_on_input=True." + ) # --------------------------------------------------------------------------- @@ -203,6 +212,7 @@ class CuteDslNvfp4Runner(MoERunner): # CuteDSL has no in-kernel router; it only consumes pre-routed packs. supported_routing_modes = (RoutingInputMode.PackedPrecomputed,) supported_quant_variants = (QuantVariant.NVFP4, QuantVariant.W4A16) + supports_apply_router_weight_on_input = True def check_support(self) -> None: super().check_support() @@ -245,6 +255,9 @@ def __init__(self, config: MoEConfig, device: torch.device): enable_pdl=enable_pdl, activation_type=int(config.activation.type), use_per_token_activation=bool(config.quant.per_token_scale), + apply_router_weight_on_input=( + config.execution.apply_router_weight_on_input + ), ) elif config.quant.variant is QuantVariant.W4A16: self._inner = CuteDslFusedMoEW4A16Runner( @@ -255,6 +268,9 @@ def __init__(self, config: MoEConfig, device: torch.device): use_fused_finalize=config.execution.use_fused_finalize, enable_pdl=enable_pdl, activation_type=int(config.activation.type), + apply_router_weight_on_input=( + config.execution.apply_router_weight_on_input + ), ) else: raise NotImplementedError( diff --git a/flashinfer/trace/templates/moe.py b/flashinfer/trace/templates/moe.py index 21d9314290b..78c9ac9d528 100644 --- a/flashinfer/trace/templates/moe.py +++ b/flashinfer/trace/templates/moe.py @@ -2207,6 +2207,7 @@ def _moe_bf16_run_experts( activation_type=ActivationType.Swiglu.value, situ_beta=None, situ_linear_beta=None, + apply_router_weight_on_input=False, ): """Un-quantized (bf16) MoE expert computation.""" activation_type = normalize_activation_type(activation_type) @@ -2245,6 +2246,9 @@ def _moe_bf16_run_experts( token_idx = torch.nonzero(sel_mask, as_tuple=False).squeeze(1) A_e = A.index_select(0, token_idx) G1 = A_e.matmul(W1[le].t()) + w_tok = weights.index_select(0, token_idx) + match = (topk_idx.index_select(0, token_idx) == ge).float() + w_e = (w_tok * match).sum(dim=1) if activation_type == ActivationType.Relu2: act = torch.relu(G1) ** 2 else: @@ -2268,11 +2272,12 @@ def _moe_bf16_run_experts( up = torch.clamp(X1, min=-limit, max=limit) gate = torch.clamp(X2, max=limit) act = gate * torch.sigmoid(alpha * gate) * (up + beta) + if apply_router_weight_on_input: + act = act * w_e.unsqueeze(1) expert_out = act.matmul(W2[le].t()) - w_tok = weights.index_select(0, token_idx) - match = (topk_idx.index_select(0, token_idx) == ge).float() - w_e = (w_tok * match).sum(dim=1) - output.index_add_(0, token_idx, expert_out * w_e.unsqueeze(1)) + if not apply_router_weight_on_input: + expert_out = expert_out * w_e.unsqueeze(1) + output.index_add_(0, token_idx, expert_out) return output.to(torch.bfloat16) @@ -3407,6 +3412,11 @@ def _mono_moe_init( optional=True, description="Compute mode: 'nvfp4'/'w4a4' or 'w4a16'.", ), + "apply_router_weight_on_input": Scalar( + "bool", + optional=True, + description="Apply routing weights to the expert activation output.", + ), "num_experts": Scalar("int32", description="Total number of experts."), "top_k": Scalar("int32", description="Number of experts per token."), "local_expert_offset": Scalar( @@ -3509,6 +3519,11 @@ def _mono_moe_init( optional=True, description="Set at wrapper __init__, not passed to run().", ) +_cute_dsl_wrapper_inputs["apply_router_weight_on_input"] = Scalar( + "bool", + optional=True, + description="Set at wrapper __init__, not passed to run().", +) _cute_dsl_wrapper_axes = dict(cute_dsl_fused_moe_nvfp4_trace.axes) # num_experts / top_k are set at __init__ time — no tensor on run() has a @@ -3711,6 +3726,7 @@ def _cute_dsl_fused_moe_nvfp4_reference( situ_beta=None, situ_linear_beta=None, per_token_scale=None, + apply_router_weight_on_input=False, **_unused, ): """Reference for CuteDSL NvFP4 fused MoE — bridges to the FP4 @@ -3749,6 +3765,7 @@ def _cute_dsl_fused_moe_nvfp4_reference( gemm1_clamp_limit=swiglu_limit, situ_beta=situ_beta, situ_linear_beta=situ_linear_beta, + apply_router_weight_on_input=apply_router_weight_on_input, ) diff --git a/tests/moe/test_cute_dsl_fused_moe.py b/tests/moe/test_cute_dsl_fused_moe.py index 166765bf4ec..d041c3781d0 100644 --- a/tests/moe/test_cute_dsl_fused_moe.py +++ b/tests/moe/test_cute_dsl_fused_moe.py @@ -1231,6 +1231,7 @@ def test_weight_scale_mapping( swiglu_beta=DEFAULT_SWIGLU_BETA, swiglu_limit=DEFAULT_SWIGLU_LIMIT, use_fused_finalize=False, + apply_router_weight_on_input=False, permuted_idx_to_expanded_idx=None, token_final_scales=None, enable_pdl=False, @@ -1284,11 +1285,13 @@ def test_weight_scale_mapping( ), ], ) + @pytest.mark.parametrize("apply_router_weight_on_input", [False, True]) def test_route_tile_boundary_accuracy( self, route_tile: int, gemm1_tactic: tuple, gemm2_tactic: tuple, + apply_router_weight_on_input: bool, ): from flashinfer.fused_moe.cute_dsl.blackwell.moe_w4a16 import ( launch_w4a16_moe, @@ -1332,6 +1335,7 @@ def test_route_tile_boundary_accuracy( use_fused_finalize=False, enable_pdl=False, activation_type=ActivationType.Swiglu, + apply_router_weight_on_input=apply_router_weight_on_input, tactic=tactic, ) ref_output = compute_reference_moe_fp4( @@ -1343,6 +1347,7 @@ def test_route_tile_boundary_accuracy( hidden_size=hidden_size, intermediate_size=intermediate_size, activation_type=ActivationType.Swiglu, + apply_router_weight_on_input=apply_router_weight_on_input, **reference_inputs, ) @@ -1398,6 +1403,7 @@ def test_numerical_accuracy( quant_mode, use_per_token_activation, use_fused_finalize=True, + apply_router_weight_on_input=True, ) @pytest.mark.parametrize( @@ -1470,17 +1476,20 @@ def test_deterministic_finalize_numerical_accuracy( quant_mode, use_per_token_activation, use_fused_finalize=False, + apply_router_weight_on_input=True, ) @pytest.mark.parametrize( "quant_mode, use_per_token_activation", _MOE_QUANT_MODE_CASES, ) + @pytest.mark.parametrize("apply_router_weight_on_input", [False, True]) @pytest.mark.parametrize("hidden_size", [256, 384]) def test_finalize_handles_cluster_padding_and_partial_tiles( self, quant_mode: str, use_per_token_activation: bool, + apply_router_weight_on_input: bool, hidden_size: int, monkeypatch: pytest.MonkeyPatch, ): @@ -1522,6 +1531,7 @@ def choose_tail_config( quant_mode=quant_mode, use_per_token_activation=use_per_token_activation, use_fused_finalize=True, + apply_router_weight_on_input=apply_router_weight_on_input, ) def _run_numerical_accuracy( @@ -1535,6 +1545,7 @@ def _run_numerical_accuracy( quant_mode: str, use_per_token_activation: bool, use_fused_finalize: bool, + apply_router_weight_on_input: bool = False, ): from flashinfer import cute_dsl_fused_moe_nvfp4 @@ -1570,6 +1581,7 @@ def _run_numerical_accuracy( activation_type=activation_type, use_fused_finalize=use_fused_finalize, quant_mode=quant_mode, + apply_router_weight_on_input=apply_router_weight_on_input, **api_inputs, ) @@ -1587,6 +1599,7 @@ def _run_numerical_accuracy( hidden_size=hidden_size, intermediate_size=intermediate_size, activation_type=activation_type, + apply_router_weight_on_input=apply_router_weight_on_input, **reference_inputs, ) @@ -1893,6 +1906,10 @@ class TestCuteDslMoEWrapper: pytestmark = _requires_dsl_arch @pytest.mark.parametrize("num_tokens", [128, 256, 512]) + @pytest.mark.parametrize( + "activation_type", [ActivationType.Swiglu, ActivationType.Relu2] + ) + @pytest.mark.parametrize("apply_router_weight_on_input", [False, True]) @pytest.mark.parametrize("use_fused_finalize", [False, True]) @pytest.mark.parametrize( "quant_mode,use_per_token_activation", _MOE_QUANT_MODE_CASES @@ -1907,12 +1924,15 @@ def test_wrapper_accuracy( use_fused_finalize: bool, quant_mode: str, use_per_token_activation: bool, + apply_router_weight_on_input: bool, + activation_type: ActivationType, ): """Accuracy test for wrapper API.""" from flashinfer import CuteDslMoEWrapper hidden_size, intermediate_size = 256, 512 + _, gated = normalize_cute_dsl_moe_activation_type(activation_type) tensors = create_moe_tensors( num_tokens=num_tokens, hidden_size=hidden_size, @@ -1920,6 +1940,7 @@ def test_wrapper_accuracy( num_experts=num_experts, num_local_experts=num_experts, top_k=top_k, + gated=gated, use_per_token_activation=use_per_token_activation, ) api_inputs, reference_inputs = _prepare_moe_quant_mode_inputs( @@ -1934,7 +1955,9 @@ def test_wrapper_accuracy( intermediate_size=intermediate_size, use_cuda_graph=False, use_fused_finalize=use_fused_finalize, + activation_type=activation_type, quant_mode=quant_mode, + apply_router_weight_on_input=apply_router_weight_on_input, ) result = moe.run( @@ -1961,6 +1984,8 @@ def test_wrapper_accuracy( top_k=top_k, hidden_size=hidden_size, intermediate_size=intermediate_size, + activation_type=activation_type, + apply_router_weight_on_input=apply_router_weight_on_input, **reference_inputs, ) diff --git a/tests/moe/utils.py b/tests/moe/utils.py index 3df2537cebb..41098d8f70d 100644 --- a/tests/moe/utils.py +++ b/tests/moe/utils.py @@ -343,6 +343,7 @@ def compute_reference_moe_fp4( situ_linear_beta: float | None = None, gemm1_alpha: torch.Tensor | None = None, gemm2_alpha: torch.Tensor | None = None, + apply_router_weight_on_input: bool = False, ) -> torch.Tensor: """Compute reference MoE output using PyTorch operations on GPU. @@ -374,6 +375,8 @@ def compute_reference_moe_fp4( situ_linear_beta: Optional SiTU tanh clamp for the linear branch. gemm1_alpha: GEMM1 per-expert scalar scales [num_local_experts] gemm2_alpha: GEMM2 per-expert scalar scales [num_local_experts] + apply_router_weight_on_input: Apply the routing weight to the expert + activation output, on the FC2 input, instead of the FC2 output. Returns: Output tensor [num_tokens, hidden_size] @@ -483,6 +486,9 @@ def compute_reference_moe_fp4( else: act_out = torch.relu(gemm1_out) ** 2 + if apply_router_weight_on_input: + act_out = scale * act_out + if fc2_input_scale is not None: if use_per_token_activation: act_out, per_token_scale = quant_dequant_fp4_per_token_reference( @@ -496,7 +502,9 @@ def compute_reference_moe_fp4( w2 = gemm2_weights[local_idx] gemm2_out = act_out @ w2.T - output_scale = scale * gemm2_alpha[local_idx] + output_scale = gemm2_alpha[local_idx] + if not apply_router_weight_on_input: + output_scale = scale * output_scale if per_token_scale is not None: output_scale = output_scale * per_token_scale[0] output[token_idx] += output_scale * gemm2_out.squeeze(0)