Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions benchmarks/bench_cute_dsl_moe_distributed.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,8 @@ def _profile_worker_arguments(args, num_tokens):
]
if args.use_per_token_activation:
arguments.append("--use-per-token-activation")
if args.apply_router_weight_on_input:
arguments.append("--apply-router-weight-on-input")
if not args.use_fused_finalize:
arguments.append("--no-fused-finalize")
if not args.enable_pdl:
Expand Down Expand Up @@ -451,6 +453,7 @@ def _distributed_moe_config(
enable_pdl=args.enable_pdl,
tune_max_num_tokens=tune_max_num_tokens,
use_fused_finalize=args.use_fused_finalize,
apply_router_weight_on_input=args.apply_router_weight_on_input,
),
)

Expand Down Expand Up @@ -1267,6 +1270,10 @@ def _run_distributed_benchmark(args, token_counts):
)
if rank == 0:
print("\nDeepSeek-V3 distributed CuTe DSL MoE benchmark")
print(
"CuTe DSL router weight placement: "
f"{'input' if args.apply_router_weight_on_input else 'output'}"
)

selected_mode = None
selected_variant_name = None
Expand Down Expand Up @@ -1337,6 +1344,13 @@ def main():
dest="use_fused_finalize",
help="Use deterministic two-stage finalize.",
)
parser.add_argument(
"--apply-router-weight-on-input",
action="store_true",
help=(
"Apply routing weights to the CuTe DSL FC2 input instead of after GEMM2."
),
)
parser.add_argument(
"--no-pdl",
action="store_false",
Expand Down
31 changes: 31 additions & 0 deletions benchmarks/bench_moe_deepseek.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@
python bench_moe_deepseek.py --functional-api # atomic fused (default)
python bench_moe_deepseek.py --functional-api --no-fused-finalize # deterministic

# Apply routing weights to the FC2 input
python bench_moe_deepseek.py --apply-router-weight-on-input

Metrics:
- ms: Latency in milliseconds
- TFLOPS: Computational throughput
Expand Down Expand Up @@ -279,6 +282,7 @@ def bench_cute_dsl(
use_fused_finalize=True,
profile_cuda=False,
profile_iters=10,
apply_router_weight_on_input=False,
):
"""Benchmark CuteDSL MoE.

Expand All @@ -300,6 +304,8 @@ def bench_cute_dsl(
profile_cuda: Capture steady-state CUDA graph replays between
cudaProfilerStart/Stop instead of benchmarking.
profile_iters: Number of cold-L2 graph replays to capture.
apply_router_weight_on_input: Apply routing weights after the expert
activation, on the FC2 input, instead of after GEMM2.
"""
import contextlib

Expand Down Expand Up @@ -417,6 +423,7 @@ def prepare_activation(x, x_sf):
local_expert_offset=local_expert_offset,
use_fused_finalize=use_fused_finalize,
quant_mode=quant_mode,
apply_router_weight_on_input=apply_router_weight_on_input,
)

def run(x, x_sf, router_logits, routing_bias, topk_values, topk_indices):
Expand Down Expand Up @@ -480,6 +487,7 @@ def run(x, x_sf, router_logits, routing_bias, topk_values, topk_indices):
quant_mode=quant_mode,
per_token_scale=per_token_scale,
use_fused_finalize=use_fused_finalize,
apply_router_weight_on_input=apply_router_weight_on_input,
)

# Pass input tensors via input_kwargs for cold L2 cache rotation
Expand Down Expand Up @@ -856,6 +864,7 @@ def run_benchmark(
profile_cuda=False,
profile_iters=10,
profile_backend=None,
apply_router_weight_on_input=False,
):
"""
Unified benchmark for DeepSeek-V3 MoE backends.
Expand Down Expand Up @@ -894,6 +903,8 @@ def run_benchmark(
profile_cuda: Capture one backend for an external CUDA profiler.
profile_iters: Number of cold-L2 graph replays to capture.
profile_backend: Backend to run when profile_cuda is enabled.
apply_router_weight_on_input: Apply routing weights after the expert
activation, on the FC2 input, instead of after GEMM2.

Returns:
List of BenchResult objects
Expand Down Expand Up @@ -932,6 +943,7 @@ def run_benchmark(
profile_cuda=profile_cuda,
profile_iters=profile_iters,
profile_backend=profile_backend,
apply_router_weight_on_input=apply_router_weight_on_input,
)
results.extend(row)
rows_and_histograms.append((row, histogram_record))
Expand All @@ -951,6 +963,7 @@ def run_benchmark(
use_per_token_activation=use_per_token_activation,
include_activation_quant=include_activation_quant,
use_fused_finalize=use_fused_finalize,
apply_router_weight_on_input=apply_router_weight_on_input,
)
for row, histogram_record in rows_and_histograms:
_print_row(row, histogram_record)
Expand All @@ -976,6 +989,7 @@ def _benchmark_single(
profile_cuda=False,
profile_iters=10,
profile_backend=None,
apply_router_weight_on_input=False,
):
"""Benchmark all backends for a single token count.

Expand Down Expand Up @@ -1009,6 +1023,7 @@ def _benchmark_single(
use_fused_finalize=use_fused_finalize,
profile_cuda=profile_cuda,
profile_iters=profile_iters,
apply_router_weight_on_input=apply_router_weight_on_input,
)
if run_cute_dsl_w4a16:
lat["CuteDSL W4A16"] = bench_cute_dsl(
Expand All @@ -1027,6 +1042,7 @@ def _benchmark_single(
use_fused_finalize=use_fused_finalize,
profile_cuda=profile_cuda,
profile_iters=profile_iters,
apply_router_weight_on_input=apply_router_weight_on_input,
)
if run_cutlass and not use_per_token_activation:
lat["CUTLASS"] = bench_cutlass(
Expand Down Expand Up @@ -1082,6 +1098,7 @@ def _print_header(
use_per_token_activation=False,
include_activation_quant=False,
use_fused_finalize=True,
apply_router_weight_on_input=False,
):
"""Print benchmark header."""
table_width = 120 if use_per_token_activation else 159
Expand Down Expand Up @@ -1124,6 +1141,10 @@ def _print_header(
"CuteDSL finalize: "
f"{'atomic fused' if use_fused_finalize else 'deterministic two-stage'}"
)
print(
"CuteDSL router weight placement: "
f"{'input' if apply_router_weight_on_input else 'output'}"
)
if use_per_token_activation:
print("CUTLASS omitted: it does not consume the per-token activation scale.")
print("-" * table_width)
Expand Down Expand Up @@ -1357,6 +1378,11 @@ def main():
dest="use_fused_finalize",
help="Use deterministic two-stage CuTe DSL finalize instead of atomic fused finalize.",
)
parser.add_argument(
"--apply-router-weight-on-input",
action="store_true",
help="Apply routing weights to the CuTe DSL FC2 input instead of after GEMM2.",
)
parser.add_argument(
"--profile-cuda",
action="store_true",
Expand Down Expand Up @@ -1418,6 +1444,10 @@ def main():
"CuteDSL finalize: "
f"{'atomic fused' if args.use_fused_finalize else 'deterministic two-stage'}"
)
print(
"CuteDSL router weight placement: "
f"{'input' if args.apply_router_weight_on_input else 'output'}"
)

run_benchmark(
token_counts=tokens,
Expand All @@ -1437,6 +1467,7 @@ def main():
profile_cuda=args.profile_cuda,
profile_iters=args.profile_iters,
profile_backend=args.profile_backend,
apply_router_weight_on_input=args.apply_router_weight_on_input,
)

return 0
Expand Down
30 changes: 16 additions & 14 deletions csrc/moe_utils_binding.cu
Original file line number Diff line number Diff line change
Expand Up @@ -126,59 +126,61 @@ void moe_unpermute_fp16_float_scale(int64_t permuted_input_ptr, int64_t output_p
int64_t expanded_idx_to_permuted_idx_ptr,
int64_t topk_scales_ptr, int32_t num_tokens,
int32_t hidden_size, int32_t top_k, bool input_is_expanded,
bool enable_pdl, int64_t cuda_stream_ptr) {
bool apply_topk_scales, bool enable_pdl,
int64_t cuda_stream_ptr) {
cudaStream_t stream =
cuda_stream_ptr != 0 ? reinterpret_cast<cudaStream_t>(cuda_stream_ptr) : get_current_stream();
moeUnpermute<half, float>(reinterpret_cast<half const*>(permuted_input_ptr),
reinterpret_cast<half*>(output_ptr),
reinterpret_cast<int32_t const*>(expanded_idx_to_permuted_idx_ptr),
reinterpret_cast<float const*>(topk_scales_ptr), num_tokens,
hidden_size, top_k, input_is_expanded, enable_pdl, stream);
moeUnpermute<half, float>(
reinterpret_cast<half const*>(permuted_input_ptr), reinterpret_cast<half*>(output_ptr),
reinterpret_cast<int32_t const*>(expanded_idx_to_permuted_idx_ptr),
reinterpret_cast<float const*>(topk_scales_ptr), num_tokens, hidden_size, top_k,
input_is_expanded, apply_topk_scales, enable_pdl, stream);
}

void moe_unpermute_fp16_half_scale(int64_t permuted_input_ptr, int64_t output_ptr,
int64_t expanded_idx_to_permuted_idx_ptr,
int64_t topk_scales_ptr, int32_t num_tokens, int32_t hidden_size,
int32_t top_k, bool input_is_expanded, bool enable_pdl,
int64_t cuda_stream_ptr) {
int32_t top_k, bool input_is_expanded, bool apply_topk_scales,
bool enable_pdl, int64_t cuda_stream_ptr) {
cudaStream_t stream =
cuda_stream_ptr != 0 ? reinterpret_cast<cudaStream_t>(cuda_stream_ptr) : get_current_stream();
moeUnpermute<half, half>(reinterpret_cast<half const*>(permuted_input_ptr),
reinterpret_cast<half*>(output_ptr),
reinterpret_cast<int32_t const*>(expanded_idx_to_permuted_idx_ptr),
reinterpret_cast<half const*>(topk_scales_ptr), num_tokens, hidden_size,
top_k, input_is_expanded, enable_pdl, stream);
top_k, input_is_expanded, apply_topk_scales, enable_pdl, stream);
}

#ifdef ENABLE_BF16
void moe_unpermute_bf16_float_scale(int64_t permuted_input_ptr, int64_t output_ptr,
int64_t expanded_idx_to_permuted_idx_ptr,
int64_t topk_scales_ptr, int32_t num_tokens,
int32_t hidden_size, int32_t top_k, bool input_is_expanded,
bool enable_pdl, int64_t cuda_stream_ptr) {
bool apply_topk_scales, bool enable_pdl,
int64_t cuda_stream_ptr) {
cudaStream_t stream =
cuda_stream_ptr != 0 ? reinterpret_cast<cudaStream_t>(cuda_stream_ptr) : get_current_stream();
moeUnpermute<__nv_bfloat16, float>(
reinterpret_cast<__nv_bfloat16 const*>(permuted_input_ptr),
reinterpret_cast<__nv_bfloat16*>(output_ptr),
reinterpret_cast<int32_t const*>(expanded_idx_to_permuted_idx_ptr),
reinterpret_cast<float const*>(topk_scales_ptr), num_tokens, hidden_size, top_k,
input_is_expanded, enable_pdl, stream);
input_is_expanded, apply_topk_scales, enable_pdl, stream);
}

void moe_unpermute_bf16_bf16_scale(int64_t permuted_input_ptr, int64_t output_ptr,
int64_t expanded_idx_to_permuted_idx_ptr,
int64_t topk_scales_ptr, int32_t num_tokens, int32_t hidden_size,
int32_t top_k, bool input_is_expanded, bool enable_pdl,
int64_t cuda_stream_ptr) {
int32_t top_k, bool input_is_expanded, bool apply_topk_scales,
bool enable_pdl, int64_t cuda_stream_ptr) {
cudaStream_t stream =
cuda_stream_ptr != 0 ? reinterpret_cast<cudaStream_t>(cuda_stream_ptr) : get_current_stream();
moeUnpermute<__nv_bfloat16, __nv_bfloat16>(
reinterpret_cast<__nv_bfloat16 const*>(permuted_input_ptr),
reinterpret_cast<__nv_bfloat16*>(output_ptr),
reinterpret_cast<int32_t const*>(expanded_idx_to_permuted_idx_ptr),
reinterpret_cast<__nv_bfloat16 const*>(topk_scales_ptr), num_tokens, hidden_size, top_k,
input_is_expanded, enable_pdl, stream);
input_is_expanded, apply_topk_scales, enable_pdl, stream);
}
#endif

Expand Down
34 changes: 24 additions & 10 deletions csrc/nv_internal/tensorrt_llm/kernels/cuteDslKernels/moeUtils.cu
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,8 @@ INSTANTIATE_MOE_PERMUTE(__nv_fp4_e2m1, uint8_t);
#endif
#undef INSTANTIATE_MOE_PERMUTE

template <typename InputType, typename TopKScaleType, int32_t kThreadsPerBlock>
template <typename InputType, typename TopKScaleType, int32_t kThreadsPerBlock,
bool ApplyTopKScales>
__global__ void moeUnpermuteKernel(InputType const* permuted_input, InputType* output,
int32_t const* expanded_idx_to_permuted_idx,
TopKScaleType const* topk_scales, int32_t const hidden_size,
Expand Down Expand Up @@ -190,11 +191,18 @@ __global__ void moeUnpermuteKernel(InputType const* permuted_input, InputType* o
auto const* src_ptr =
reinterpret_cast<ElemCopyType const*>(permuted_input) + input_idx * kCopyPerToken;
*reinterpret_cast<ElemCopyType*>(rmem) = src_ptr[i];
TopKScaleType const scale = topk_scales[expanded_idx];

if constexpr (ApplyTopKScales) {
TopKScaleType const scale = topk_scales[expanded_idx];
#pragma unroll
for (int32_t j = 0; j < kElemPerCopy; j++) {
rmemAccum[j] += static_cast<AccumType>(rmem[j]) * static_cast<AccumType>(scale);
for (int32_t j = 0; j < kElemPerCopy; j++) {
rmemAccum[j] += static_cast<AccumType>(rmem[j]) * static_cast<AccumType>(scale);
}
} else {
#pragma unroll
for (int32_t j = 0; j < kElemPerCopy; j++) {
rmemAccum[j] += static_cast<AccumType>(rmem[j]);
}
}
}
#pragma unroll
Expand All @@ -213,7 +221,8 @@ template <typename InputType, typename TopKScaleType>
void moeUnpermute(InputType const* permuted_input, InputType* output,
int32_t const* expanded_idx_to_permuted_idx, TopKScaleType const* topk_scales,
int32_t const num_tokens, int32_t const hidden_size, int32_t const top_k,
bool input_is_expanded, bool enable_pdl, cudaStream_t stream) {
bool input_is_expanded, bool apply_topk_scales, bool enable_pdl,
cudaStream_t stream) {
int32_t constexpr kThreadsPerBlock = 256;
int32_t constexpr kElemPerCopy = elemPerCopy<InputType>();
TLLM_CHECK_WITH_INFO(hidden_size % kElemPerCopy == 0, "hidden_size must be divisible by %d.",
Expand All @@ -222,8 +231,6 @@ void moeUnpermute(InputType const* permuted_input, InputType* output,
int32_t const blocks = num_tokens;
int32_t const threads = kThreadsPerBlock;

auto kernel = &moeUnpermuteKernel<InputType, TopKScaleType, kThreadsPerBlock>;

cudaLaunchConfig_t config;
config.gridDim = blocks;
config.blockDim = threads;
Expand All @@ -234,16 +241,23 @@ void moeUnpermute(InputType const* permuted_input, InputType* output,
attrs[0].val.programmaticStreamSerializationAllowed = enable_pdl;
config.numAttrs = 1;
config.attrs = attrs;
cudaLaunchKernelEx(&config, kernel, permuted_input, output, expanded_idx_to_permuted_idx,
topk_scales, hidden_size, top_k, input_is_expanded);
if (apply_topk_scales) {
auto kernel = &moeUnpermuteKernel<InputType, TopKScaleType, kThreadsPerBlock, true>;
cudaLaunchKernelEx(&config, kernel, permuted_input, output, expanded_idx_to_permuted_idx,
topk_scales, hidden_size, top_k, input_is_expanded);
} else {
auto kernel = &moeUnpermuteKernel<InputType, TopKScaleType, kThreadsPerBlock, false>;
cudaLaunchKernelEx(&config, kernel, permuted_input, output, expanded_idx_to_permuted_idx,
topk_scales, hidden_size, top_k, input_is_expanded);
}
}

#define INSTANTIATE_MOE_UNPERMUTE(InputType, TopKScaleType) \
template void moeUnpermute<InputType>( \
InputType const* permuted_input, InputType* output, \
int32_t const* expanded_idx_to_permuted_idx, TopKScaleType const* topk_scales, \
int32_t const num_tokens, int32_t const hidden_size, int32_t const top_k, \
bool input_is_expanded, bool enable_pdl, cudaStream_t stream)
bool input_is_expanded, bool apply_topk_scales, bool enable_pdl, cudaStream_t stream)

INSTANTIATE_MOE_UNPERMUTE(half, float);
INSTANTIATE_MOE_UNPERMUTE(half, half);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ template <typename InputType, typename TopKScaleType>
void moeUnpermute(InputType const* permuted_input, InputType* output,
int32_t const* expanded_idx_to_permuted_idx, TopKScaleType const* topk_scales,
int32_t const num_tokens, int32_t const hidden_size, int32_t const top_k,
bool input_is_expanded, bool enable_pdl, cudaStream_t stream);
bool input_is_expanded, bool apply_topk_scales, bool enable_pdl,
cudaStream_t stream);

template <typename InputType>
void moeOutputMemset(InputType* input, int32_t const* tile_idx_to_mn_limit,
Expand Down
8 changes: 8 additions & 0 deletions flashinfer/fused_moe/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -204,12 +204,16 @@ class ExecutionConfig:
Token budget hint for autotuner / CUDA graph capture.
use_fused_finalize : bool
Whether supported backends reduce routed outputs in the GEMM2 epilogue.
apply_router_weight_on_input : bool
Whether supported backends apply routing weights after the expert
activation, on the FC2 input, instead of after GEMM2.
"""

do_finalize: bool = True
enable_pdl: Optional[bool] = None
tune_max_num_tokens: int = 8192
use_fused_finalize: bool = True
apply_router_weight_on_input: bool = False
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

def __repr__(self) -> str:
parts = []
Expand All @@ -221,6 +225,10 @@ def __repr__(self) -> str:
parts.append(f"tune_max_num_tokens={self.tune_max_num_tokens!r}")
if not self.use_fused_finalize:
parts.append(f"use_fused_finalize={self.use_fused_finalize!r}")
if self.apply_router_weight_on_input:
parts.append(
f"apply_router_weight_on_input={self.apply_router_weight_on_input!r}"
)
return f"ExecutionConfig({', '.join(parts)})"


Expand Down
Loading
Loading