diff --git a/include/flashinfer/comm/trtllm_moe_allreduce_fusion.cuh b/include/flashinfer/comm/trtllm_moe_allreduce_fusion.cuh index e5e1cf284c2..a7596aac93e 100644 --- a/include/flashinfer/comm/trtllm_moe_allreduce_fusion.cuh +++ b/include/flashinfer/comm/trtllm_moe_allreduce_fusion.cuh @@ -1292,9 +1292,8 @@ __global__ void moefinalize_allreduce_fusion_kernel_oneshot_lamport( int thread_offset_within_token = threadid_in_cluster * VEC_SIZE; int top_k = params.top_k; - bool use_scale_factor = params.expert_scale_factor != nullptr; + auto const* expert_scales = static_cast(params.expert_scale_factor); float routed_scaling_factor = params.routed_scaling_factor; - bool use_routed_scaling_factor = routed_scaling_factor != 1.0f; // Persistent Kernel // Each cluster iterate through all token it need to handle @@ -1303,52 +1302,63 @@ __global__ void moefinalize_allreduce_fusion_kernel_oneshot_lamport( break; } - // * MoE finalize - vec_t accumulator; - accumulator.fill(0.f); - - for (int k = 0; k < top_k; k++) { - int const expanded_idx = token_id * top_k + k; - int32_t const permuted_idx = params.expanded_idx_to_permuted_idx[expanded_idx]; - - if (permuted_idx == -1) continue; - - int thread_offset_across_token = - permuted_idx * params.hidden_dim + thread_offset_within_token; - vec_t permuted_data; - permuted_data.load(reinterpret_cast(params.allreduce_in) + thread_offset_across_token); + // * MoE finalize, accumulated in FP32. + float acc_f32[VEC_SIZE] = {}; + // The shared expert row does not depend on the routing; load it first. + vec_t shared_expert_output; + if (params.shared_expert_output) { + shared_expert_output.load(reinterpret_cast(params.shared_expert_output) + + token_id * params.hidden_dim + thread_offset_within_token); + } - // * acc += scale(data) - if (use_scale_factor) { - float block_scale = - static_cast(static_cast(params.expert_scale_factor)[expanded_idx]); + // Experts are gathered kTopKChunk at a time: all permute indices and scales first, then + // all rows, so their loads are in flight together instead of one dependent pair per expert. + constexpr int kTopKChunk = 8; + for (int k0 = 0; k0 < top_k; k0 += kTopKChunk) { + int32_t permuted_idx[kTopKChunk]; + float block_scale[kTopKChunk]; #pragma unroll - for (int i = 0; i < VEC_SIZE; ++i) { - // assume computation is done in ScaleType - accumulator[i] += static_cast(static_cast(permuted_data[i]) * block_scale); + for (int j = 0; j < kTopKChunk; ++j) { + int const k = k0 + j; + int const expanded_idx = token_id * top_k + k; + permuted_idx[j] = -1; + block_scale[j] = 1.f; + if (k < top_k) { + permuted_idx[j] = params.expanded_idx_to_permuted_idx[expanded_idx]; + if (expert_scales) { + block_scale[j] = static_cast(expert_scales[expanded_idx]); + } } - } else { - accumulator = vec_add(accumulator, permuted_data); } - } - - // Apply the global routed scaling once after accumulating all routed experts. - if (use_routed_scaling_factor) { + vec_t permuted_data[kTopKChunk]; #pragma unroll - for (int i = 0; i < VEC_SIZE; ++i) { - accumulator[i] = static_cast(static_cast(accumulator[i]) * routed_scaling_factor); + for (int j = 0; j < kTopKChunk; ++j) { + if (permuted_idx[j] != -1) { + permuted_data[j].load(reinterpret_cast(params.allreduce_in) + + permuted_idx[j] * params.hidden_dim + thread_offset_within_token); + } + } +#pragma unroll + for (int j = 0; j < kTopKChunk; ++j) { + if (permuted_idx[j] != -1) { +#pragma unroll + for (int i = 0; i < VEC_SIZE; ++i) { + acc_f32[i] += static_cast(permuted_data[j][i]) * block_scale[j]; + } + } } } - // * Add shared expert output - if (params.shared_expert_output) { - // * Load shared expert output - int thread_offset_across_token = token_id * params.hidden_dim + thread_offset_within_token; - vec_t shared_expert_output; - shared_expert_output.load(reinterpret_cast(params.shared_expert_output) + - thread_offset_across_token); + // Apply the global routed scaling once after accumulating all routed experts, then add the + // shared expert output. + vec_t accumulator; #pragma unroll - accumulator = vec_add(accumulator, shared_expert_output); + for (int i = 0; i < VEC_SIZE; ++i) { + float v = acc_f32[i] * routed_scaling_factor; + if (params.shared_expert_output) { + v += static_cast(shared_expert_output[i]); + } + accumulator[i] = static_cast(v); } // * AR Store @@ -1379,15 +1389,21 @@ __global__ void moefinalize_allreduce_fusion_kernel_oneshot_lamport( #pragma unroll for (int r = 0; r < NRanks; ++r) { // LDG.128 from local rank - vals[r].load_global_volatile(reinterpret_cast(comm.data_bufs[r]) + + vals[r].load_global_volatile(reinterpret_cast(comm.data_bufs[params.rank]) + (r * tot_access + idx) * VEC_SIZE); done &= !has_neg_zero(vals[r]); } } - vec_t sum_val = vals[0]; + // Sum the ranks in FP32. + vec_t sum_val; #pragma unroll - for (int r = 1; r < NRanks; ++r) { - sum_val = vec_add(sum_val, vals[r]); + for (int i = 0; i < VEC_SIZE; ++i) { + float sum = static_cast(vals[0][i]); +#pragma unroll + for (int r = 1; r < NRanks; ++r) { + sum += static_cast(vals[r][i]); + } + sum_val[i] = static_cast(sum); } // * Fuse: AllReduceOut is always false in finalize_moe_allreduce diff --git a/tests/comm/test_cake_moe_finalize_allreduce.py b/tests/comm/test_cake_moe_finalize_allreduce.py index 7ae325ee611..0b633abe386 100644 --- a/tests/comm/test_cake_moe_finalize_allreduce.py +++ b/tests/comm/test_cake_moe_finalize_allreduce.py @@ -100,9 +100,13 @@ def _bounded_rand( return ((values - 0.5) * 0.125).to(dtype).contiguous() -def _rank_order_sum(local: torch.Tensor, group: dist.ProcessGroup) -> torch.Tensor: +def _rank_order_sum( + local: torch.Tensor, group: dist.ProcessGroup, fp32_acc: bool +) -> torch.Tensor: peers = [torch.empty_like(local) for _ in range(dist.get_world_size(group))] dist.all_gather(peers, local, group=group) + if fp32_acc: + return torch.stack(peers).float().sum(dim=0).to(local.dtype) total = peers[0] for peer in peers[1:]: total = (total.float() + peer.float()).to(local.dtype) @@ -121,18 +125,31 @@ def _reference( eps: float, weight_bias: float, group: dist.ProcessGroup, + fp32_acc: bool, ) -> tuple[torch.Tensor, torch.Tensor]: + """With ``fp32_acc`` (the TRT-LLM kernel), the routes, the shared expert add and + the rank sum accumulate in FP32 and round once each; otherwise (Cake) every step + rounds to the activation dtype.""" gathered = allreduce_in[inverse_indices] - local = torch.zeros_like(residual_in) - for route in range(inverse_indices.shape[1]): - contribution = ( - gathered[:, route].float() * expert_scales[:, route].float().unsqueeze(-1) - ).to(local.dtype) - local = (local.float() + contribution.float()).to(local.dtype) - local = (local.float() * routed_scaling_factor).to(local.dtype) - if shared_expert_output is not None: - local = (local.float() + shared_expert_output.float()).to(local.dtype) - reduced = _rank_order_sum(local, group) + if fp32_acc: + local_f32 = (gathered.float() * expert_scales.float().unsqueeze(-1)).sum( + dim=1 + ) * routed_scaling_factor + if shared_expert_output is not None: + local_f32 = local_f32 + shared_expert_output.float() + local = local_f32.to(residual_in.dtype) + else: + local = torch.zeros_like(residual_in) + for route in range(inverse_indices.shape[1]): + contribution = ( + gathered[:, route].float() + * expert_scales[:, route].float().unsqueeze(-1) + ).to(local.dtype) + local = (local.float() + contribution.float()).to(local.dtype) + local = (local.float() * routed_scaling_factor).to(local.dtype) + if shared_expert_output is not None: + local = (local.float() + shared_expert_output.float()).to(local.dtype) + reduced = _rank_order_sum(local, group, fp32_acc) residual = (reduced.float() + residual_in.float()).to(local.dtype) residual_f32 = residual.float() norm = ( @@ -280,20 +297,26 @@ def _worker(world_size: int, rank: int, dtype: torch.dtype, port: int) -> None: else None ) routed_scaling_factor = 2.5 if use_shared else None - residual_ref, norm_ref = _reference( - allreduce_in=allreduce_in, - residual_in=residual_in, - norm_weight=norm_weight, - inverse_indices=inverse_indices, - expert_scales=expert_scales, - shared_expert_output=shared_expert_output, - routed_scaling_factor=( - 1.0 if routed_scaling_factor is None else routed_scaling_factor - ), - eps=1e-5, - weight_bias=weight_bias, - group=group, - ) + refs = { + backend: _reference( + allreduce_in=allreduce_in, + residual_in=residual_in, + norm_weight=norm_weight, + inverse_indices=inverse_indices, + expert_scales=expert_scales, + shared_expert_output=shared_expert_output, + routed_scaling_factor=( + 1.0 + if routed_scaling_factor is None + else routed_scaling_factor + ), + eps=1e-5, + weight_bias=weight_bias, + group=group, + fp32_acc=backend == "trtllm", + ) + for backend in ("trtllm", "cake") + } outputs: dict[str, tuple[torch.Tensor, ...]] = {} for backend in ("trtllm", "cake"): @@ -327,6 +350,7 @@ def _worker(world_size: int, rank: int, dtype: torch.dtype, port: int) -> None: trtllm_result = outputs["trtllm"] for backend, result in outputs.items(): + residual_ref, norm_ref = refs[backend] torch.testing.assert_close( result[0].float(), residual_ref.float(), diff --git a/tests/comm/test_trtllm_moe_allreduce_fusion_finalize.py b/tests/comm/test_trtllm_moe_allreduce_fusion_finalize.py index 47e9a98b91f..1fb0a93ba3a 100644 --- a/tests/comm/test_trtllm_moe_allreduce_fusion_finalize.py +++ b/tests/comm/test_trtllm_moe_allreduce_fusion_finalize.py @@ -1,5 +1,7 @@ import multiprocessing as mp +import os import socket +import time from typing import Any import numpy as np @@ -53,7 +55,7 @@ def rms_norm(x: torch.Tensor, weight: torch.Tensor = None, eps: float = 1e-6): try: device = torch.device(f"cuda:{rank}") seq_lens = [16] - top_k = 8 + top_k = expanded_idx_to_permuted_idx.shape[1] eps = 1e-5 launch_with_pdls = [True, False] @@ -99,8 +101,13 @@ def rms_norm(x: torch.Tensor, weight: torch.Tensor = None, eps: float = 1e-6): quant_out = torch.empty( seq_len * hidden_size // 4, dtype=dtype, device=device ) + # E4M3 scales in the SWIZZLED_128x4 layout. scale_out = torch.empty( - seq_len * hidden_size // SF_VEC_SIZE, dtype=dtype, device=device + comm.compute_fp4_swizzled_layout_sf_size( + seq_len, hidden_size // SF_VEC_SIZE + ), + dtype=torch.uint8, + device=device, ) routed_scaling_factor = 2.5 @@ -159,19 +166,22 @@ def rms_norm(x: torch.Tensor, weight: torch.Tensor = None, eps: float = 1e-6): torch.cuda.synchronize() - # == Calculate reference output == + # == Calculate reference output (in FP32, like the kernel) == + local = (expanded_idx_to_permuted_idx >= 0).unsqueeze(-1) expert_reduction = torch.sum( - fc2_output_clone[expanded_idx_to_permuted_idx] - * scale.unsqueeze(-1) + fc2_output_clone.float()[ + expanded_idx_to_permuted_idx.clamp(min=0) + ] + * local + * scale.float().unsqueeze(-1) * routed_scaling_factor, dim=1, ) torch_before_residual = ( - expert_reduction + shared_expert_output + expert_reduction + shared_expert_output.float() ) * world_size - torch_residual = torch_before_residual + residual - torch_residual = torch_residual.to(torch.float32) + torch_residual = torch_before_residual + residual.float() torch_output_hidden_states = rms_norm( torch_residual, norm_weight, eps ).to(dtype) @@ -257,6 +267,86 @@ def rms_norm(x: torch.Tensor, weight: torch.Tensor = None, eps: float = 1e-6): dist.destroy_process_group(group=group) +def _run_back_to_back_worker(world_size, rank, dtype, distributed_init_port): + """Replays many back-to-back launches, so a rank can start the next launch + while its peers are still in the previous one. Exits with code 2 if the + launches do not finish.""" + device = torch.device(f"cuda:{rank}") + torch.cuda.set_device(device) + dist.init_process_group( + backend="nccl", + init_method=f"tcp://localhost:{distributed_init_port}", + rank=rank, + world_size=world_size, + ) + group = dist.group.WORLD + token_num, top_k = kOneShotMaxTokenNum, 8 + _, workspace_tensor = comm.trtllm_create_ipc_workspace_for_all_reduce_fusion( + rank, world_size, MAX_TOKEN_NUM, HIDDEN_SIZE, group=group + ) + torch.manual_seed(0) + fc2_output = torch.randn(token_num * top_k, HIDDEN_SIZE, dtype=dtype, device=device) + expanded_idx_to_permuted_idx = torch.arange( + token_num * top_k, dtype=torch.int32, device=device + ).view(token_num, top_k) + scale = torch.rand(token_num, top_k, dtype=dtype, device=device) + residual = torch.randn(token_num, HIDDEN_SIZE, dtype=dtype, device=device) + norm_weight = torch.ones(HIDDEN_SIZE, dtype=dtype, device=device) + norm_out = torch.empty_like(residual) + residual_out = torch.empty_like(residual) + + def run(): + comm.trtllm_moe_finalize_allreduce_fusion( + allreduce_in=fc2_output, + residual_in=residual, + norm_weight=norm_weight, + expanded_idx_to_permuted_idx=expanded_idx_to_permuted_idx, + workspace_ptrs=workspace_tensor, + launch_with_pdl=True, + world_rank=rank, + world_size=world_size, + eps=1e-5, + shared_expert_output=None, + expert_scale_factor=scale, + norm_out=norm_out, + residual_out=residual_out, + quant_out=None, + scale_out=None, + routed_scaling_factor=1.0, + ) + + s = torch.cuda.Stream() + s.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(s): + run() + torch.cuda.current_stream().wait_stream(s) + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + for _ in range(20): + run() + torch.cuda.synchronize() + dist.barrier(group=group) + for _ in range(50): + g.replay() + done = torch.cuda.Event() + done.record() + deadline = time.monotonic() + 60 + while not done.query(): + if time.monotonic() > deadline: + print(f"Rank {rank}: back-to-back launches did not finish", flush=True) + os._exit(2) + time.sleep(0.01) + + # Every rank has the same inputs. + expected = residual.float() + world_size * ( + fc2_output.float()[expanded_idx_to_permuted_idx.long()] + * scale.float().unsqueeze(-1) + ).sum(dim=1) + torch.testing.assert_close(residual_out.float(), expected, rtol=2e-2, atol=0.5) + dist.barrier(group=group) + dist.destroy_process_group(group=group) + + def get_open_port() -> int: try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: @@ -290,7 +380,9 @@ def multi_process_parallel( @pytest.mark.parametrize("world_size", [2, 4]) @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_trtllm_moe_finalize_allreduce_fusion(world_size, dtype): +# The kernel gathers routes 8 at a time: a partial chunk, one chunk, and two. +@pytest.mark.parametrize("top_k", [6, 8, 10]) +def test_trtllm_moe_finalize_allreduce_fusion(world_size, dtype, top_k): np.random.seed(42) torch.manual_seed(42) torch.cuda.manual_seed_all(42) @@ -304,7 +396,6 @@ def test_trtllm_moe_finalize_allreduce_fusion(world_size, dtype): # generate shared random input tensor across all ranks seq_len = 16 hidden_size = 7168 - top_k = 8 shared_expert_output = torch.randn((seq_len, hidden_size), dtype=dtype) fc2_output = torch.randn((seq_len * top_k, hidden_size), dtype=dtype) @@ -312,6 +403,8 @@ def test_trtllm_moe_finalize_allreduce_fusion(world_size, dtype): expanded_idx_to_permuted_idx = torch.randint( 0, seq_len * top_k, (seq_len, top_k), dtype=torch.int32 ) + # Routes to experts on other ranks are -1. + expanded_idx_to_permuted_idx[torch.rand(seq_len, top_k) < 0.25] = -1 residual = torch.randn_like(shared_expert_output) multi_process_parallel( @@ -329,5 +422,12 @@ def test_trtllm_moe_finalize_allreduce_fusion(world_size, dtype): print(f"moe finalize allreduce fusion tp = {world_size}: OK") +@pytest.mark.parametrize("world_size", [2, 4]) +def test_trtllm_moe_finalize_allreduce_fusion_back_to_back(world_size): + if world_size > torch.cuda.device_count(): + pytest.skip(f"world_size {world_size} is greater than available GPUs") + multi_process_parallel(world_size, torch.bfloat16, _run_back_to_back_worker) + + if __name__ == "__main__": - test_trtllm_moe_finalize_allreduce_fusion(2, torch.float16) + test_trtllm_moe_finalize_allreduce_fusion(2, torch.float16, 8)