Skip to content
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 59 additions & 41 deletions include/flashinfer/comm/trtllm_moe_allreduce_fusion.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -1303,52 +1303,64 @@ __global__ void moefinalize_allreduce_fusion_kernel_oneshot_lamport(
break;
}

// * MoE finalize
vec_t<T, VEC_SIZE> accumulator;
accumulator.fill(0.f);

for (int k = 0; k < top_k; k++) {
int const expanded_idx = token_id * top_k + k;
int32_t const permuted_idx = params.expanded_idx_to_permuted_idx[expanded_idx];

if (permuted_idx == -1) continue;

int thread_offset_across_token =
permuted_idx * params.hidden_dim + thread_offset_within_token;
vec_t<T, VEC_SIZE> permuted_data;
permuted_data.load(reinterpret_cast<T*>(params.allreduce_in) + thread_offset_across_token);
// * MoE finalize, accumulated in FP32.
float acc[VEC_SIZE];
#pragma unroll
for (int i = 0; i < VEC_SIZE; ++i) {
acc[i] = 0.f;
}
// The shared expert row does not depend on the routing; load it first.
vec_t<T, VEC_SIZE> shared_expert_output;
if (params.shared_expert_output) {
shared_expert_output.load(reinterpret_cast<T*>(params.shared_expert_output) +
token_id * params.hidden_dim + thread_offset_within_token);
}

// * acc += scale(data)
if (use_scale_factor) {
float block_scale =
static_cast<float>(static_cast<ScaleType*>(params.expert_scale_factor)[expanded_idx]);
// Experts are gathered kTopKChunk at a time: all permute indices and scales first, then
// all rows, so their loads are in flight together instead of one dependent pair per expert.
constexpr int kTopKChunk = 8;
for (int k0 = 0; k0 < top_k; k0 += kTopKChunk) {
int32_t permuted_idx[kTopKChunk];
float block_scale[kTopKChunk];
#pragma unroll
for (int i = 0; i < VEC_SIZE; ++i) {
// assume computation is done in ScaleType
accumulator[i] += static_cast<T>(static_cast<float>(permuted_data[i]) * block_scale);
for (int j = 0; j < kTopKChunk; ++j) {
int const k = k0 + j;
int const expanded_idx = token_id * top_k + k;
Comment thread
haosdent marked this conversation as resolved.
Resolved
permuted_idx[j] = k < top_k ? params.expanded_idx_to_permuted_idx[expanded_idx] : -1;
block_scale[j] = k < top_k && use_scale_factor
? static_cast<float>(static_cast<ScaleType*>(
params.expert_scale_factor)[expanded_idx])
: 1.f;
}
vec_t<T, VEC_SIZE> permuted_data[kTopKChunk];
#pragma unroll
for (int j = 0; j < kTopKChunk; ++j) {
if (permuted_idx[j] != -1) {
permuted_data[j].load(reinterpret_cast<T*>(params.allreduce_in) +
permuted_idx[j] * params.hidden_dim + thread_offset_within_token);
}
} else {
accumulator = vec_add<T, VEC_SIZE>(accumulator, permuted_data);
}
}

// Apply the global routed scaling once after accumulating all routed experts.
if (use_routed_scaling_factor) {
#pragma unroll
for (int i = 0; i < VEC_SIZE; ++i) {
accumulator[i] = static_cast<T>(static_cast<float>(accumulator[i]) * routed_scaling_factor);
for (int j = 0; j < kTopKChunk; ++j) {
if (permuted_idx[j] != -1) {
#pragma unroll
for (int i = 0; i < VEC_SIZE; ++i) {
acc[i] += static_cast<float>(permuted_data[j][i]) * block_scale[j];
}
}
}
}

// * Add shared expert output
if (params.shared_expert_output) {
// * Load shared expert output
int thread_offset_across_token = token_id * params.hidden_dim + thread_offset_within_token;
vec_t<T, VEC_SIZE> shared_expert_output;
shared_expert_output.load(reinterpret_cast<T*>(params.shared_expert_output) +
thread_offset_across_token);
// Apply the global routed scaling once after accumulating all routed experts, then add the
// shared expert output.
vec_t<T, VEC_SIZE> accumulator;
#pragma unroll
accumulator = vec_add<T, VEC_SIZE>(accumulator, shared_expert_output);
for (int i = 0; i < VEC_SIZE; ++i) {
float v = use_routed_scaling_factor ? acc[i] * routed_scaling_factor : acc[i];
if (params.shared_expert_output) {
v += static_cast<float>(shared_expert_output[i]);
}
accumulator[i] = static_cast<T>(v);
}

// * AR Store
Expand Down Expand Up @@ -1379,15 +1391,21 @@ __global__ void moefinalize_allreduce_fusion_kernel_oneshot_lamport(
#pragma unroll
for (int r = 0; r < NRanks; ++r) {
// LDG.128 from local rank
vals[r].load_global_volatile(reinterpret_cast<T*>(comm.data_bufs[r]) +
vals[r].load_global_volatile(reinterpret_cast<T*>(comm.data_bufs[params.rank]) +
(r * tot_access + idx) * VEC_SIZE);
done &= !has_neg_zero<T, VEC_SIZE>(vals[r]);
}
}
vec_t<T, VEC_SIZE> sum_val = vals[0];
// Sum the ranks in FP32.
vec_t<T, VEC_SIZE> sum_val;
#pragma unroll
for (int r = 1; r < NRanks; ++r) {
sum_val = vec_add<T, VEC_SIZE>(sum_val, vals[r]);
for (int i = 0; i < VEC_SIZE; ++i) {
float sum = static_cast<float>(vals[0][i]);
#pragma unroll
for (int r = 1; r < NRanks; ++r) {
sum += static_cast<float>(vals[r][i]);
}
sum_val[i] = static_cast<T>(sum);
}

// * Fuse: AllReduceOut is always false in finalize_moe_allreduce
Expand Down
107 changes: 100 additions & 7 deletions tests/comm/test_trtllm_moe_allreduce_fusion_finalize.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import multiprocessing as mp
import os
import socket
import time
from typing import Any

import numpy as np
Expand All @@ -8,6 +10,7 @@
import torch.distributed as dist

import flashinfer.comm as comm
from flashinfer.utils import round_up

# todo(Yingyi): add benchmark and quant test

Expand Down Expand Up @@ -99,8 +102,12 @@ def rms_norm(x: torch.Tensor, weight: torch.Tensor = None, eps: float = 1e-6):
quant_out = torch.empty(
seq_len * hidden_size // 4, dtype=dtype, device=device
)
# E4M3 scales in the SWIZZLED_128x4 layout.
scale_out = torch.empty(
seq_len * hidden_size // SF_VEC_SIZE, dtype=dtype, device=device
round_up(seq_len, 128)
* round_up(hidden_size // SF_VEC_SIZE, 4),
dtype=torch.uint8,
device=device,
)
routed_scaling_factor = 2.5

Expand Down Expand Up @@ -159,19 +166,18 @@ def rms_norm(x: torch.Tensor, weight: torch.Tensor = None, eps: float = 1e-6):

torch.cuda.synchronize()

# == Calculate reference output ==
# == Calculate reference output (in FP32, like the kernel) ==
expert_reduction = torch.sum(
fc2_output_clone[expanded_idx_to_permuted_idx]
* scale.unsqueeze(-1)
fc2_output_clone.float()[expanded_idx_to_permuted_idx]
* scale.float().unsqueeze(-1)
* routed_scaling_factor,
dim=1,
)

torch_before_residual = (
expert_reduction + shared_expert_output
expert_reduction + shared_expert_output.float()
) * world_size
torch_residual = torch_before_residual + residual
torch_residual = torch_residual.to(torch.float32)
torch_residual = torch_before_residual + residual.float()
torch_output_hidden_states = rms_norm(
torch_residual, norm_weight, eps
).to(dtype)
Expand Down Expand Up @@ -257,6 +263,86 @@ def rms_norm(x: torch.Tensor, weight: torch.Tensor = None, eps: float = 1e-6):
dist.destroy_process_group(group=group)


def _run_back_to_back_worker(world_size, rank, dtype, distributed_init_port):
"""Replays many back-to-back launches, so a rank can start the next launch
while its peers are still in the previous one. Exits with code 2 if the
launches do not finish."""
device = torch.device(f"cuda:{rank}")
torch.cuda.set_device(device)
dist.init_process_group(
backend="nccl",
init_method=f"tcp://localhost:{distributed_init_port}",
rank=rank,
world_size=world_size,
)
group = dist.group.WORLD
token_num, top_k = kOneShotMaxTokenNum, 8
_, workspace_tensor = comm.trtllm_create_ipc_workspace_for_all_reduce_fusion(
rank, world_size, MAX_TOKEN_NUM, HIDDEN_SIZE, group=group
)
torch.manual_seed(0)
fc2_output = torch.randn(token_num * top_k, HIDDEN_SIZE, dtype=dtype, device=device)
expanded_idx_to_permuted_idx = torch.arange(
token_num * top_k, dtype=torch.int32, device=device
).view(token_num, top_k)
scale = torch.rand(token_num, top_k, dtype=dtype, device=device)
residual = torch.randn(token_num, HIDDEN_SIZE, dtype=dtype, device=device)
norm_weight = torch.ones(HIDDEN_SIZE, dtype=dtype, device=device)
norm_out = torch.empty_like(residual)
residual_out = torch.empty_like(residual)

def run():
comm.trtllm_moe_finalize_allreduce_fusion(
allreduce_in=fc2_output,
residual_in=residual,
norm_weight=norm_weight,
expanded_idx_to_permuted_idx=expanded_idx_to_permuted_idx,
workspace_ptrs=workspace_tensor,
launch_with_pdl=True,
world_rank=rank,
world_size=world_size,
eps=1e-5,
shared_expert_output=None,
expert_scale_factor=scale,
norm_out=norm_out,
residual_out=residual_out,
quant_out=None,
scale_out=None,
routed_scaling_factor=1.0,
)

s = torch.cuda.Stream()
s.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(s):
run()
torch.cuda.current_stream().wait_stream(s)
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
for _ in range(20):
run()
torch.cuda.synchronize()
dist.barrier(group=group)
for _ in range(50):
g.replay()
done = torch.cuda.Event()
done.record()
deadline = time.monotonic() + 60
while not done.query():
if time.monotonic() > deadline:
print(f"Rank {rank}: back-to-back launches did not finish", flush=True)
os._exit(2)
time.sleep(0.01)

# Every rank has the same inputs.
expected = residual.float() + world_size * (
fc2_output.float()[expanded_idx_to_permuted_idx.long()]
* scale.float().unsqueeze(-1)
).sum(dim=1)
torch.testing.assert_close(residual_out.float(), expected, rtol=2e-2, atol=0.5)
dist.barrier(group=group)
dist.destroy_process_group(group=group)


def get_open_port() -> int:
try:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
Expand Down Expand Up @@ -329,5 +415,12 @@ def test_trtllm_moe_finalize_allreduce_fusion(world_size, dtype):
print(f"moe finalize allreduce fusion tp = {world_size}: OK")


@pytest.mark.parametrize("world_size", [2, 4])
def test_trtllm_moe_finalize_allreduce_fusion_back_to_back(world_size):
if world_size > torch.cuda.device_count():
pytest.skip(f"world_size {world_size} is greater than available GPUs")
multi_process_parallel(world_size, torch.bfloat16, _run_back_to_back_worker)


if __name__ == "__main__":
test_trtllm_moe_finalize_allreduce_fusion(2, torch.float16)
Loading