Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion .buildkite/test_areas/distributed.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ steps:

- label: Distributed Tests (8xH100)
key: distributed-tests-8xh100
timeout_in_minutes: 20
timeout_in_minutes: 30
device: h100
num_devices: 8
working_dir: "/vllm-workspace/tests"
Expand All @@ -194,13 +194,19 @@ steps:
- vllm/v1/engine/llm_engine.py
- vllm/v1/executor/uniproc_executor.py
- vllm/v1/worker/gpu_worker.py
- vllm/v1/attention/ops/dcp_utils.py
- vllm/model_executor/layers/attention/mla_attention.py
- vllm/model_executor/layers/attention/sparse_mla_attention.py
- csrc/libtorch_stable/attention/dcp_utils/
- tests/distributed/test_dcp_direct_a2a_lse_reduce.py
- tests/distributed/test_mnnvl_alltoall.py

commands:
# https://github.com/NVIDIA/nccl/issues/1838
- export NCCL_CUMEM_HOST_ENABLE=0
# test with torchrun tp=2 and dp=4 with ep
- torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
- pytest -v -s distributed/test_dcp_direct_a2a_lse_reduce.py

- label: Distributed Tests (4xA100)
key: distributed-tests-4xa100
Expand Down
171 changes: 93 additions & 78 deletions csrc/libtorch_stable/attention/dcp_utils/dcp_direct_kv_gather.cu
Original file line number Diff line number Diff line change
Expand Up @@ -20,28 +20,53 @@ using vllm::direct_dcp::wait_for_epoch;
constexpr int kThreads = 256;
// KV chunks need many blocks in flight to saturate the fabric.
constexpr int64_t kMaxMulticastBlocks = 128;
constexpr int64_t kMaxCopyBlocks = 1024;

// Multicast each rank's KV slice and completion epoch to every replica.
// Multicast each rank's valid local rows directly into compact, request-major
// kv_c and k_pe planes. dst_rows maps every padded local input row to its final
// output row, or -1 for padding. Destination rows are disjoint across source
// ranks, so all ranks can publish concurrently without atomics.
__global__ void direct_dcp_kv_gather_multimem_kernel(
const uint4* local_kv, uint4* mc_kv, uint32_t* mc_signal,
const uint32_t* received_signal, const int64_t* epoch_ptr,
uint32_t* completion, int64_t world_size, int64_t rank, int64_t slice_bytes,
int64_t slot_stride_bytes) {
const uint4* local_kv, const int32_t* dst_rows, uint4* mc_kv,
uint32_t* mc_signal, const uint32_t* received_signal,
const int64_t* epoch_ptr, uint32_t* completion, int64_t world_size,
int64_t rank, int64_t num_tokens, int64_t items_per_row,
int64_t kv_c_items_per_row, int64_t output_tokens,
int64_t max_gathered_tokens, int64_t buffer_slot,
int64_t slot_stride_items) {
uint32_t epoch = static_cast<uint32_t>(epoch_ptr[0]);
int64_t buffer_slot = static_cast<int64_t>(epoch & 1u);
int64_t item_count = slice_bytes / sizeof(uint4);
mc_kv += buffer_slot * slot_stride_bytes / sizeof(uint4);
mc_kv += rank * item_count;
mc_kv += buffer_slot * slot_stride_items;

int64_t item_count = num_tokens * items_per_row;
int64_t item_stride = static_cast<int64_t>(gridDim.x) * blockDim.x;
for (int64_t item =
static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x;
item < item_count; item += item_stride) {
multimem_store_16(mc_kv + item, local_kv[item]);
int64_t src_row = item / items_per_row;
int32_t dst_row = dst_rows[src_row];
if (dst_row < 0) {
continue;
}
if (dst_row >= output_tokens) {
printf(
"direct DCP final-layout kv-gather destination out of bounds "
"source=%lld dst=%d output_tokens=%lld\n",
static_cast<long long>(rank), dst_row,
static_cast<long long>(output_tokens));
asm volatile("trap;");
}
int64_t row_item = item - src_row * items_per_row;
int64_t dst_item;
if (row_item < kv_c_items_per_row) {
dst_item = static_cast<int64_t>(dst_row) * kv_c_items_per_row + row_item;
} else {
int64_t k_pe_items_per_row = items_per_row - kv_c_items_per_row;
dst_item = max_gathered_tokens * kv_c_items_per_row +
static_cast<int64_t>(dst_row) * k_pe_items_per_row + row_item -
kv_c_items_per_row;
}
multimem_store_16(mc_kv + dst_item, local_kv[item]);
}

// Publish all multicast writes before incrementing completion.
__threadfence_system();
__syncthreads();
if (threadIdx.x != 0) {
Expand All @@ -56,59 +81,57 @@ __global__ void direct_dcp_kv_gather_multimem_kernel(

multimem_store_release_system(mc_signal + buffer_slot * world_size + rank,
epoch);

for (int64_t source_rank = 0; source_rank < world_size; ++source_rank) {
int64_t signal_item = buffer_slot * world_size + source_rank;
if (!wait_for_epoch(received_signal + signal_item, epoch)) {
printf("direct DCP kv-gather multimem timeout source=%lld epoch=%u\n",
printf("direct DCP final-layout kv-gather timeout source=%lld epoch=%u\n",
static_cast<long long>(source_rank), epoch);
asm volatile("trap;");
}
}
}

// Materialize the acquired slot into the caller's workspace.
template <typename copy_t>
__global__ void materialize_kv_gather_kernel(const copy_t* received_kv,
const int64_t* epoch_ptr,
copy_t* gathered_kv,
int64_t gathered_item_count,
int64_t slot_stride_items) {
uint32_t epoch = static_cast<uint32_t>(epoch_ptr[0]);
int64_t buffer_slot = static_cast<int64_t>(epoch & 1u);
received_kv += buffer_slot * slot_stride_items;
int64_t item = static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x;
int64_t item_stride = static_cast<int64_t>(gridDim.x) * blockDim.x;
for (; item < gathered_item_count; item += item_stride) {
gathered_kv[item] = received_kv[item];
}
}

void direct_dcp_kv_gather(
const torch::stable::Tensor& local_kv, torch::stable::Tensor& received_kv,
torch::stable::Tensor& received_signal, torch::stable::Tensor& completion,
torch::stable::Tensor& epoch, torch::stable::Tensor& gathered_kv,
int64_t world_size, int64_t rank, int64_t max_gathered_tokens,
int64_t kv_mc_ptr, int64_t signal_mc_ptr) {
void direct_dcp_kv_gather(const torch::stable::Tensor& local_kv,
const torch::stable::Tensor& dst_rows,
torch::stable::Tensor& received_kv,
torch::stable::Tensor& received_signal,
torch::stable::Tensor& completion,
torch::stable::Tensor& epoch, int64_t output_tokens,
int64_t plane_split_dim, int64_t buffer_slot,
int64_t world_size, int64_t rank,
int64_t max_gathered_tokens, int64_t kv_mc_ptr,
int64_t signal_mc_ptr) {
using torch::headeronly::ScalarType;

STD_TORCH_CHECK(local_kv.is_cuda(), "local kv must be a CUDA tensor");
ScalarType dtype = local_kv.scalar_type();
STD_TORCH_CHECK(dtype == ScalarType::Half || dtype == ScalarType::BFloat16 ||
dtype == ScalarType::Float8_e4m3fn,
"direct DCP kv-gather only supports FP16, BF16, and FP8");
"direct DCP final-layout kv-gather only supports FP16, "
"BF16, and FP8");
STD_TORCH_CHECK(local_kv.dim() == 2 && local_kv.is_contiguous(),
"local kv must be a contiguous [T,D] tensor");
STD_TORCH_CHECK(dst_rows.is_cuda() && dst_rows.is_contiguous() &&
dst_rows.scalar_type() == ScalarType::Int &&
dst_rows.dim() == 1 &&
dst_rows.numel() == local_kv.size(0),
"final-layout destination rows must be CUDA int32 [T]");
STD_TORCH_CHECK(world_size > 1, "world_size must be greater than 1");
STD_TORCH_CHECK(rank >= 0 && rank < world_size, "invalid rank");
STD_TORCH_CHECK(buffer_slot == 0 || buffer_slot == 1,
"final-layout buffer slot must be 0 or 1");
STD_TORCH_CHECK(output_tokens > 0 && output_tokens <= max_gathered_tokens,
"final-layout output exceeds symmetric buffer capacity");

int64_t num_tokens = local_kv.size(0);
int64_t token_dim = local_kv.size(1);
int64_t element_size = local_kv.element_size();
STD_TORCH_CHECK(num_tokens > 0 && token_dim > 0,
"local kv dimensions must be positive");
STD_TORCH_CHECK(plane_split_dim > 0 && plane_split_dim < token_dim,
"final-layout plane split must be within the token row");
STD_TORCH_CHECK(num_tokens * world_size <= max_gathered_tokens,
"gathered kv exceeds symmetric kv-gather buffer capacity");
"padded gathered kv exceeds symmetric buffer capacity");

STD_TORCH_CHECK(received_kv.is_cuda() && received_kv.scalar_type() == dtype &&
received_kv.is_contiguous() && received_kv.dim() == 3 &&
Expand All @@ -130,75 +153,67 @@ void direct_dcp_kv_gather(
epoch.scalar_type() == ScalarType::Long &&
epoch.numel() == 1,
"epoch must be a one-element CUDA int64 tensor");
STD_TORCH_CHECK(gathered_kv.is_cuda() && gathered_kv.scalar_type() == dtype &&
gathered_kv.is_contiguous() && gathered_kv.dim() == 2 &&
gathered_kv.size(0) == num_tokens * world_size &&
gathered_kv.size(1) == token_dim,
"gathered kv must be contiguous with shape [world_size*T,D]");

int64_t device_index = local_kv.get_device_index();
STD_TORCH_CHECK(
received_kv.get_device_index() == device_index &&
received_signal.get_device_index() == device_index &&
completion.get_device_index() == device_index &&
epoch.get_device_index() == device_index &&
gathered_kv.get_device_index() == device_index,
"direct DCP kv-gather tensors must be on the same CUDA device");

int64_t slice_bytes = num_tokens * token_dim * element_size;
int64_t slot_stride_bytes = max_gathered_tokens * token_dim * element_size;
STD_TORCH_CHECK(dst_rows.get_device_index() == device_index &&
received_kv.get_device_index() == device_index &&
received_signal.get_device_index() == device_index &&
completion.get_device_index() == device_index &&
epoch.get_device_index() == device_index,
"direct DCP final-layout tensors must share a CUDA device");

int64_t row_bytes = token_dim * element_size;
int64_t kv_c_row_bytes = plane_split_dim * element_size;
int64_t k_pe_row_bytes = row_bytes - kv_c_row_bytes;
int64_t slot_stride_bytes = max_gathered_tokens * row_bytes;
bool vectorized =
reinterpret_cast<uintptr_t>(local_kv.data_ptr()) % alignof(uint4) == 0 &&
reinterpret_cast<uintptr_t>(received_kv.data_ptr()) % alignof(uint4) ==
0 &&
reinterpret_cast<uintptr_t>(gathered_kv.data_ptr()) % alignof(uint4) ==
0 &&
slice_bytes % sizeof(uint4) == 0 &&
row_bytes % sizeof(uint4) == 0 && kv_c_row_bytes % sizeof(uint4) == 0 &&
k_pe_row_bytes % sizeof(uint4) == 0 &&
slot_stride_bytes % sizeof(uint4) == 0;
STD_TORCH_CHECK(
vectorized,
"direct DCP kv-gather requires 16-byte-aligned pointers and strides");
STD_TORCH_CHECK(vectorized,
"direct DCP final-layout kv-gather requires 16-byte-aligned "
"planes, rows, and pointers");
STD_TORCH_CHECK(kv_mc_ptr != 0 && signal_mc_ptr != 0,
"direct DCP kv-gather requires multicast pointers");
"direct DCP final-layout kv-gather requires multicast "
"pointers");

const torch::stable::accelerator::DeviceGuard device_guard(device_index);
cudaStream_t stream = get_current_cuda_stream();
increment_epoch_kernel<<<1, 1, 0, stream>>>(
epoch.mutable_data_ptr<int64_t>());
check_cuda_launch("direct DCP kv-gather");
check_cuda_launch("direct DCP final-layout kv-gather");

int64_t item_count = slice_bytes / sizeof(uint4);
int64_t items_per_row = row_bytes / sizeof(uint4);
int64_t kv_c_items_per_row = kv_c_row_bytes / sizeof(uint4);
int64_t item_count = num_tokens * items_per_row;
int64_t blocks = (item_count + kThreads - 1) / kThreads;
blocks = blocks < kMaxMulticastBlocks ? blocks : kMaxMulticastBlocks;
direct_dcp_kv_gather_multimem_kernel<<<blocks, kThreads, 0, stream>>>(
reinterpret_cast<const uint4*>(local_kv.data_ptr()),
dst_rows.const_data_ptr<int32_t>(),
reinterpret_cast<uint4*>(static_cast<uintptr_t>(kv_mc_ptr)),
reinterpret_cast<uint32_t*>(static_cast<uintptr_t>(signal_mc_ptr)),
reinterpret_cast<const uint32_t*>(
received_signal.const_data_ptr<int32_t>()),
epoch.const_data_ptr<int64_t>(),
reinterpret_cast<uint32_t*>(completion.mutable_data_ptr<int32_t>()),
world_size, rank, slice_bytes, slot_stride_bytes);
check_cuda_launch("direct DCP kv-gather");

int64_t gathered_item_count = world_size * item_count;
int64_t copy_blocks = (gathered_item_count + kThreads - 1) / kThreads;
copy_blocks = copy_blocks < kMaxCopyBlocks ? copy_blocks : kMaxCopyBlocks;
materialize_kv_gather_kernel<uint4><<<copy_blocks, kThreads, 0, stream>>>(
reinterpret_cast<const uint4*>(received_kv.data_ptr()),
epoch.const_data_ptr<int64_t>(),
reinterpret_cast<uint4*>(gathered_kv.mutable_data_ptr()),
gathered_item_count, slot_stride_bytes / sizeof(uint4));
check_cuda_launch("direct DCP kv-gather");
world_size, rank, num_tokens, items_per_row, kv_c_items_per_row,
output_tokens, max_gathered_tokens, buffer_slot,
slot_stride_bytes / sizeof(uint4));
check_cuda_launch("direct DCP final-layout kv-gather");
}

} // namespace

STABLE_TORCH_LIBRARY_FRAGMENT(_C, direct_dcp_kv_gather_ops) {
direct_dcp_kv_gather_ops.def(
"direct_dcp_kv_gather("
"Tensor local_kv, Tensor! received_kv, Tensor! received_signal, "
"Tensor! completion, "
"Tensor! epoch, Tensor! gathered_kv, "
"Tensor local_kv, Tensor dst_rows, Tensor! received_kv, "
"Tensor! received_signal, Tensor! completion, Tensor! epoch, "
"int output_tokens, int plane_split_dim, int buffer_slot, "
"int world_size, int rank, int max_gathered_tokens, "
"int kv_mc_ptr, int signal_mc_ptr) -> ()");
}
Expand Down
Loading
Loading