Skip to content
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions cpp/include/rapidsmpf/utils/misc.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@
*/
#pragma once

#include <algorithm>
#include <chrono>
#include <cmath>
#include <concepts>
#include <cstdlib>
#include <functional>
#include <memory>
Expand Down Expand Up @@ -193,6 +195,50 @@ constexpr T safe_div(T x, T y) {
return (y == 0) ? 0 : x / y;
}

/**
* @brief Computes the ceiling of the division of two integers.
*
* Returns the smallest integer not less than `x / y`. Both operands must be
* non-negative and the denominator must be non-zero. Computed as `x / y + (x % y != 0)`
* to avoid the overflow (and signed UB)
*
* @tparam T An integral type.
* @param x The numerator (must be non-negative).
* @param y The denominator (must be positive).
* @return T The ceiling of `x / y`.
*/
template <std::integral T>
constexpr T ceil_div(T x, T y) {
return x / y + (x % y != 0 ? T{1} : T{0});
}

/**
* @brief Splits the index range `[0, count)` into exactly `num_chunks` contiguous chunks.
*
* Each chunk is a half-open `[begin, end)` index pair. Chunks are front-loaded with
* size `ceil(count / num_chunks)`; the last non-empty chunk may be smaller and, when
* `count < num_chunks`, the trailing chunks are empty (`begin == end`). The chunks
* exactly tile `[0, count)`, so their sizes sum to `count`.
*
* Unlike `std::ranges::chunk_view` (C++23), this always yields exactly `num_chunks`
* chunks, injecting empty trailing chunks as needed.
*
* @param count The number of elements to split.
* @param num_chunks The number of chunks to produce (must be positive).
* @return A lazy view of `num_chunks` `std::pair<std::size_t, std::size_t>` `[begin,
* end)` index pairs.
*/
[[nodiscard]] inline auto chunk_indices(std::size_t count, std::size_t num_chunks) {
std::size_t const chunk_size = ceil_div(count, num_chunks);
return std::views::iota(std::size_t{0}, num_chunks)
| std::views::transform([count, chunk_size](std::size_t k) {
return std::pair<std::size_t, std::size_t>{
std::min(k * chunk_size, count),
std::min((k + 1) * chunk_size, count)
};
});
}

// Macro to concatenate two tokens x and y.
#define RAPIDSMPF_CONCAT_DETAIL_(x, y) x##y
#define RAPIDSMPF_CONCAT(x, y) RAPIDSMPF_CONCAT_DETAIL_(x, y)
Expand Down
14 changes: 6 additions & 8 deletions cpp/tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ target_sources(
test_pausable_thread_loop.cpp
test_progress_thread.cpp
test_rmm_resource_adaptor.cpp
test_shuffler.cpp
test_sparse_alltoall.cpp
test_spill_manager.cpp
test_spilling.cpp
Expand All @@ -99,7 +100,8 @@ target_sources(
if(RAPIDSMPF_HAVE_STREAMING)
target_sources(
test_sources
PRIVATE streaming/test_allreduce.cpp
PRIVATE streaming/test_allgather.cpp
streaming/test_allreduce.cpp
streaming/test_channel.cpp
streaming/test_error_handling.cpp
streaming/test_fanout.cpp
Expand All @@ -108,24 +110,20 @@ if(RAPIDSMPF_HAVE_STREAMING)
streaming/test_message.cpp
streaming/test_sparse_alltoall.cpp
streaming/test_spillable_messages.cpp
streaming/test_shuffler.cpp
)
endif()

# cudf-dependent test sources, gated behind BUILD_CUDF_TESTS
if(BUILD_CUDF_TESTS)
target_sources(
test_sources PRIVATE test_partition.cpp test_shuffler.cpp test_shuffler_many_streams.cpp
)
target_sources(test_sources PRIVATE test_partition.cpp test_shuffler_many_streams.cpp)
target_link_libraries(
test_sources PRIVATE cudf_streaming::cudf_streaming cudf::cudftestutil cudf::cudftestutil_impl
)
target_compile_definitions(test_sources PRIVATE RAPIDSMPF_HAVE_CUDF)

if(RAPIDSMPF_HAVE_STREAMING)
target_sources(
test_sources PRIVATE streaming/test_allgather.cpp streaming/test_leaf_actor.cpp
streaming/test_shuffler.cpp
)
target_sources(test_sources PRIVATE streaming/test_leaf_actor.cpp)
endif()
endif()

Expand Down
2 changes: 0 additions & 2 deletions cpp/tests/streaming/test_allgather.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@
#include <gmock/gmock.h>
#include <gtest/gtest.h>

#include <cudf_streaming/integrations/partition.hpp>

#include <coro/latch.hpp>

#include <rapidsmpf/communicator/single.hpp>
Expand Down
221 changes: 110 additions & 111 deletions cpp/tests/streaming/test_shuffler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,17 @@
* SPDX-License-Identifier: Apache-2.0
*/

#include <algorithm>
#include <ranges>

#include <gmock/gmock.h>
#include <gtest/gtest.h>

#include <cudf/copying.hpp>
#include <cudf_streaming/integrations/partition.hpp>
#include <cudf_streaming/streaming/partition.hpp>
#include <cudf_streaming/streaming/table_chunk.hpp>
#include <cudf_test/table_utilities.hpp>

#include <rapidsmpf/communicator/single.hpp>
#include <rapidsmpf/cuda_stream.hpp>
#include <rapidsmpf/memory/buffer.hpp>
#include <rapidsmpf/shuffler/shuffler.hpp>
#include <rapidsmpf/streaming/chunks/partition.hpp>
#include <rapidsmpf/streaming/coll/shuffler.hpp>
#include <rapidsmpf/streaming/core/actor.hpp>
#include <rapidsmpf/streaming/core/context.hpp>
Expand Down Expand Up @@ -55,13 +54,12 @@ TEST_F(BaseStreamingShuffle, zero_owned_partitions_completes) {
class StreamingShuffler : public BaseStreamingShuffle,
public ::testing::WithParamInterface<int> {
public:
const unsigned int num_partitions = 10;
const unsigned int num_rows = 1000;
const unsigned int num_chunks = 5;
const unsigned int chunk_size = num_rows / num_chunks;
const std::int64_t seed = 42;
const cudf::hash_id hash_function = cudf::hash_id::HASH_MURMUR3;
const OpID op_id = 0;
static constexpr size_t num_partitions = 10;
static constexpr size_t num_rows = 1000;
static constexpr size_t num_chunks = 5;
static constexpr OpID op_id = 0;

shuffler::Shuffler::PartitionOwner partition_owner = shuffler::Shuffler::round_robin;

void SetUp() override {
BaseStreamingShuffle::SetUpWithThreads(GetParam());
Expand All @@ -70,100 +68,6 @@ class StreamingShuffler : public BaseStreamingShuffle,
void TearDown() override {
BaseStreamingShuffle::TearDown();
}

void run_test(auto make_shuffler_actor_fn) {
// Create the full input table and slice it into chunks.
cudf::table full_input_table = random_table_with_index(seed, num_rows, 0, 10);
std::vector<Message> input_chunks;
for (unsigned int i = 0; i < num_chunks; ++i) {
input_chunks.emplace_back(
cudf_streaming::streaming::to_message(
i,
std::make_unique<cudf_streaming::streaming::TableChunk>(
std::make_unique<cudf::table>(
cudf::slice(
full_input_table,
{static_cast<cudf::size_type>(i * chunk_size),
static_cast<cudf::size_type>((i + 1) * chunk_size)},
stream
)
.at(0),
stream,
ctx->br()->device_mr()
),
stream
)
)
);
}

// Create and run the streaming pipeline.
std::vector<Message> output_chunks;
{
std::vector<Actor> actors;
auto ch1 = ctx->create_channel();
actors.push_back(actor::push_to_channel(ctx, ch1, std::move(input_chunks)));

auto ch2 = ctx->create_channel();
actors.push_back(
cudf_streaming::streaming::actor::partition_and_pack(
ctx, ch1, ch2, {1}, num_partitions, hash_function, seed
)
);

auto ch3 = ctx->create_channel();
actors.emplace_back(make_shuffler_actor_fn(ch2, ch3));

auto ch4 = ctx->create_channel();
actors.push_back(
cudf_streaming::streaming::actor::unpack_and_concat(ctx, ch3, ch4)
);

actors.push_back(actor::pull_from_channel(ctx, ch4, output_chunks));

run_actor_network(std::move(actors));
}

auto comm = GlobalEnvironment->comm_;
std::unique_ptr<cudf::table> expected_table;
if (comm->nranks() == 1) { // full_input table is expected
expected_table = std::make_unique<cudf::table>(std::move(full_input_table));
} else { // full_input table is replicated on all ranks
// local partitions
auto [table, offsets] = cudf::hash_partition(
full_input_table.view(), {1}, num_partitions, hash_function, seed
);

auto local_pids = shuffler::Shuffler::local_partitions(
comm, num_partitions, shuffler::Shuffler::round_robin
);

// every partition is replicated on all ranks
std::vector<cudf::table_view> expected_tables;
for (auto pid : local_pids) {
auto t_view =
cudf::slice(table->view(), {offsets[pid], offsets[pid + 1]}).at(0);
// this will be replicated on all ranks
for (rapidsmpf::Rank rank = 0; rank < comm->nranks(); ++rank) {
expected_tables.push_back(t_view);
}
}
expected_table = cudf::concatenate(expected_tables);
}

// Concat all output chunks to a single table.
std::vector<cudf::table_view> output_chunks_as_views;
for (auto& chunk : output_chunks) {
output_chunks_as_views.push_back(
chunk.get<cudf_streaming::streaming::TableChunk>().table_view()
);
}
auto result_table = cudf::concatenate(output_chunks_as_views);

CUDF_TEST_EXPECT_TABLES_EQUIVALENT(
sort_table(result_table->view()), sort_table(expected_table->view())
);
}
};

INSTANTIATE_TEST_SUITE_P(
Expand All @@ -175,12 +79,107 @@ INSTANTIATE_TEST_SUITE_P(
}
);

// Verifies end-to-end correctness of the streaming shuffler actor.
//
// Each rank sends num_chunks messages, where each message is a PartitionMapChunk covering
// all num_partitions partitions. The data for partition j in chunk c_idx on rank r is a
// contiguous integer sequence whose values encode both the rank and the row range, making
// them globally unique and independently verifiable.
//
// After shuffling, each local partition should have received exactly num_chunks * nranks
// packed-data items (one per (rank, chunk) pair). The items are sorted by their first
// element — which equals rank * num_rows + row_start — to reconstruct rank-major,
// chunk-minor order, and then validated against the expected row range from `pieces`.
TEST_P(StreamingShuffler, basic_shuffler) {
EXPECT_NO_FATAL_FAILURE(run_test([&](auto ch_in, auto ch_out) -> Actor {
return actor::shuffler(
ctx, GlobalEnvironment->comm_, ch_in, ch_out, op_id, num_partitions
auto comm = GlobalEnvironment->comm_;
// split a span [0, num_rows) into num_partitions * num_chunks contiguous pieces.
auto const pieces = rapidsmpf::chunk_indices(num_rows, num_partitions * num_chunks);

// each rank contributes num_chunks messages, each covering num_partitions pieces.
const int64_t base = static_cast<int64_t>(comm->rank()) * num_rows;
std::vector<Message> input_chunks; // Message contains a PartitionMapChunk
for (size_t chunk_idx = 0; chunk_idx < num_chunks; ++chunk_idx) {
std::unordered_map<shuffler::PartID, PackedData> chunks;
chunks.reserve(num_partitions);
for (size_t j = 0; j < num_partitions; ++j) {
auto [start, end] = pieces[chunk_idx * num_partitions + j];
// end > start is guaranteed.
chunks.emplace(
static_cast<shuffler::PartID>(j),
generate_packed_data<int64_t>(
end - start, base + static_cast<int64_t>(start), stream, *br
)
);
}
input_chunks.emplace_back(
to_message(chunk_idx, std::make_unique<PartitionMapChunk>(std::move(chunks)))
);
}
EXPECT_EQ(input_chunks.size(), num_chunks);

// Create and run the streaming pipeline.
std::vector<Message> output_chunks;
{
std::vector<Actor> actors;
auto ch1 = ctx->create_channel();
actors.push_back(actor::push_to_channel(ctx, ch1, std::move(input_chunks)));

auto ch2 = ctx->create_channel();
actors.emplace_back(
actor::shuffler(ctx, comm, ch1, ch2, op_id, num_partitions, partition_owner)
);
}));

actors.push_back(actor::pull_from_channel(ctx, ch2, output_chunks));

run_actor_network(std::move(actors));
}

auto local_pids =
shuffler::Shuffler::local_partitions(comm, num_partitions, partition_owner);

// Since all partitions are non-empty, each local partition ID should a corresponding
// output chunk.
EXPECT_EQ(local_pids.size(), output_chunks.size());
const size_t n_ranks = static_cast<size_t>(comm->nranks());
for (auto& chunk : output_chunks) {
auto pid = chunk.sequence_number();
std::erase_if(local_pids, [pid](auto& p) { return p == pid; });

auto p_vec = chunk.release<PartitionVectorChunk>();
// for each local pid, it should receive num_chunks * nranks chunks.
ASSERT_EQ(p_vec.data.size(), num_chunks * n_ranks);

// since values are offset by rank, if we sort packed data by their first element,
// then it will be in rank & chunk-index order.
std::ranges::sort(p_vec.data, [](auto& a, auto& b) {
std::int64_t oa{}, ob{};
std::memcpy(&oa, a.metadata->data(), sizeof(std::int64_t));
std::memcpy(&ob, b.metadata->data(), sizeof(std::int64_t));
return oa < ob;
});

// p_vec.data is sorted by first element, so entries are ordered
// (rank=0,chunk=0), (rank=0,chunk=1), ..., (rank=1,chunk=0), ...
// i.e. flat index r*num_chunks + c_idx.
for (size_t r = 0; r < n_ranks; ++r) {
auto r_base = static_cast<int64_t>(r) * num_rows; // rank-base offset
for (size_t c_idx = 0; c_idx < num_chunks; ++c_idx) {
const auto [start, end] = pieces[c_idx * num_partitions + pid];
SCOPED_TRACE(
"pid=" + std::to_string(pid) + ", rank=" + std::to_string(r)
+ ", chunk_idx=" + std::to_string(c_idx)
);
validate_packed_data<int64_t>(
std::move(p_vec.data[r * num_chunks + c_idx]),
end - start,
r_base + static_cast<int64_t>(start),
stream,
*br
);
}
}
}
EXPECT_TRUE(local_pids.empty());
}

class ShufflerAsyncTest
Expand Down
Loading
Loading