Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
0c6cc9c
moving stats creation to a factory
nirandaperera Apr 30, 2026
4d1ee56
minor fix
nirandaperera May 7, 2026
38a73ac
enable shared from this
nirandaperera May 7, 2026
17b638a
stat const ref
nirandaperera May 7, 2026
dc782fc
Merge branch 'main' of github.com:rapidsai/rapidsmpf into stats-refactor
nirandaperera May 7, 2026
09e7369
fix macro
nirandaperera May 7, 2026
aaeaaa3
fix python bindings
nirandaperera May 7, 2026
e4d3796
addressing comments
nirandaperera May 11, 2026
26056e6
Merge branch 'main' of github.com:rapidsai/rapidsmpf into stats-refactor
nirandaperera May 11, 2026
7246300
stats return by value
nirandaperera May 13, 2026
0104a69
Merge branch 'main' of github.com:rapidsai/rapidsmpf into stats-refactor
nirandaperera May 13, 2026
520b018
Merge branch 'main' into stats-refactor
nirandaperera May 14, 2026
b646b6e
Merge branch 'main' into stats-refactor
pentschev May 18, 2026
e491d09
Merge branch 'main' into stats-refactor
nirandaperera May 18, 2026
7249723
Merge branch 'main' into stats-refactor
nirandaperera May 18, 2026
965a2d3
Update cpp/include/rapidsmpf/statistics.hpp
nirandaperera May 19, 2026
0a12bd8
addressing comments
nirandaperera May 19, 2026
f88518d
Merge branch 'main' into stats-refactor
nirandaperera May 20, 2026
785e49e
Merge branch 'main' into stats-refactor
nirandaperera May 21, 2026
f9a2b55
remove opt
nirandaperera May 22, 2026
04a3dae
use a enum for statistics create factory
nirandaperera May 22, 2026
e2100f0
Revert "use a enum for statistics create factory"
nirandaperera May 26, 2026
144bbc8
simplify stats
nirandaperera May 26, 2026
1c2d223
Merge branch 'main' of github.com:rapidsai/rapidsmpf into stats-refactor
nirandaperera May 26, 2026
58c8172
addressing comments
nirandaperera May 26, 2026
ca7a194
remove disabled() method and replace with enum
nirandaperera May 26, 2026
ae9ec20
minor change
nirandaperera May 26, 2026
9127688
add disabled()
nirandaperera May 26, 2026
cfbaaa2
Apply suggestions from code review
nirandaperera May 27, 2026
c1f5c8e
addressing PR comments
nirandaperera May 27, 2026
7d52bc3
Merge branch 'stats-refactor' of github.com:nirandaperera/rapidsmpf i…
nirandaperera May 27, 2026
ce53e7a
Merge branch 'main' of github.com:rapidsai/rapidsmpf into stats-refactor
nirandaperera May 27, 2026
378aaa5
merge conflict
nirandaperera May 27, 2026
09bd4a8
Merge branch 'main' of github.com:rapidsai/rapidsmpf into stats-refactor
nirandaperera May 27, 2026
f866424
Merge branch 'main' of github.com:rapidsai/rapidsmpf into stats-refactor
nirandaperera May 27, 2026
4febceb
merge conflicts
nirandaperera May 27, 2026
39dbd7f
docs fix
nirandaperera May 27, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cpp/benchmarks/bench_shuffle.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -543,7 +543,7 @@ int main(int argc, char** argv) {
memory_limits[rapidsmpf::MemoryType::DEVICE] = args.device_mem_limit_mb << 20;
}

auto stats = std::make_shared<rapidsmpf::Statistics>(/* enable = */ true);
auto stats = rapidsmpf::Statistics::create();

// We're only going to measure the last run, so disable initially.
stats->disable();
Expand Down
7 changes: 4 additions & 3 deletions cpp/benchmarks/streaming/bench_streaming_shuffle.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -364,7 +364,7 @@ int main(int argc, char** argv) {
memory_limits[rapidsmpf::MemoryType::DEVICE] = args.device_mem_limit_mb << 20;
}

auto stats = std::make_shared<rapidsmpf::Statistics>(/* enable = */ true);
auto stats = rapidsmpf::Statistics::create();

auto pinned_mr = args.pinned_mem_disable
? rapidsmpf::PinnedMemoryResource::Disabled
Expand Down Expand Up @@ -458,14 +458,15 @@ int main(int argc, char** argv) {
log.print(ss.str());
}

auto statistics = ctx->statistics();
if (args.enable_memory_profiler) {
log.print(ctx->statistics()->report({
log.print(statistics->report({
.mr = stat_enabled_mr,
.pinned_mr = pinned_mr,
.header = "Statistics (of the last run):",
}));
} else {
log.print(ctx->statistics()->report({.header = "Statistics (of the last run):"}));
log.print(statistics->report({.header = "Statistics (of the last run):"}));
}

if (!use_bootstrap) {
Expand Down
5 changes: 3 additions & 2 deletions cpp/benchmarks/streaming/ndsh/bench_read.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -411,10 +411,11 @@ int main(int argc, char** argv) {
std::chrono::duration<double> compute = end - start;
timings.push_back(pipeline.count());
timings.push_back(compute.count());
comm->logger()->print(ctx->statistics()->report(
auto statistics = ctx->statistics();
comm->logger()->print(statistics->report(
Comment thread
nirandaperera marked this conversation as resolved.
{.mr = ctx->br()->device_mr(), .pinned_mr = ctx->br()->try_pinned_mr()}
));
ctx->statistics()->clear();
statistics->clear();
}

if (comm->rank() == 0) {
Expand Down
5 changes: 3 additions & 2 deletions cpp/benchmarks/streaming/ndsh/q01.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -433,10 +433,11 @@ int main(int argc, char** argv) {
std::chrono::duration<double> compute = end - start;
timings.push_back(pipeline.count());
timings.push_back(compute.count());
comm->logger()->print(ctx->statistics()->report(
auto statistics = ctx->statistics();
comm->logger()->print(statistics->report(
{.mr = ctx->br()->device_mr(), .pinned_mr = ctx->br()->try_pinned_mr()}
));
ctx->statistics()->clear();
statistics->clear();
}

if (comm->rank() == 0) {
Expand Down
5 changes: 3 additions & 2 deletions cpp/benchmarks/streaming/ndsh/q03.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -695,10 +695,11 @@ int main(int argc, char** argv) {
std::chrono::duration<double> compute = end - start;
timings.push_back(pipeline.count());
timings.push_back(compute.count());
comm->logger()->print(ctx->statistics()->report(
auto statistics = ctx->statistics();
comm->logger()->print(statistics->report(
{.mr = ctx->br()->device_mr(), .pinned_mr = ctx->br()->try_pinned_mr()}
));
ctx->statistics()->clear();
statistics->clear();
}
if (comm->rank() == 0) {
for (int i = 0; i < arguments.num_iterations; i++) {
Expand Down
5 changes: 3 additions & 2 deletions cpp/benchmarks/streaming/ndsh/q04.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -518,10 +518,11 @@ int main(int argc, char** argv) {
std::chrono::duration<double> compute = end - start;
timings.push_back(pipeline.count());
timings.push_back(compute.count());
comm->logger()->print(ctx->statistics()->report(
auto statistics = ctx->statistics();
comm->logger()->print(statistics->report(
{.mr = ctx->br()->device_mr(), .pinned_mr = ctx->br()->try_pinned_mr()}
));
ctx->statistics()->clear();
statistics->clear();
}

if (comm->rank() == 0) {
Expand Down
5 changes: 3 additions & 2 deletions cpp/benchmarks/streaming/ndsh/q09.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -653,10 +653,11 @@ int main(int argc, char** argv) {
std::chrono::duration<double> compute = end - start;
timings.push_back(pipeline.count());
timings.push_back(compute.count());
comm->logger()->print(ctx->statistics()->report(
auto statistics = ctx->statistics();
comm->logger()->print(statistics->report(
{.mr = ctx->br()->device_mr(), .pinned_mr = ctx->br()->try_pinned_mr()}
));
ctx->statistics()->clear();
statistics->clear();
}
if (comm->rank() == 0) {
for (std::size_t i = 0; i < safe_cast<std::size_t>(arguments.num_iterations); i++)
Expand Down
5 changes: 3 additions & 2 deletions cpp/benchmarks/streaming/ndsh/q21.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -995,10 +995,11 @@ int main(int argc, char** argv) {
std::chrono::duration<double> compute = end - start;
timings.push_back(pipeline.count());
timings.push_back(compute.count());
comm->logger()->print(ctx->statistics()->report(
auto statistics = ctx->statistics();
comm->logger()->print(statistics->report(
{.mr = ctx->br()->device_mr(), .pinned_mr = ctx->br()->try_pinned_mr()}
));
ctx->statistics()->clear();
statistics->clear();
}

if (comm->rank() == 0) {
Expand Down
2 changes: 1 addition & 1 deletion cpp/benchmarks/streaming/ndsh/utils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ create_context(

memory_limits[MemoryType::DEVICE] = static_cast<std::int64_t>(limit_size);
}
auto statistics = std::make_shared<Statistics>(/* enable = */ true);
auto statistics = Statistics::create();

RAPIDSMPF_EXPECTS(
arguments.no_pinned_host_memory || is_pinned_memory_resources_supported(),
Expand Down
2 changes: 1 addition & 1 deletion cpp/examples/example_shuffle.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ int main(int argc, char** argv) {
rapidsmpf::config::Options options{rapidsmpf::config::get_environment_variables()};

// Create a statistics instance for the shuffler that tracks useful information.
auto stats = std::make_shared<rapidsmpf::Statistics>();
auto stats = rapidsmpf::Statistics::create();

// The communicator has a progress thread where the shuffler event loop executes. A
// single progress thread may be used by multiple shufflers simultaneously.
Expand Down
5 changes: 0 additions & 5 deletions cpp/include/rapidsmpf/coll/allgather.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
#include <rapidsmpf/memory/packed_data.hpp>
#include <rapidsmpf/memory/spill_manager.hpp>
#include <rapidsmpf/progress_thread.hpp>
#include <rapidsmpf/statistics.hpp>

/**
* @namespace rapidsmpf::coll
Expand Down Expand Up @@ -101,8 +100,6 @@ class AllGather {
* @param comm The communicator for communication.
* @param op_id Unique operation identifier for this allgather.
* @param br Buffer resource for memory allocation.
* @param statistics Statistics collection instance (disabled by
* default).
* @param finished_callback Optional callback run when partitions are locally
* finished. The callback is guaranteed to be called by the progress thread exactly
* once when the allgather is locally ready.
Expand All @@ -118,7 +115,6 @@ class AllGather {
std::shared_ptr<Communicator> comm,
OpID op_id,
BufferResource* br,
std::shared_ptr<Statistics> statistics = Statistics::disabled(),
std::function<void(void)>&& finished_callback = nullptr
);

Expand Down Expand Up @@ -195,7 +191,6 @@ class AllGather {

std::shared_ptr<Communicator> comm_; ///< Communicator
BufferResource* br_; ///< Buffer resource for memory allocation
std::shared_ptr<Statistics> statistics_; ///< Statistics collection instance
std::function<void(void)> finished_callback_{
nullptr
}; ///< Optional callback to run when allgather is finished and ready for extraction.
Expand Down
9 changes: 7 additions & 2 deletions cpp/include/rapidsmpf/memory/buffer_resource.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -101,12 +101,15 @@ class BufferResource {
*
* @param mr A device-accessible RMM memory resource.
* @param options Configuration options.
* @param statistics The statistics instance to use (disabled by default).
*
* @return A shared pointer to a BufferResource instance configured according to the
* options.
*/
static std::shared_ptr<BufferResource> from_options(
cuda::mr::any_resource<cuda::mr::device_accessible> mr, config::Options options
cuda::mr::any_resource<cuda::mr::device_accessible> mr,
config::Options options,
std::shared_ptr<Statistics> statistics = Statistics::disabled()
Comment thread
nirandaperera marked this conversation as resolved.
);

~BufferResource() noexcept = default;
Expand Down Expand Up @@ -416,7 +419,7 @@ class BufferResource {
*
* @return Shared pointer the Statistics instance.
*/
std::shared_ptr<Statistics> statistics();
std::shared_ptr<Statistics> statistics() const noexcept;

private:
std::mutex mutex_;
Expand All @@ -435,6 +438,8 @@ class BufferResource {
std::shared_ptr<Statistics> statistics_;
};

static_assert(StatisticsProvider<BufferResource>);

/**
* @brief Parse the `spill_device_limit` parameter from configuration options.
*
Expand Down
2 changes: 2 additions & 0 deletions cpp/include/rapidsmpf/progress_thread.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -205,4 +205,6 @@ class ProgressThread {
detail::PausableThreadLoop thread_;
};

static_assert(StatisticsProvider<ProgressThread>);

} // namespace rapidsmpf
Loading