From 82a3dffa53b33a491ca79e407f737db6fc9e2302 Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Mon, 13 Jul 2026 10:38:24 +0200 Subject: [PATCH 1/8] Split RF histogram and split scoring kernels --- .../batched-levelalgo/builder.cuh | 94 ++++----- .../kernels/builder_kernels.cuh | 9 +- .../kernels/builder_kernels_impl.cuh | 190 +++++++++--------- .../kernels/classification-double.cu | 5 +- .../kernels/classification-float.cu | 5 +- .../kernels/regression-double.cu | 5 +- .../kernels/regression-float.cu | 5 +- .../kernels/weighted-classification-double.cu | 5 +- .../kernels/weighted-classification-float.cu | 5 +- .../kernels/weighted-regression-double.cu | 5 +- .../kernels/weighted-regression-float.cu | 5 +- 11 files changed, 155 insertions(+), 178 deletions(-) diff --git a/cpp/src/decisiontree/batched-levelalgo/builder.cuh b/cpp/src/decisiontree/batched-levelalgo/builder.cuh index f557f30de3..35995b6ad4 100644 --- a/cpp/src/decisiontree/batched-levelalgo/builder.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/builder.cuh @@ -181,8 +181,6 @@ struct Builder { IdxT* n_nodes; /** buffer of segmented histograms*/ BinT* histograms; - /** threadblock arrival count */ - int* done_count; /** mutex array used for atomically updating best split */ int* mutex; /** best splits for the current batch of nodes */ @@ -292,13 +290,12 @@ struct Builder { size_t max_len_histograms = max_batch * params.max_n_bins * n_blks_for_cols * dataset.num_outputs; - d_wsize += calculateAlignedBytes(sizeof(IdxT)); // n_nodes - d_wsize += calculateAlignedBytes(sizeof(BinT) * max_len_histograms); // histograms - d_wsize += calculateAlignedBytes(sizeof(int) * max_batch * n_blks_for_cols); // done_count - d_wsize += calculateAlignedBytes(sizeof(int) * max_batch); // mutex - d_wsize += calculateAlignedBytes(sizeof(SplitT) * max_batch); // splits - d_wsize += calculateAlignedBytes(sizeof(NodeWorkItem) * max_batch); // d_work_Items - d_wsize += // workload_info + d_wsize += calculateAlignedBytes(sizeof(IdxT)); // n_nodes + d_wsize += calculateAlignedBytes(sizeof(BinT) * max_len_histograms); // histograms + d_wsize += calculateAlignedBytes(sizeof(int) * max_batch); // mutex + d_wsize += calculateAlignedBytes(sizeof(SplitT) * max_batch); // splits + d_wsize += calculateAlignedBytes(sizeof(NodeWorkItem) * max_batch); // d_work_Items + d_wsize += // workload_info calculateAlignedBytes(sizeof(WorkloadInfo) * max_blocks_dimx); d_wsize += calculateAlignedBytes(sizeof(IdxT) * max_batch * dataset.n_sampled_cols); // column_samples @@ -323,8 +320,7 @@ struct Builder { { raft::common::nvtx::range fun_scope( "Builder::assignWorkspace @builder.cuh [batched-levelalgo]"); - auto max_batch = params.max_batch_size; - auto n_col_blks = n_blks_for_cols; + auto max_batch = params.max_batch_size; size_t max_len_histograms = max_batch * (params.max_n_bins) * n_blks_for_cols * dataset.num_outputs; // device @@ -332,8 +328,6 @@ struct Builder { d_wspace += calculateAlignedBytes(sizeof(IdxT)); histograms = reinterpret_cast(d_wspace); d_wspace += calculateAlignedBytes(sizeof(BinT) * max_len_histograms); - done_count = reinterpret_cast(d_wspace); - d_wspace += calculateAlignedBytes(sizeof(int) * max_batch * n_col_blks); mutex = reinterpret_cast(d_wspace); d_wspace += calculateAlignedBytes(sizeof(int) * max_batch); splits = reinterpret_cast(d_wspace); @@ -347,8 +341,6 @@ struct Builder { partition_row_ids = reinterpret_cast(d_wspace); d_wspace += calculateAlignedBytes(sizeof(IdxT) * dataset.n_sampled_rows); - RAFT_CUDA_TRY( - cudaMemsetAsync(done_count, 0, sizeof(int) * max_batch * n_col_blks, builder_stream)); RAFT_CUDA_TRY(cudaMemsetAsync(mutex, 0, sizeof(int) * max_batch, builder_stream)); // host @@ -383,23 +375,19 @@ struct Builder { private: auto updateWorkloadInfo(const std::vector& work_items) { - int n_large_nodes = 0; // large nodes are nodes having training instances larger than block - // size, hence require global memory for histogram construction - int n_blocks_dimx = 0; // gridDim.x required for computeSplitKernel + int n_blocks_dimx = 0; // gridDim.x required for histogram construction for (std::size_t i = 0; i < work_items.size(); i++) { auto item = work_items[i]; int n_blocks_per_node = std::max(raft::ceildiv(item.instances.count, size_t(TPB_DEFAULT)), size_t(1)); - if (n_blocks_per_node > 1) ++n_large_nodes; - for (int b = 0; b < n_blocks_per_node; b++) { - h_workload_info[n_blocks_dimx + b] = {int(i), n_large_nodes - 1, b, n_blocks_per_node}; + h_workload_info[n_blocks_dimx + b] = {int(i), b, n_blocks_per_node}; } n_blocks_dimx += n_blocks_per_node; } raft::update_device(workload_info, h_workload_info, n_blocks_dimx, builder_stream); - return std::make_pair(n_blocks_dimx, n_large_nodes); + return n_blocks_dimx; } auto doSplit(const std::vector& work_items) @@ -461,7 +449,7 @@ struct Builder { cudaMemcpyHostToDevice, builder_stream)); raft::update_device(d_work_items, work_items.data(), work_items.size(), builder_stream); - const auto partition_workload = this->updateWorkloadInfo(work_items); + const auto n_partition_blocks = this->updateWorkloadInfo(work_items); raft::common::nvtx::push_range("nodeSplitKernel @builder.cuh [batched-levelalgo]"); launchNodeSplitKernel(params.min_samples_leaf, params.min_impurity_decrease, @@ -469,7 +457,7 @@ struct Builder { d_work_items, splits, workload_info, - partition_workload.first, + n_partition_blocks, partition_row_ids, builder_stream); RAFT_CUDA_TRY(cudaPeekAtLastError()); @@ -484,17 +472,15 @@ struct Builder { IdxT sample_offset) { initSplit(splits, work_items.size(), builder_stream); - RAFT_CUDA_TRY(cudaMemsetAsync( - done_count, 0, sizeof(int) * params.max_batch_size * n_blks_for_cols, builder_stream)); RAFT_CUDA_TRY(cudaMemsetAsync(mutex, 0, sizeof(int) * params.max_batch_size, builder_stream)); raft::update_device(d_work_items, work_items.data(), work_items.size(), builder_stream); - auto [n_blocks_dimx, n_large_nodes] = this->updateWorkloadInfo(work_items); - auto split_smem_config = computeSplitSharedMemoryConfig(); + auto n_blocks_dimx = this->updateWorkloadInfo(work_items); + auto split_smem_config = computeSplitSharedMemoryConfig(); sampleFeatures(work_items, sampling_seed, sample_offset); for (IdxT c = 0; c < dataset.n_sampled_cols; c += n_blks_for_cols) { - computeSplit(c, n_blocks_dimx, n_large_nodes, work_items.size(), split_smem_config); + computeSplit(c, n_blocks_dimx, work_items.size(), split_smem_config); RAFT_CUDA_TRY(cudaPeekAtLastError()); } raft::update_host(h_splits, splits, work_items.size(), builder_stream); @@ -520,7 +506,8 @@ struct Builder { struct SplitSharedMemoryConfig { bool use_global_memory_histogram; - size_t dynamic_smem_size; + size_t histogram_dynamic_smem_size; + size_t split_dynamic_smem_size; }; SplitSharedMemoryConfig computeSplitSharedMemoryConfig() const @@ -530,46 +517,39 @@ struct Builder { auto shared_histogram_size = ML::checked_mul(params.max_n_bins, dataset.num_outputs, sizeof(BinT)); auto shared_quantiles_size = ML::checked_mul(params.max_n_bins, sizeof(DataT)); - auto shared_dynamic_smem_size = - ML::checked_add(shared_histogram_size, shared_quantiles_size, sizeof(int)); - auto alignment_smem_size = - ML::checked_add(sizeof(DataT), ML::checked_mul(3, sizeof(int))); - shared_dynamic_smem_size = - ML::checked_add(shared_dynamic_smem_size, alignment_smem_size); - - // Dynamic shared memory for the fallback path only needs the per-block done - // flag used by the inter-block completion handshake. - auto global_dynamic_smem_size = ML::checked_add(sizeof(int), sizeof(int)); - - // Static shared memory is reserved by the kernel regardless of where the - // histogram lives. + auto histogram_dynamic_smem_size = + ML::checked_add(shared_histogram_size, shared_quantiles_size); + auto histogram_alignment_smem_size = ML::checked_add(sizeof(BinT), sizeof(DataT)); + histogram_dynamic_smem_size = + ML::checked_add(histogram_dynamic_smem_size, histogram_alignment_smem_size); + auto split_dynamic_smem_size = + ML::checked_add(shared_quantiles_size, sizeof(DataT)); + auto cdf_scan_smem_size = sizeof(typename cub::BlockScan::TempStorage); auto split_scratch_smem_size = ML::checked_mul(raft::ceildiv(TPB_DEFAULT, raft::WarpSize), sizeof(SplitT)); - auto static_smem_size = + auto split_static_smem_size = ML::checked_add(cdf_scan_smem_size, split_scratch_smem_size); + auto split_total_smem_size = + ML::checked_add(split_dynamic_smem_size, split_static_smem_size); auto available_smem = size_t(handle.get_device_properties().sharedMemPerBlock); - auto global_total_smem_size = - ML::checked_add(global_dynamic_smem_size, static_smem_size); - ASSERT(available_smem >= global_total_smem_size, + ASSERT(available_smem >= split_static_smem_size, "Not enough shared memory for RF split bookkeeping."); // Prefer shared memory when it fits and stays small enough for good occupancy; // otherwise use the global histogram path to avoid launch failure or slowdown. - auto shared_total_smem_size = - ML::checked_add(shared_dynamic_smem_size, static_smem_size); bool use_global_memory_histogram = - shared_total_smem_size > available_smem || - shared_dynamic_smem_size > tunable_split_histogram_dynamic_smem_limit_bytes; + histogram_dynamic_smem_size > available_smem || split_total_smem_size > available_smem || + histogram_dynamic_smem_size > tunable_split_histogram_dynamic_smem_limit_bytes; return {use_global_memory_histogram, - use_global_memory_histogram ? global_dynamic_smem_size : shared_dynamic_smem_size}; + use_global_memory_histogram ? 0 : histogram_dynamic_smem_size, + use_global_memory_histogram ? 0 : split_dynamic_smem_size}; } void computeSplit(IdxT col, size_t n_blocks_dimx, - size_t n_large_nodes, size_t n_work_items, const SplitSharedMemoryConfig& split_smem_config) { @@ -582,9 +562,8 @@ struct Builder { // if columns left to be processed lesser than `n_blks_for_cols`, shrink the blocks along dimy auto n_blocks_dimy = std::min(n_blks_for_cols, dataset.n_sampled_cols - col); dim3 grid(n_blocks_dimx, n_blocks_dimy, 1); - auto histogram_node_count = use_global_memory_histogram ? n_work_items : n_large_nodes; auto len_histograms = - ML::checked_mul(n_bins, n_classes, n_blocks_dimy, histogram_node_count); + ML::checked_mul(n_bins, n_classes, n_blocks_dimy, n_work_items); auto histograms_bytes = ML::checked_mul(sizeof(BinT), len_histograms); RAFT_CUDA_TRY(cudaMemsetAsync(histograms, 0, histograms_bytes, builder_stream)); // create the objective function object @@ -601,16 +580,17 @@ struct Builder { d_work_items, col, column_samples, - done_count, mutex, splits, objective, treeid, workload_info, seed, + n_work_items, use_global_memory_histogram, grid, - split_smem_config.dynamic_smem_size, + split_smem_config.histogram_dynamic_smem_size, + split_smem_config.split_dynamic_smem_size, builder_stream); } diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh index 882f7fbdcb..fe1329ed6f 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh @@ -42,9 +42,7 @@ struct NodeWorkItem { */ template struct WorkloadInfo { - IdxT nodeid; // Node in the batch on which the threadblock needs to work - IdxT large_nodeid; // counts only large nodes (nodes that require more than one block along x-dim - // for histogram calculation) + IdxT nodeid; // Node in the batch on which the threadblock needs to work IdxT offset_blockid; // Offset threadblock id among all the blocks that are // working on this node IdxT num_blocks; // Total number of blocks that are working on the node @@ -156,16 +154,17 @@ void launchComputeSplitKernel(typename ObjectiveT::BinT* histograms, const NodeWorkItem* work_items, IdxT colStart, const IdxT* column_samples, - int* done_count, int* mutex, volatile Split* splits, ObjectiveT& objective, IdxT treeid, const WorkloadInfo* workload_info, uint64_t seed, + size_t n_work_items, bool use_global_memory_histogram, dim3 grid, - size_t smem_size, + size_t histogram_smem_size, + size_t split_smem_size, cudaStream_t builder_stream); } // namespace DT diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh index 4afe592819..dd2b96ce18 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh @@ -251,36 +251,22 @@ DI BinT pdf_to_cdf(BinT* histogram, IdxT n_bins) } template -static __global__ void computeSplitKernel(typename ObjectiveT::BinT* histograms, - IdxT max_n_bins, - IdxT min_samples_split, - IdxT max_leaves, - const Dataset dataset, - const Quantiles quantiles, - const NodeWorkItem* work_items, - IdxT colStart, - const IdxT* column_samples, - int* done_count, - int* mutex, - volatile Split* splits, - ObjectiveT objective, - IdxT treeid, - const WorkloadInfo* workload_info, - uint64_t seed, - bool use_global_memory_histogram) +static __global__ void buildHistogramsKernel(typename ObjectiveT::BinT* histograms, + IdxT max_n_bins, + const Dataset dataset, + const Quantiles quantiles, + const NodeWorkItem* work_items, + IdxT colStart, + const IdxT* column_samples, + ObjectiveT objective, + const WorkloadInfo* workload_info, + bool use_global_memory_histogram) { using BinT = typename ObjectiveT::BinT; - // dynamic shared memory extern __shared__ char smem[]; - constexpr int n_split_warps = (TPB + raft::WarpSize - 1) / raft::WarpSize; - __shared__ __align__(alignof(Split)) unsigned char - split_scratch_storage[sizeof(Split) * n_split_warps]; - auto* split_scratch = reinterpret_cast*>(split_scratch_storage); - // Read workload info for this block WorkloadInfo workload_info_cta = workload_info[blockIdx.x]; IdxT nid = workload_info_cta.nodeid; - IdxT large_nid = workload_info_cta.large_nodeid; const auto work_item = work_items[nid]; auto range_start = work_item.instances.begin; auto range_len = work_item.instances.count; @@ -288,30 +274,23 @@ static __global__ void computeSplitKernel(typename ObjectiveT::BinT* histograms, IdxT offset_blockid = workload_info_cta.offset_blockid; IdxT num_blocks = workload_info_cta.num_blocks; - // obtaining the feature to test split on IdxT colIndex = colStart + blockIdx.y; IdxT col = column_samples[nid * dataset.n_sampled_cols + colIndex]; - - // getting the n_bins for that feature - int n_bins = quantiles.n_bins_array[col]; + int n_bins = quantiles.n_bins_array[col]; auto n_classes = objective.NumClasses(); auto end = range_start + range_len; auto histogram_len = n_bins * n_classes; - auto* histogram = static_cast(nullptr); - auto* shared_done = static_cast(nullptr); + auto histograms_offset = (std::size_t(nid) * gridDim.y + blockIdx.y) * max_n_bins * n_classes; + auto* global_histogram = histograms + histograms_offset; + auto* histogram = global_histogram; auto* quantiles_for_split = quantiles.quantiles_array + std::size_t(max_n_bins) * col; IdxT stride = blockDim.x * num_blocks; IdxT tid = threadIdx.x + offset_blockid * blockDim.x; - if (use_global_memory_histogram) { - auto histograms_offset = (std::size_t(nid) * gridDim.y + blockIdx.y) * max_n_bins * n_classes; - histogram = histograms + histograms_offset; - shared_done = alignPointer(smem); - } else { + if (!use_global_memory_histogram) { histogram = alignPointer(smem); auto* shared_quantiles = alignPointer(histogram + histogram_len); - shared_done = alignPointer(shared_quantiles + n_bins); quantiles_for_split = shared_quantiles; for (IdxT i = threadIdx.x; i < histogram_len; i += blockDim.x) { histogram[i] = BinT(); @@ -319,80 +298,78 @@ static __global__ void computeSplitKernel(typename ObjectiveT::BinT* histograms, for (IdxT b = threadIdx.x; b < n_bins; b += blockDim.x) { shared_quantiles[b] = quantiles.quantiles_array[max_n_bins * col + b]; } + __syncthreads(); } - __syncthreads(); - - // compute pdf histogram for all bins for all classes - - // Must be 64 bit - can easily grow larger than a 32 bit int for (auto i = range_start + tid; i < end; i += stride) { - // each thread works over a data point and strides to the next auto row = dataset.row_ids[i]; auto data = dataset.value(row, col); auto label = dataset.labels[row]; - // `start` is lowest index such that data <= quantiles_for_split[start] IdxT start = lower_bound(quantiles_for_split, n_bins, data); - // ++histogram[start] objective.IncrementHistogram(histogram, n_bins, start, label, dataset, row); } - __syncthreads(); - if (use_global_memory_histogram) { - __threadfence(); // for commit guarantee before the last block scores the split + if (!use_global_memory_histogram) { __syncthreads(); - - bool last = MLCommon::signalDone( - done_count + nid * gridDim.y + blockIdx.y, num_blocks, offset_blockid == 0, shared_done); - if (!last) return; - } else if (num_blocks > 1) { - // Shared-memory histogram path: each block built a partial histogram, so unify those - // partial histograms in global memory before scoring the split. - // update the corresponding global location - auto histograms_offset = - (std::size_t(large_nid) * gridDim.y + blockIdx.y) * max_n_bins * n_classes; for (IdxT i = threadIdx.x; i < histogram_len; i += blockDim.x) { - BinT::AtomicAdd(histograms + histograms_offset + i, histogram[i]); + BinT::AtomicAdd(global_histogram + i, histogram[i]); } + } +} - __threadfence(); // for commit guarantee - __syncthreads(); +template +static __global__ void findBestSplitsKernel(typename ObjectiveT::BinT* histograms, + IdxT max_n_bins, + const Dataset dataset, + const Quantiles quantiles, + const NodeWorkItem* work_items, + IdxT colStart, + const IdxT* column_samples, + int* mutex, + volatile Split* splits, + ObjectiveT objective, + bool use_global_memory_histogram) +{ + using BinT = typename ObjectiveT::BinT; + extern __shared__ char smem[]; + constexpr int n_split_warps = (TPB + raft::WarpSize - 1) / raft::WarpSize; + __shared__ __align__(alignof(Split)) unsigned char + split_scratch_storage[sizeof(Split) * n_split_warps]; + auto* split_scratch = reinterpret_cast*>(split_scratch_storage); - // last threadblock will go ahead and compute the best split - bool last = MLCommon::signalDone( - done_count + nid * gridDim.y + blockIdx.y, num_blocks, offset_blockid == 0, shared_done); - // if not the last threadblock, exit - if (!last) return; + IdxT nid = blockIdx.x; + auto work_item = work_items[nid]; + auto range_len = work_item.instances.count; - // store the complete global histogram in shared memory of last block - for (IdxT i = threadIdx.x; i < histogram_len; i += blockDim.x) { - histogram[i] = histograms[histograms_offset + i]; - } + IdxT colIndex = colStart + blockIdx.y; + IdxT col = column_samples[nid * dataset.n_sampled_cols + colIndex]; + int n_bins = quantiles.n_bins_array[col]; + + auto n_classes = objective.NumClasses(); + auto histograms_offset = (std::size_t(nid) * gridDim.y + blockIdx.y) * max_n_bins * n_classes; + auto* histogram = histograms + histograms_offset; + auto* quantiles_for_split = quantiles.quantiles_array + std::size_t(max_n_bins) * col; + if (!use_global_memory_histogram) { + auto* shared_quantiles = alignPointer(smem); + quantiles_for_split = shared_quantiles; + for (IdxT b = threadIdx.x; b < n_bins; b += blockDim.x) { + shared_quantiles[b] = quantiles.quantiles_array[max_n_bins * col + b]; + } __syncthreads(); } - // PDF to CDF inplace in `histogram` for (IdxT c = 0; c < n_classes; ++c) { - // left to right scan operation for scanning - // "lesser-than-or-equal" counts - BinT total_sum = pdf_to_cdf(histogram + n_bins * c, n_bins); - // now, `histogram[n_bins * c + i]` will have count of datapoints of class `c` - // that are less than or equal to `quantiles_for_split[i]`. + pdf_to_cdf(histogram + n_bins * c, n_bins); } __syncthreads(); - // calculate the best candidate bins (one for each thread in the block) in current feature and - // corresponding information gain for splitting Split sp = objective.Gain(histogram, quantiles_for_split, col, range_len, n_bins); __syncthreads(); - // calculate best bins among candidate bins per feature using warp reduce - // then atomically update across features to get best split per node - // (in split[nid]) sp.evalBestSplit(split_scratch, splits + nid, mutex + nid, quantiles_for_split, n_bins); } @@ -406,36 +383,49 @@ void launchComputeSplitKernel(typename ObjectiveT::BinT* histograms, const NodeWorkItem* work_items, IdxT colStart, const IdxT* column_samples, - int* done_count, int* mutex, volatile Split* splits, ObjectiveT& objective, IdxT treeid, const WorkloadInfo* workload_info, uint64_t seed, + size_t n_work_items, bool use_global_memory_histogram, dim3 grid, - size_t smem_size, + size_t histogram_smem_size, + size_t split_smem_size, cudaStream_t builder_stream) { - computeSplitKernel - <<>>(histograms, - max_n_bins, - min_samples_split, - max_leaves, - dataset, - quantiles, - work_items, - colStart, - column_samples, - done_count, - mutex, - splits, - objective, - treeid, - workload_info, - seed, - use_global_memory_histogram); + (void)min_samples_split; + (void)max_leaves; + (void)treeid; + (void)seed; + + buildHistogramsKernel + <<>>(histograms, + max_n_bins, + dataset, + quantiles, + work_items, + colStart, + column_samples, + objective, + workload_info, + use_global_memory_histogram); + + dim3 split_grid(n_work_items, grid.y, 1); + findBestSplitsKernel + <<>>(histograms, + max_n_bins, + dataset, + quantiles, + work_items, + colStart, + column_samples, + mutex, + splits, + objective, + use_global_memory_histogram); } } // namespace DT diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu index 919429375c..3dc8ae283e 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu @@ -39,16 +39,17 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, IdxT treeid, const WorkloadInfo* workload_info, uint64_t seed, + size_t n_work_items, bool use_global_memory_histogram, dim3 grid, - size_t smem_size, + size_t histogram_smem_size, + size_t split_smem_size, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu index 32560e72cf..b98500decf 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu @@ -39,16 +39,17 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, IdxT treeid, const WorkloadInfo* workload_info, uint64_t seed, + size_t n_work_items, bool use_global_memory_histogram, dim3 grid, - size_t smem_size, + size_t histogram_smem_size, + size_t split_smem_size, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu index b9b72a6495..6b44a935ed 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu @@ -39,16 +39,17 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, IdxT treeid, const WorkloadInfo* workload_info, uint64_t seed, + size_t n_work_items, bool use_global_memory_histogram, dim3 grid, - size_t smem_size, + size_t histogram_smem_size, + size_t split_smem_size, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu index 68d888fd97..4b48aa83b9 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu @@ -39,16 +39,17 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, IdxT treeid, const WorkloadInfo* workload_info, uint64_t seed, + size_t n_work_items, bool use_global_memory_histogram, dim3 grid, - size_t smem_size, + size_t histogram_smem_size, + size_t split_smem_size, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu index 2d9590d097..63866cd0cf 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu @@ -39,16 +39,17 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, IdxT treeid, const WorkloadInfo* workload_info, uint64_t seed, + size_t n_work_items, bool use_global_memory_histogram, dim3 grid, - size_t smem_size, + size_t histogram_smem_size, + size_t split_smem_size, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu index f74ff643ac..a43641baba 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu @@ -39,16 +39,17 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, IdxT treeid, const WorkloadInfo* workload_info, uint64_t seed, + size_t n_work_items, bool use_global_memory_histogram, dim3 grid, - size_t smem_size, + size_t histogram_smem_size, + size_t split_smem_size, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu index 442cc9c024..bec3558d01 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu @@ -39,16 +39,17 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, IdxT treeid, const WorkloadInfo* workload_info, uint64_t seed, + size_t n_work_items, bool use_global_memory_histogram, dim3 grid, - size_t smem_size, + size_t histogram_smem_size, + size_t split_smem_size, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu index 115d00af22..d92ecb85ac 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu @@ -39,16 +39,17 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, IdxT treeid, const WorkloadInfo* workload_info, uint64_t seed, + size_t n_work_items, bool use_global_memory_histogram, dim3 grid, - size_t smem_size, + size_t histogram_smem_size, + size_t split_smem_size, cudaStream_t builder_stream); } // namespace DT } // namespace ML From df073de7207348000db6fb483a75834aeb38c053 Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Mon, 13 Jul 2026 10:49:25 +0200 Subject: [PATCH 2/8] Remove unused RF split CDF return value --- .../batched-levelalgo/kernels/builder_kernels_impl.cuh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh index dd2b96ce18..96f451fdc7 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh @@ -222,13 +222,10 @@ void launchLeafKernel(ObjectiveT objective, /** * @brief For every threadblock, converts a pdf-histogram to a - * cdf-histogram inplace using inclusive block-sum-scan and returns - * the total_sum - * @return The total sum aggregated over the sumscan, - * as well as the modified cdf-histogram pointer + * cdf-histogram inplace using inclusive block-sum-scan. */ template -DI BinT pdf_to_cdf(BinT* histogram, IdxT n_bins) +DI void pdf_to_cdf(BinT* histogram, IdxT n_bins) { // Blockscan instance preparation typedef cub::BlockScan BlockScan; @@ -246,8 +243,6 @@ DI BinT pdf_to_cdf(BinT* histogram, IdxT n_bins) if (tix < n_bins) { histogram[tix] = result + total_aggregate; } total_aggregate += block_aggregate; } - // return the total sum - return total_aggregate; } template From c37521fcf78fa9ec6c6207870802948d00e4a3d5 Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Mon, 13 Jul 2026 10:53:20 +0200 Subject: [PATCH 3/8] Remove unused RF split launch parameters --- cpp/src/decisiontree/batched-levelalgo/builder.cuh | 4 ---- .../batched-levelalgo/kernels/builder_kernels.cuh | 4 ---- .../batched-levelalgo/kernels/builder_kernels_impl.cuh | 9 --------- .../batched-levelalgo/kernels/classification-double.cu | 4 ---- .../batched-levelalgo/kernels/classification-float.cu | 4 ---- .../batched-levelalgo/kernels/regression-double.cu | 4 ---- .../batched-levelalgo/kernels/regression-float.cu | 4 ---- .../kernels/weighted-classification-double.cu | 4 ---- .../kernels/weighted-classification-float.cu | 4 ---- .../kernels/weighted-regression-double.cu | 4 ---- .../kernels/weighted-regression-float.cu | 4 ---- 11 files changed, 49 deletions(-) diff --git a/cpp/src/decisiontree/batched-levelalgo/builder.cuh b/cpp/src/decisiontree/batched-levelalgo/builder.cuh index 35995b6ad4..4321b7b855 100644 --- a/cpp/src/decisiontree/batched-levelalgo/builder.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/builder.cuh @@ -573,8 +573,6 @@ struct Builder { launchComputeSplitKernel( histograms, params.max_n_bins, - params.min_samples_split, - params.max_leaves, dataset, quantiles, d_work_items, @@ -583,9 +581,7 @@ struct Builder { mutex, splits, objective, - treeid, workload_info, - seed, n_work_items, use_global_memory_histogram, grid, diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh index fe1329ed6f..3c683813a6 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh @@ -147,8 +147,6 @@ HDI IdxT lower_bound(DataT const* array, IdxT len, DataT element) template void launchComputeSplitKernel(typename ObjectiveT::BinT* histograms, IdxT n_bins, - IdxT min_samples_split, - IdxT max_leaves, const Dataset& dataset, const Quantiles& quantiles, const NodeWorkItem* work_items, @@ -157,9 +155,7 @@ void launchComputeSplitKernel(typename ObjectiveT::BinT* histograms, int* mutex, volatile Split* splits, ObjectiveT& objective, - IdxT treeid, const WorkloadInfo* workload_info, - uint64_t seed, size_t n_work_items, bool use_global_memory_histogram, dim3 grid, diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh index 96f451fdc7..160d7069de 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh @@ -371,8 +371,6 @@ static __global__ void findBestSplitsKernel(typename ObjectiveT::BinT* histogram template void launchComputeSplitKernel(typename ObjectiveT::BinT* histograms, IdxT max_n_bins, - IdxT min_samples_split, - IdxT max_leaves, const Dataset& dataset, const Quantiles& quantiles, const NodeWorkItem* work_items, @@ -381,9 +379,7 @@ void launchComputeSplitKernel(typename ObjectiveT::BinT* histograms, int* mutex, volatile Split* splits, ObjectiveT& objective, - IdxT treeid, const WorkloadInfo* workload_info, - uint64_t seed, size_t n_work_items, bool use_global_memory_histogram, dim3 grid, @@ -391,11 +387,6 @@ void launchComputeSplitKernel(typename ObjectiveT::BinT* histograms, size_t split_smem_size, cudaStream_t builder_stream) { - (void)min_samples_split; - (void)max_leaves; - (void)treeid; - (void)seed; - buildHistogramsKernel <<>>(histograms, max_n_bins, diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu index 3dc8ae283e..a5e9f5a174 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu @@ -32,8 +32,6 @@ template void launchLeafKernel( template void launchComputeSplitKernel( BinT* histograms, IdxT n_bins, - IdxT min_samples_split, - IdxT max_leaves, const DatasetT& dataset, const Quantiles& quantiles, const NodeWorkItem* work_items, @@ -42,9 +40,7 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, - IdxT treeid, const WorkloadInfo* workload_info, - uint64_t seed, size_t n_work_items, bool use_global_memory_histogram, dim3 grid, diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu index b98500decf..6ecee6dde1 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu @@ -32,8 +32,6 @@ template void launchLeafKernel( template void launchComputeSplitKernel( BinT* histograms, IdxT n_bins, - IdxT min_samples_split, - IdxT max_leaves, const DatasetT& dataset, const Quantiles& quantiles, const NodeWorkItem* work_items, @@ -42,9 +40,7 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, - IdxT treeid, const WorkloadInfo* workload_info, - uint64_t seed, size_t n_work_items, bool use_global_memory_histogram, dim3 grid, diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu index 6b44a935ed..d8fbfb8dab 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu @@ -32,8 +32,6 @@ template void launchLeafKernel( template void launchComputeSplitKernel( BinT* histograms, IdxT n_bins, - IdxT min_samples_split, - IdxT max_leaves, const DatasetT& dataset, const Quantiles& quantiles, const NodeWorkItem* work_items, @@ -42,9 +40,7 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, - IdxT treeid, const WorkloadInfo* workload_info, - uint64_t seed, size_t n_work_items, bool use_global_memory_histogram, dim3 grid, diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu index 4b48aa83b9..b5d9415484 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu @@ -32,8 +32,6 @@ template void launchLeafKernel( template void launchComputeSplitKernel( BinT* histograms, IdxT n_bins, - IdxT min_samples_split, - IdxT max_leaves, const DatasetT& dataset, const Quantiles& quantiles, const NodeWorkItem* work_items, @@ -42,9 +40,7 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, - IdxT treeid, const WorkloadInfo* workload_info, - uint64_t seed, size_t n_work_items, bool use_global_memory_histogram, dim3 grid, diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu index 63866cd0cf..0bc7b56570 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu @@ -32,8 +32,6 @@ template void launchLeafKernel( template void launchComputeSplitKernel( BinT* histograms, IdxT n_bins, - IdxT min_samples_split, - IdxT max_leaves, const DatasetT& dataset, const Quantiles& quantiles, const NodeWorkItem* work_items, @@ -42,9 +40,7 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, - IdxT treeid, const WorkloadInfo* workload_info, - uint64_t seed, size_t n_work_items, bool use_global_memory_histogram, dim3 grid, diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu index a43641baba..61c8d93396 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu @@ -32,8 +32,6 @@ template void launchLeafKernel( template void launchComputeSplitKernel( BinT* histograms, IdxT n_bins, - IdxT min_samples_split, - IdxT max_leaves, const DatasetT& dataset, const Quantiles& quantiles, const NodeWorkItem* work_items, @@ -42,9 +40,7 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, - IdxT treeid, const WorkloadInfo* workload_info, - uint64_t seed, size_t n_work_items, bool use_global_memory_histogram, dim3 grid, diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu index bec3558d01..ee1f418785 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu @@ -32,8 +32,6 @@ template void launchLeafKernel( template void launchComputeSplitKernel( BinT* histograms, IdxT n_bins, - IdxT min_samples_split, - IdxT max_leaves, const DatasetT& dataset, const Quantiles& quantiles, const NodeWorkItem* work_items, @@ -42,9 +40,7 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, - IdxT treeid, const WorkloadInfo* workload_info, - uint64_t seed, size_t n_work_items, bool use_global_memory_histogram, dim3 grid, diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu index d92ecb85ac..934c6a3592 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu @@ -32,8 +32,6 @@ template void launchLeafKernel( template void launchComputeSplitKernel( BinT* histograms, IdxT n_bins, - IdxT min_samples_split, - IdxT max_leaves, const DatasetT& dataset, const Quantiles& quantiles, const NodeWorkItem* work_items, @@ -42,9 +40,7 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, - IdxT treeid, const WorkloadInfo* workload_info, - uint64_t seed, size_t n_work_items, bool use_global_memory_histogram, dim3 grid, From 73008c234c7d30d105bdd50fa6bce42a8e2b67e5 Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Mon, 13 Jul 2026 11:15:49 +0200 Subject: [PATCH 4/8] Tidy RF split kernel launch API --- .../batched-levelalgo/builder.cuh | 57 ++++++------- .../kernels/builder_kernels.cuh | 38 +++++---- .../kernels/builder_kernels_impl.cuh | 79 +++++++++---------- .../kernels/classification-double.cu | 10 +-- .../kernels/classification-float.cu | 10 +-- .../kernels/regression-double.cu | 10 +-- .../kernels/regression-float.cu | 10 +-- .../kernels/weighted-classification-double.cu | 10 +-- .../kernels/weighted-classification-float.cu | 10 +-- .../kernels/weighted-regression-double.cu | 10 +-- .../kernels/weighted-regression-float.cu | 10 +-- 11 files changed, 116 insertions(+), 138 deletions(-) diff --git a/cpp/src/decisiontree/batched-levelalgo/builder.cuh b/cpp/src/decisiontree/batched-levelalgo/builder.cuh index 4321b7b855..347cbe3bbf 100644 --- a/cpp/src/decisiontree/batched-levelalgo/builder.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/builder.cuh @@ -475,7 +475,7 @@ struct Builder { RAFT_CUDA_TRY(cudaMemsetAsync(mutex, 0, sizeof(int) * params.max_batch_size, builder_stream)); raft::update_device(d_work_items, work_items.data(), work_items.size(), builder_stream); auto n_blocks_dimx = this->updateWorkloadInfo(work_items); - auto split_smem_config = computeSplitSharedMemoryConfig(); + auto split_smem_config = computeSharedMemoryConfig(); sampleFeatures(work_items, sampling_seed, sample_offset); @@ -504,13 +504,7 @@ struct Builder { RAFT_CUDA_TRY(cudaPeekAtLastError()); } - struct SplitSharedMemoryConfig { - bool use_global_memory_histogram; - size_t histogram_dynamic_smem_size; - size_t split_dynamic_smem_size; - }; - - SplitSharedMemoryConfig computeSplitSharedMemoryConfig() const + SharedMemoryConfig computeSharedMemoryConfig() const { // Dynamic shared memory for the fast path: histogram, copied quantiles, and // alignment padding for the kernel's shared-memory layout. @@ -551,43 +545,40 @@ struct Builder { void computeSplit(IdxT col, size_t n_blocks_dimx, size_t n_work_items, - const SplitSharedMemoryConfig& split_smem_config) + const SharedMemoryConfig& split_smem_config) { // if no instances to split, return if (n_blocks_dimx == 0) return; raft::common::nvtx::range fun_scope("Builder::computeSplit @builder.cuh [batched-levelalgo]"); - auto n_bins = params.max_n_bins; - auto n_classes = dataset.num_outputs; - auto use_global_memory_histogram = split_smem_config.use_global_memory_histogram; + auto n_bins = params.max_n_bins; + auto n_classes = dataset.num_outputs; // if columns left to be processed lesser than `n_blks_for_cols`, shrink the blocks along dimy auto n_blocks_dimy = std::min(n_blks_for_cols, dataset.n_sampled_cols - col); - dim3 grid(n_blocks_dimx, n_blocks_dimy, 1); + dim3 histogram_grid(n_blocks_dimx, n_blocks_dimy, 1); + dim3 split_grid(n_work_items, n_blocks_dimy, 1); auto len_histograms = ML::checked_mul(n_bins, n_classes, n_blocks_dimy, n_work_items); auto histograms_bytes = ML::checked_mul(sizeof(BinT), len_histograms); RAFT_CUDA_TRY(cudaMemsetAsync(histograms, 0, histograms_bytes, builder_stream)); // create the objective function object ObjectiveT objective(dataset.num_outputs, params.min_samples_leaf, params.split_criterion); - // call the computeSplitKernel - raft::common::nvtx::range kernel_scope("computeSplitKernel @builder.cuh [batched-levelalgo]"); - launchComputeSplitKernel( - histograms, - params.max_n_bins, - dataset, - quantiles, - d_work_items, - col, - column_samples, - mutex, - splits, - objective, - workload_info, - n_work_items, - use_global_memory_histogram, - grid, - split_smem_config.histogram_dynamic_smem_size, - split_smem_config.split_dynamic_smem_size, - builder_stream); + // call the compute split kernels + raft::common::nvtx::range kernel_scope("computeSplitKernels @builder.cuh [batched-levelalgo]"); + launchComputeSplitKernels(histograms, + params.max_n_bins, + dataset, + quantiles, + d_work_items, + col, + column_samples, + mutex, + splits, + objective, + workload_info, + histogram_grid, + split_grid, + split_smem_config, + builder_stream); } // Set the leaf value predictions in batch diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh index 3c683813a6..d013a10a96 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh @@ -48,6 +48,12 @@ struct WorkloadInfo { IdxT num_blocks; // Total number of blocks that are working on the node }; +struct SharedMemoryConfig { + bool use_global_memory_histogram; + size_t histogram_dynamic_smem_size; + size_t split_dynamic_smem_size; +}; + template HDI bool SplitPartitionNotValid(const SplitT& split, IdxT min_samples_leaf, std::size_t num_rows) { @@ -145,23 +151,21 @@ HDI IdxT lower_bound(DataT const* array, IdxT len, DataT element) } template -void launchComputeSplitKernel(typename ObjectiveT::BinT* histograms, - IdxT n_bins, - const Dataset& dataset, - const Quantiles& quantiles, - const NodeWorkItem* work_items, - IdxT colStart, - const IdxT* column_samples, - int* mutex, - volatile Split* splits, - ObjectiveT& objective, - const WorkloadInfo* workload_info, - size_t n_work_items, - bool use_global_memory_histogram, - dim3 grid, - size_t histogram_smem_size, - size_t split_smem_size, - cudaStream_t builder_stream); +void launchComputeSplitKernels(typename ObjectiveT::BinT* histograms, + IdxT n_bins, + const Dataset& dataset, + const Quantiles& quantiles, + const NodeWorkItem* work_items, + IdxT colStart, + const IdxT* column_samples, + int* mutex, + volatile Split* splits, + ObjectiveT& objective, + const WorkloadInfo* workload_info, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, + cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh index 160d7069de..b657bf1c8f 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh @@ -369,49 +369,48 @@ static __global__ void findBestSplitsKernel(typename ObjectiveT::BinT* histogram } template -void launchComputeSplitKernel(typename ObjectiveT::BinT* histograms, - IdxT max_n_bins, - const Dataset& dataset, - const Quantiles& quantiles, - const NodeWorkItem* work_items, - IdxT colStart, - const IdxT* column_samples, - int* mutex, - volatile Split* splits, - ObjectiveT& objective, - const WorkloadInfo* workload_info, - size_t n_work_items, - bool use_global_memory_histogram, - dim3 grid, - size_t histogram_smem_size, - size_t split_smem_size, - cudaStream_t builder_stream) +void launchComputeSplitKernels(typename ObjectiveT::BinT* histograms, + IdxT max_n_bins, + const Dataset& dataset, + const Quantiles& quantiles, + const NodeWorkItem* work_items, + IdxT colStart, + const IdxT* column_samples, + int* mutex, + volatile Split* splits, + ObjectiveT& objective, + const WorkloadInfo* workload_info, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, + cudaStream_t builder_stream) { buildHistogramsKernel - <<>>(histograms, - max_n_bins, - dataset, - quantiles, - work_items, - colStart, - column_samples, - objective, - workload_info, - use_global_memory_histogram); - - dim3 split_grid(n_work_items, grid.y, 1); + <<>>( + histograms, + max_n_bins, + dataset, + quantiles, + work_items, + colStart, + column_samples, + objective, + workload_info, + split_smem_config.use_global_memory_histogram); + findBestSplitsKernel - <<>>(histograms, - max_n_bins, - dataset, - quantiles, - work_items, - colStart, - column_samples, - mutex, - splits, - objective, - use_global_memory_histogram); + <<>>( + histograms, + max_n_bins, + dataset, + quantiles, + work_items, + colStart, + column_samples, + mutex, + splits, + objective, + split_smem_config.use_global_memory_histogram); } } // namespace DT diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu index a5e9f5a174..a9ebd850f7 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu @@ -29,7 +29,7 @@ template void launchLeafKernel( cudaStream_t builder_stream); // Explicit instantiations are split across separate .cu files to increase compilation parallelism. -template void launchComputeSplitKernel( +template void launchComputeSplitKernels( BinT* histograms, IdxT n_bins, const DatasetT& dataset, @@ -41,11 +41,9 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, const WorkloadInfo* workload_info, - size_t n_work_items, - bool use_global_memory_histogram, - dim3 grid, - size_t histogram_smem_size, - size_t split_smem_size, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu index 6ecee6dde1..c995c72e59 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu @@ -29,7 +29,7 @@ template void launchLeafKernel( cudaStream_t builder_stream); // Explicit instantiations are split across separate .cu files to increase compilation parallelism. -template void launchComputeSplitKernel( +template void launchComputeSplitKernels( BinT* histograms, IdxT n_bins, const DatasetT& dataset, @@ -41,11 +41,9 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, const WorkloadInfo* workload_info, - size_t n_work_items, - bool use_global_memory_histogram, - dim3 grid, - size_t histogram_smem_size, - size_t split_smem_size, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu index d8fbfb8dab..a57a9372f7 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu @@ -29,7 +29,7 @@ template void launchLeafKernel( cudaStream_t builder_stream); // Explicit instantiations are split across separate .cu files to increase compilation parallelism. -template void launchComputeSplitKernel( +template void launchComputeSplitKernels( BinT* histograms, IdxT n_bins, const DatasetT& dataset, @@ -41,11 +41,9 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, const WorkloadInfo* workload_info, - size_t n_work_items, - bool use_global_memory_histogram, - dim3 grid, - size_t histogram_smem_size, - size_t split_smem_size, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu index b5d9415484..479333187c 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu @@ -29,7 +29,7 @@ template void launchLeafKernel( cudaStream_t builder_stream); // Explicit instantiations are split across separate .cu files to increase compilation parallelism. -template void launchComputeSplitKernel( +template void launchComputeSplitKernels( BinT* histograms, IdxT n_bins, const DatasetT& dataset, @@ -41,11 +41,9 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, const WorkloadInfo* workload_info, - size_t n_work_items, - bool use_global_memory_histogram, - dim3 grid, - size_t histogram_smem_size, - size_t split_smem_size, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu index 0bc7b56570..e7ae90c8b4 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu @@ -29,7 +29,7 @@ template void launchLeafKernel( cudaStream_t builder_stream); // Explicit instantiations are split across separate .cu files to increase compilation parallelism. -template void launchComputeSplitKernel( +template void launchComputeSplitKernels( BinT* histograms, IdxT n_bins, const DatasetT& dataset, @@ -41,11 +41,9 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, const WorkloadInfo* workload_info, - size_t n_work_items, - bool use_global_memory_histogram, - dim3 grid, - size_t histogram_smem_size, - size_t split_smem_size, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu index 61c8d93396..0822787576 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu @@ -29,7 +29,7 @@ template void launchLeafKernel( cudaStream_t builder_stream); // Explicit instantiations are split across separate .cu files to increase compilation parallelism. -template void launchComputeSplitKernel( +template void launchComputeSplitKernels( BinT* histograms, IdxT n_bins, const DatasetT& dataset, @@ -41,11 +41,9 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, const WorkloadInfo* workload_info, - size_t n_work_items, - bool use_global_memory_histogram, - dim3 grid, - size_t histogram_smem_size, - size_t split_smem_size, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu index ee1f418785..8e37f931c8 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu @@ -29,7 +29,7 @@ template void launchLeafKernel( cudaStream_t builder_stream); // Explicit instantiations are split across separate .cu files to increase compilation parallelism. -template void launchComputeSplitKernel( +template void launchComputeSplitKernels( BinT* histograms, IdxT n_bins, const DatasetT& dataset, @@ -41,11 +41,9 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, const WorkloadInfo* workload_info, - size_t n_work_items, - bool use_global_memory_histogram, - dim3 grid, - size_t histogram_smem_size, - size_t split_smem_size, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu index 934c6a3592..717f362197 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu @@ -29,7 +29,7 @@ template void launchLeafKernel( cudaStream_t builder_stream); // Explicit instantiations are split across separate .cu files to increase compilation parallelism. -template void launchComputeSplitKernel( +template void launchComputeSplitKernels( BinT* histograms, IdxT n_bins, const DatasetT& dataset, @@ -41,11 +41,9 @@ template void launchComputeSplitKernel* splits, ObjectiveT& objective, const WorkloadInfo* workload_info, - size_t n_work_items, - bool use_global_memory_histogram, - dim3 grid, - size_t histogram_smem_size, - size_t split_smem_size, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, cudaStream_t builder_stream); } // namespace DT } // namespace ML From 13695e8cb129f623cdc05a5d672de6ce740a3063 Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Mon, 13 Jul 2026 12:48:22 +0200 Subject: [PATCH 5/8] Remove split scoring dynamic shared memory --- .../batched-levelalgo/builder.cuh | 13 ++----- .../kernels/builder_kernels.cuh | 1 - .../kernels/builder_kernels_impl.cuh | 37 ++++++------------- 3 files changed, 15 insertions(+), 36 deletions(-) diff --git a/cpp/src/decisiontree/batched-levelalgo/builder.cuh b/cpp/src/decisiontree/batched-levelalgo/builder.cuh index 347cbe3bbf..c010c6d927 100644 --- a/cpp/src/decisiontree/batched-levelalgo/builder.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/builder.cuh @@ -506,7 +506,7 @@ struct Builder { SharedMemoryConfig computeSharedMemoryConfig() const { - // Dynamic shared memory for the fast path: histogram, copied quantiles, and + // Dynamic shared memory for the histogram fast path: histogram, copied quantiles, and // alignment padding for the kernel's shared-memory layout. auto shared_histogram_size = ML::checked_mul(params.max_n_bins, dataset.num_outputs, sizeof(BinT)); @@ -516,17 +516,11 @@ struct Builder { auto histogram_alignment_smem_size = ML::checked_add(sizeof(BinT), sizeof(DataT)); histogram_dynamic_smem_size = ML::checked_add(histogram_dynamic_smem_size, histogram_alignment_smem_size); - auto split_dynamic_smem_size = - ML::checked_add(shared_quantiles_size, sizeof(DataT)); - auto cdf_scan_smem_size = sizeof(typename cub::BlockScan::TempStorage); auto split_scratch_smem_size = ML::checked_mul(raft::ceildiv(TPB_DEFAULT, raft::WarpSize), sizeof(SplitT)); auto split_static_smem_size = ML::checked_add(cdf_scan_smem_size, split_scratch_smem_size); - auto split_total_smem_size = - ML::checked_add(split_dynamic_smem_size, split_static_smem_size); - auto available_smem = size_t(handle.get_device_properties().sharedMemPerBlock); ASSERT(available_smem >= split_static_smem_size, "Not enough shared memory for RF split bookkeeping."); @@ -534,12 +528,11 @@ struct Builder { // Prefer shared memory when it fits and stays small enough for good occupancy; // otherwise use the global histogram path to avoid launch failure or slowdown. bool use_global_memory_histogram = - histogram_dynamic_smem_size > available_smem || split_total_smem_size > available_smem || + histogram_dynamic_smem_size > available_smem || split_static_smem_size > available_smem || histogram_dynamic_smem_size > tunable_split_histogram_dynamic_smem_limit_bytes; return {use_global_memory_histogram, - use_global_memory_histogram ? 0 : histogram_dynamic_smem_size, - use_global_memory_histogram ? 0 : split_dynamic_smem_size}; + use_global_memory_histogram ? 0 : histogram_dynamic_smem_size}; } void computeSplit(IdxT col, diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh index d013a10a96..24e82deb82 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh @@ -51,7 +51,6 @@ struct WorkloadInfo { struct SharedMemoryConfig { bool use_global_memory_histogram; size_t histogram_dynamic_smem_size; - size_t split_dynamic_smem_size; }; template diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh index b657bf1c8f..d1311f4386 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh @@ -323,11 +323,9 @@ static __global__ void findBestSplitsKernel(typename ObjectiveT::BinT* histogram const IdxT* column_samples, int* mutex, volatile Split* splits, - ObjectiveT objective, - bool use_global_memory_histogram) + ObjectiveT objective) { - using BinT = typename ObjectiveT::BinT; - extern __shared__ char smem[]; + using BinT = typename ObjectiveT::BinT; constexpr int n_split_warps = (TPB + raft::WarpSize - 1) / raft::WarpSize; __shared__ __align__(alignof(Split)) unsigned char split_scratch_storage[sizeof(Split) * n_split_warps]; @@ -346,15 +344,6 @@ static __global__ void findBestSplitsKernel(typename ObjectiveT::BinT* histogram auto* histogram = histograms + histograms_offset; auto* quantiles_for_split = quantiles.quantiles_array + std::size_t(max_n_bins) * col; - if (!use_global_memory_histogram) { - auto* shared_quantiles = alignPointer(smem); - quantiles_for_split = shared_quantiles; - for (IdxT b = threadIdx.x; b < n_bins; b += blockDim.x) { - shared_quantiles[b] = quantiles.quantiles_array[max_n_bins * col + b]; - } - __syncthreads(); - } - for (IdxT c = 0; c < n_classes; ++c) { pdf_to_cdf(histogram + n_bins * c, n_bins); } @@ -399,18 +388,16 @@ void launchComputeSplitKernels(typename ObjectiveT::BinT* histograms, split_smem_config.use_global_memory_histogram); findBestSplitsKernel - <<>>( - histograms, - max_n_bins, - dataset, - quantiles, - work_items, - colStart, - column_samples, - mutex, - splits, - objective, - split_smem_config.use_global_memory_histogram); + <<>>(histograms, + max_n_bins, + dataset, + quantiles, + work_items, + colStart, + column_samples, + mutex, + splits, + objective); } } // namespace DT From 61ede93aef4a0a3e8ed560d672cbade3be5dd54e Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Mon, 13 Jul 2026 14:41:09 +0200 Subject: [PATCH 6/8] Use checked casts for RF split launch grids --- cpp/src/decisiontree/batched-levelalgo/builder.cuh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cpp/src/decisiontree/batched-levelalgo/builder.cuh b/cpp/src/decisiontree/batched-levelalgo/builder.cuh index c010c6d927..7851d60021 100644 --- a/cpp/src/decisiontree/batched-levelalgo/builder.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/builder.cuh @@ -547,8 +547,12 @@ struct Builder { auto n_classes = dataset.num_outputs; // if columns left to be processed lesser than `n_blks_for_cols`, shrink the blocks along dimy auto n_blocks_dimy = std::min(n_blks_for_cols, dataset.n_sampled_cols - col); - dim3 histogram_grid(n_blocks_dimx, n_blocks_dimy, 1); - dim3 split_grid(n_work_items, n_blocks_dimy, 1); + dim3 histogram_grid(ML::narrow_cast(n_blocks_dimx), + ML::narrow_cast(n_blocks_dimy), + 1); + dim3 split_grid(ML::narrow_cast(n_work_items), + ML::narrow_cast(n_blocks_dimy), + 1); auto len_histograms = ML::checked_mul(n_bins, n_classes, n_blocks_dimy, n_work_items); auto histograms_bytes = ML::checked_mul(sizeof(BinT), len_histograms); From b165a70e4c187290c9820ebeb0735c30fcda362f Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Tue, 14 Jul 2026 11:42:04 +0200 Subject: [PATCH 7/8] Apply canonical copyright notices --- cpp/src/decisiontree/batched-levelalgo/builder.cuh | 2 +- .../decisiontree/batched-levelalgo/kernels/builder_kernels.cuh | 2 +- .../batched-levelalgo/kernels/builder_kernels_impl.cuh | 2 +- .../batched-levelalgo/kernels/classification-double.cu | 2 +- .../batched-levelalgo/kernels/classification-float.cu | 2 +- .../decisiontree/batched-levelalgo/kernels/regression-double.cu | 2 +- .../decisiontree/batched-levelalgo/kernels/regression-float.cu | 2 +- .../batched-levelalgo/kernels/weighted-classification-double.cu | 2 +- .../batched-levelalgo/kernels/weighted-classification-float.cu | 2 +- .../batched-levelalgo/kernels/weighted-regression-double.cu | 2 +- .../batched-levelalgo/kernels/weighted-regression-float.cu | 2 +- python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py | 2 +- python/cuml/cuml/decomposition/pca.pyx | 2 +- python/cuml/cuml/decomposition/tsvd.pyx | 2 +- python/cuml/cuml/manifold/t_sne.pyx | 2 +- python/cuml/cuml/neighbors/kneighbors_regressor.pyx | 2 +- python/cuml/tests/test_sklearn_compatibility.py | 2 +- 17 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cpp/src/decisiontree/batched-levelalgo/builder.cuh b/cpp/src/decisiontree/batched-levelalgo/builder.cuh index 7851d60021..de041e212f 100644 --- a/cpp/src/decisiontree/batched-levelalgo/builder.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/builder.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh index 24e82deb82..aa76625af3 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh index d1311f4386..7116151f90 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu index a9ebd850f7..8048e3674e 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu index c995c72e59..3f74f4244b 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu index a57a9372f7..390e4aa8b6 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu index 479333187c..bbe20aad0c 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu index e7ae90c8b4..ccf5e0d6da 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu index 0822787576..414b50804b 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu index 8e37f931c8..484aeeefaf 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu index 717f362197..3353af18e3 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py index a042bb225c..86a27ce1d2 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py @@ -5,7 +5,7 @@ # SPDX-FileCopyrightText: Eric Martin # SPDX-FileCopyrightText: Giorgio Patrini # SPDX-FileCopyrightText: Eric Chang -# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause # Original authors from Sckit-Learn: diff --git a/python/cuml/cuml/decomposition/pca.pyx b/python/cuml/cuml/decomposition/pca.pyx index 5118fda13e..85aa5a6712 100644 --- a/python/cuml/cuml/decomposition/pca.pyx +++ b/python/cuml/cuml/decomposition/pca.pyx @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # import cupy as cp diff --git a/python/cuml/cuml/decomposition/tsvd.pyx b/python/cuml/cuml/decomposition/tsvd.pyx index ddfffccab6..251a1cd30e 100644 --- a/python/cuml/cuml/decomposition/tsvd.pyx +++ b/python/cuml/cuml/decomposition/tsvd.pyx @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # diff --git a/python/cuml/cuml/manifold/t_sne.pyx b/python/cuml/cuml/manifold/t_sne.pyx index 656c3d54cc..b978099a7b 100644 --- a/python/cuml/cuml/manifold/t_sne.pyx +++ b/python/cuml/cuml/manifold/t_sne.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import warnings diff --git a/python/cuml/cuml/neighbors/kneighbors_regressor.pyx b/python/cuml/cuml/neighbors/kneighbors_regressor.pyx index dc94f4a0f4..09082899b1 100644 --- a/python/cuml/cuml/neighbors/kneighbors_regressor.pyx +++ b/python/cuml/cuml/neighbors/kneighbors_regressor.pyx @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # import cupy as cp diff --git a/python/cuml/tests/test_sklearn_compatibility.py b/python/cuml/tests/test_sklearn_compatibility.py index 06aaa8c613..f365140f42 100644 --- a/python/cuml/tests/test_sklearn_compatibility.py +++ b/python/cuml/tests/test_sklearn_compatibility.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # From c8928486b586bde3ce18bab4bf52fe366c301a4b Mon Sep 17 00:00:00 2001 From: Rory Mitchell Date: Tue, 14 Jul 2026 17:00:58 +0200 Subject: [PATCH 8/8] Remove unrelated Python copyright changes --- python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py | 2 +- python/cuml/cuml/decomposition/pca.pyx | 2 +- python/cuml/cuml/decomposition/tsvd.pyx | 2 +- python/cuml/cuml/manifold/t_sne.pyx | 2 +- python/cuml/cuml/neighbors/kneighbors_regressor.pyx | 2 +- python/cuml/tests/test_sklearn_compatibility.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py index 86a27ce1d2..a042bb225c 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py @@ -5,7 +5,7 @@ # SPDX-FileCopyrightText: Eric Martin # SPDX-FileCopyrightText: Giorgio Patrini # SPDX-FileCopyrightText: Eric Chang -# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause # Original authors from Sckit-Learn: diff --git a/python/cuml/cuml/decomposition/pca.pyx b/python/cuml/cuml/decomposition/pca.pyx index 85aa5a6712..5118fda13e 100644 --- a/python/cuml/cuml/decomposition/pca.pyx +++ b/python/cuml/cuml/decomposition/pca.pyx @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # import cupy as cp diff --git a/python/cuml/cuml/decomposition/tsvd.pyx b/python/cuml/cuml/decomposition/tsvd.pyx index 251a1cd30e..ddfffccab6 100644 --- a/python/cuml/cuml/decomposition/tsvd.pyx +++ b/python/cuml/cuml/decomposition/tsvd.pyx @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # diff --git a/python/cuml/cuml/manifold/t_sne.pyx b/python/cuml/cuml/manifold/t_sne.pyx index b978099a7b..656c3d54cc 100644 --- a/python/cuml/cuml/manifold/t_sne.pyx +++ b/python/cuml/cuml/manifold/t_sne.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 import warnings diff --git a/python/cuml/cuml/neighbors/kneighbors_regressor.pyx b/python/cuml/cuml/neighbors/kneighbors_regressor.pyx index 09082899b1..dc94f4a0f4 100644 --- a/python/cuml/cuml/neighbors/kneighbors_regressor.pyx +++ b/python/cuml/cuml/neighbors/kneighbors_regressor.pyx @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # import cupy as cp diff --git a/python/cuml/tests/test_sklearn_compatibility.py b/python/cuml/tests/test_sklearn_compatibility.py index f365140f42..06aaa8c613 100644 --- a/python/cuml/tests/test_sklearn_compatibility.py +++ b/python/cuml/tests/test_sklearn_compatibility.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 #