diff --git a/cpp/src/decisiontree/batched-levelalgo/builder.cuh b/cpp/src/decisiontree/batched-levelalgo/builder.cuh index f557f30de3..de041e212f 100644 --- a/cpp/src/decisiontree/batched-levelalgo/builder.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/builder.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -181,8 +181,6 @@ struct Builder { IdxT* n_nodes; /** buffer of segmented histograms*/ BinT* histograms; - /** threadblock arrival count */ - int* done_count; /** mutex array used for atomically updating best split */ int* mutex; /** best splits for the current batch of nodes */ @@ -292,13 +290,12 @@ struct Builder { size_t max_len_histograms = max_batch * params.max_n_bins * n_blks_for_cols * dataset.num_outputs; - d_wsize += calculateAlignedBytes(sizeof(IdxT)); // n_nodes - d_wsize += calculateAlignedBytes(sizeof(BinT) * max_len_histograms); // histograms - d_wsize += calculateAlignedBytes(sizeof(int) * max_batch * n_blks_for_cols); // done_count - d_wsize += calculateAlignedBytes(sizeof(int) * max_batch); // mutex - d_wsize += calculateAlignedBytes(sizeof(SplitT) * max_batch); // splits - d_wsize += calculateAlignedBytes(sizeof(NodeWorkItem) * max_batch); // d_work_Items - d_wsize += // workload_info + d_wsize += calculateAlignedBytes(sizeof(IdxT)); // n_nodes + d_wsize += calculateAlignedBytes(sizeof(BinT) * max_len_histograms); // histograms + d_wsize += calculateAlignedBytes(sizeof(int) * max_batch); // mutex + d_wsize += calculateAlignedBytes(sizeof(SplitT) * max_batch); // splits + d_wsize += calculateAlignedBytes(sizeof(NodeWorkItem) * max_batch); // d_work_Items + d_wsize += // workload_info calculateAlignedBytes(sizeof(WorkloadInfo) * max_blocks_dimx); d_wsize += calculateAlignedBytes(sizeof(IdxT) * max_batch * dataset.n_sampled_cols); // column_samples @@ -323,8 +320,7 @@ struct Builder { { raft::common::nvtx::range fun_scope( "Builder::assignWorkspace @builder.cuh [batched-levelalgo]"); - auto max_batch = params.max_batch_size; - auto n_col_blks = n_blks_for_cols; + auto max_batch = params.max_batch_size; size_t max_len_histograms = max_batch * (params.max_n_bins) * n_blks_for_cols * dataset.num_outputs; // device @@ -332,8 +328,6 @@ struct Builder { d_wspace += calculateAlignedBytes(sizeof(IdxT)); histograms = reinterpret_cast(d_wspace); d_wspace += calculateAlignedBytes(sizeof(BinT) * max_len_histograms); - done_count = reinterpret_cast(d_wspace); - d_wspace += calculateAlignedBytes(sizeof(int) * max_batch * n_col_blks); mutex = reinterpret_cast(d_wspace); d_wspace += calculateAlignedBytes(sizeof(int) * max_batch); splits = reinterpret_cast(d_wspace); @@ -347,8 +341,6 @@ struct Builder { partition_row_ids = reinterpret_cast(d_wspace); d_wspace += calculateAlignedBytes(sizeof(IdxT) * dataset.n_sampled_rows); - RAFT_CUDA_TRY( - cudaMemsetAsync(done_count, 0, sizeof(int) * max_batch * n_col_blks, builder_stream)); RAFT_CUDA_TRY(cudaMemsetAsync(mutex, 0, sizeof(int) * max_batch, builder_stream)); // host @@ -383,23 +375,19 @@ struct Builder { private: auto updateWorkloadInfo(const std::vector& work_items) { - int n_large_nodes = 0; // large nodes are nodes having training instances larger than block - // size, hence require global memory for histogram construction - int n_blocks_dimx = 0; // gridDim.x required for computeSplitKernel + int n_blocks_dimx = 0; // gridDim.x required for histogram construction for (std::size_t i = 0; i < work_items.size(); i++) { auto item = work_items[i]; int n_blocks_per_node = std::max(raft::ceildiv(item.instances.count, size_t(TPB_DEFAULT)), size_t(1)); - if (n_blocks_per_node > 1) ++n_large_nodes; - for (int b = 0; b < n_blocks_per_node; b++) { - h_workload_info[n_blocks_dimx + b] = {int(i), n_large_nodes - 1, b, n_blocks_per_node}; + h_workload_info[n_blocks_dimx + b] = {int(i), b, n_blocks_per_node}; } n_blocks_dimx += n_blocks_per_node; } raft::update_device(workload_info, h_workload_info, n_blocks_dimx, builder_stream); - return std::make_pair(n_blocks_dimx, n_large_nodes); + return n_blocks_dimx; } auto doSplit(const std::vector& work_items) @@ -461,7 +449,7 @@ struct Builder { cudaMemcpyHostToDevice, builder_stream)); raft::update_device(d_work_items, work_items.data(), work_items.size(), builder_stream); - const auto partition_workload = this->updateWorkloadInfo(work_items); + const auto n_partition_blocks = this->updateWorkloadInfo(work_items); raft::common::nvtx::push_range("nodeSplitKernel @builder.cuh [batched-levelalgo]"); launchNodeSplitKernel(params.min_samples_leaf, params.min_impurity_decrease, @@ -469,7 +457,7 @@ struct Builder { d_work_items, splits, workload_info, - partition_workload.first, + n_partition_blocks, partition_row_ids, builder_stream); RAFT_CUDA_TRY(cudaPeekAtLastError()); @@ -484,17 +472,15 @@ struct Builder { IdxT sample_offset) { initSplit(splits, work_items.size(), builder_stream); - RAFT_CUDA_TRY(cudaMemsetAsync( - done_count, 0, sizeof(int) * params.max_batch_size * n_blks_for_cols, builder_stream)); RAFT_CUDA_TRY(cudaMemsetAsync(mutex, 0, sizeof(int) * params.max_batch_size, builder_stream)); raft::update_device(d_work_items, work_items.data(), work_items.size(), builder_stream); - auto [n_blocks_dimx, n_large_nodes] = this->updateWorkloadInfo(work_items); - auto split_smem_config = computeSplitSharedMemoryConfig(); + auto n_blocks_dimx = this->updateWorkloadInfo(work_items); + auto split_smem_config = computeSharedMemoryConfig(); sampleFeatures(work_items, sampling_seed, sample_offset); for (IdxT c = 0; c < dataset.n_sampled_cols; c += n_blks_for_cols) { - computeSplit(c, n_blocks_dimx, n_large_nodes, work_items.size(), split_smem_config); + computeSplit(c, n_blocks_dimx, work_items.size(), split_smem_config); RAFT_CUDA_TRY(cudaPeekAtLastError()); } raft::update_host(h_splits, splits, work_items.size(), builder_stream); @@ -518,100 +504,78 @@ struct Builder { RAFT_CUDA_TRY(cudaPeekAtLastError()); } - struct SplitSharedMemoryConfig { - bool use_global_memory_histogram; - size_t dynamic_smem_size; - }; - - SplitSharedMemoryConfig computeSplitSharedMemoryConfig() const + SharedMemoryConfig computeSharedMemoryConfig() const { - // Dynamic shared memory for the fast path: histogram, copied quantiles, and + // Dynamic shared memory for the histogram fast path: histogram, copied quantiles, and // alignment padding for the kernel's shared-memory layout. auto shared_histogram_size = ML::checked_mul(params.max_n_bins, dataset.num_outputs, sizeof(BinT)); auto shared_quantiles_size = ML::checked_mul(params.max_n_bins, sizeof(DataT)); - auto shared_dynamic_smem_size = - ML::checked_add(shared_histogram_size, shared_quantiles_size, sizeof(int)); - auto alignment_smem_size = - ML::checked_add(sizeof(DataT), ML::checked_mul(3, sizeof(int))); - shared_dynamic_smem_size = - ML::checked_add(shared_dynamic_smem_size, alignment_smem_size); - - // Dynamic shared memory for the fallback path only needs the per-block done - // flag used by the inter-block completion handshake. - auto global_dynamic_smem_size = ML::checked_add(sizeof(int), sizeof(int)); - - // Static shared memory is reserved by the kernel regardless of where the - // histogram lives. + auto histogram_dynamic_smem_size = + ML::checked_add(shared_histogram_size, shared_quantiles_size); + auto histogram_alignment_smem_size = ML::checked_add(sizeof(BinT), sizeof(DataT)); + histogram_dynamic_smem_size = + ML::checked_add(histogram_dynamic_smem_size, histogram_alignment_smem_size); auto cdf_scan_smem_size = sizeof(typename cub::BlockScan::TempStorage); auto split_scratch_smem_size = ML::checked_mul(raft::ceildiv(TPB_DEFAULT, raft::WarpSize), sizeof(SplitT)); - auto static_smem_size = + auto split_static_smem_size = ML::checked_add(cdf_scan_smem_size, split_scratch_smem_size); - auto available_smem = size_t(handle.get_device_properties().sharedMemPerBlock); - auto global_total_smem_size = - ML::checked_add(global_dynamic_smem_size, static_smem_size); - ASSERT(available_smem >= global_total_smem_size, + ASSERT(available_smem >= split_static_smem_size, "Not enough shared memory for RF split bookkeeping."); // Prefer shared memory when it fits and stays small enough for good occupancy; // otherwise use the global histogram path to avoid launch failure or slowdown. - auto shared_total_smem_size = - ML::checked_add(shared_dynamic_smem_size, static_smem_size); bool use_global_memory_histogram = - shared_total_smem_size > available_smem || - shared_dynamic_smem_size > tunable_split_histogram_dynamic_smem_limit_bytes; + histogram_dynamic_smem_size > available_smem || split_static_smem_size > available_smem || + histogram_dynamic_smem_size > tunable_split_histogram_dynamic_smem_limit_bytes; return {use_global_memory_histogram, - use_global_memory_histogram ? global_dynamic_smem_size : shared_dynamic_smem_size}; + use_global_memory_histogram ? 0 : histogram_dynamic_smem_size}; } void computeSplit(IdxT col, size_t n_blocks_dimx, - size_t n_large_nodes, size_t n_work_items, - const SplitSharedMemoryConfig& split_smem_config) + const SharedMemoryConfig& split_smem_config) { // if no instances to split, return if (n_blocks_dimx == 0) return; raft::common::nvtx::range fun_scope("Builder::computeSplit @builder.cuh [batched-levelalgo]"); - auto n_bins = params.max_n_bins; - auto n_classes = dataset.num_outputs; - auto use_global_memory_histogram = split_smem_config.use_global_memory_histogram; + auto n_bins = params.max_n_bins; + auto n_classes = dataset.num_outputs; // if columns left to be processed lesser than `n_blks_for_cols`, shrink the blocks along dimy auto n_blocks_dimy = std::min(n_blks_for_cols, dataset.n_sampled_cols - col); - dim3 grid(n_blocks_dimx, n_blocks_dimy, 1); - auto histogram_node_count = use_global_memory_histogram ? n_work_items : n_large_nodes; + dim3 histogram_grid(ML::narrow_cast(n_blocks_dimx), + ML::narrow_cast(n_blocks_dimy), + 1); + dim3 split_grid(ML::narrow_cast(n_work_items), + ML::narrow_cast(n_blocks_dimy), + 1); auto len_histograms = - ML::checked_mul(n_bins, n_classes, n_blocks_dimy, histogram_node_count); + ML::checked_mul(n_bins, n_classes, n_blocks_dimy, n_work_items); auto histograms_bytes = ML::checked_mul(sizeof(BinT), len_histograms); RAFT_CUDA_TRY(cudaMemsetAsync(histograms, 0, histograms_bytes, builder_stream)); // create the objective function object ObjectiveT objective(dataset.num_outputs, params.min_samples_leaf, params.split_criterion); - // call the computeSplitKernel - raft::common::nvtx::range kernel_scope("computeSplitKernel @builder.cuh [batched-levelalgo]"); - launchComputeSplitKernel( - histograms, - params.max_n_bins, - params.min_samples_split, - params.max_leaves, - dataset, - quantiles, - d_work_items, - col, - column_samples, - done_count, - mutex, - splits, - objective, - treeid, - workload_info, - seed, - use_global_memory_histogram, - grid, - split_smem_config.dynamic_smem_size, - builder_stream); + // call the compute split kernels + raft::common::nvtx::range kernel_scope("computeSplitKernels @builder.cuh [batched-levelalgo]"); + launchComputeSplitKernels(histograms, + params.max_n_bins, + dataset, + quantiles, + d_work_items, + col, + column_samples, + mutex, + splits, + objective, + workload_info, + histogram_grid, + split_grid, + split_smem_config, + builder_stream); } // Set the leaf value predictions in batch diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh index 882f7fbdcb..aa76625af3 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -42,14 +42,17 @@ struct NodeWorkItem { */ template struct WorkloadInfo { - IdxT nodeid; // Node in the batch on which the threadblock needs to work - IdxT large_nodeid; // counts only large nodes (nodes that require more than one block along x-dim - // for histogram calculation) + IdxT nodeid; // Node in the batch on which the threadblock needs to work IdxT offset_blockid; // Offset threadblock id among all the blocks that are // working on this node IdxT num_blocks; // Total number of blocks that are working on the node }; +struct SharedMemoryConfig { + bool use_global_memory_histogram; + size_t histogram_dynamic_smem_size; +}; + template HDI bool SplitPartitionNotValid(const SplitT& split, IdxT min_samples_leaf, std::size_t num_rows) { @@ -147,26 +150,21 @@ HDI IdxT lower_bound(DataT const* array, IdxT len, DataT element) } template -void launchComputeSplitKernel(typename ObjectiveT::BinT* histograms, - IdxT n_bins, - IdxT min_samples_split, - IdxT max_leaves, - const Dataset& dataset, - const Quantiles& quantiles, - const NodeWorkItem* work_items, - IdxT colStart, - const IdxT* column_samples, - int* done_count, - int* mutex, - volatile Split* splits, - ObjectiveT& objective, - IdxT treeid, - const WorkloadInfo* workload_info, - uint64_t seed, - bool use_global_memory_histogram, - dim3 grid, - size_t smem_size, - cudaStream_t builder_stream); +void launchComputeSplitKernels(typename ObjectiveT::BinT* histograms, + IdxT n_bins, + const Dataset& dataset, + const Quantiles& quantiles, + const NodeWorkItem* work_items, + IdxT colStart, + const IdxT* column_samples, + int* mutex, + volatile Split* splits, + ObjectiveT& objective, + const WorkloadInfo* workload_info, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, + cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh index 4afe592819..7116151f90 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/builder_kernels_impl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once @@ -222,13 +222,10 @@ void launchLeafKernel(ObjectiveT objective, /** * @brief For every threadblock, converts a pdf-histogram to a - * cdf-histogram inplace using inclusive block-sum-scan and returns - * the total_sum - * @return The total sum aggregated over the sumscan, - * as well as the modified cdf-histogram pointer + * cdf-histogram inplace using inclusive block-sum-scan. */ template -DI BinT pdf_to_cdf(BinT* histogram, IdxT n_bins) +DI void pdf_to_cdf(BinT* histogram, IdxT n_bins) { // Blockscan instance preparation typedef cub::BlockScan BlockScan; @@ -246,41 +243,25 @@ DI BinT pdf_to_cdf(BinT* histogram, IdxT n_bins) if (tix < n_bins) { histogram[tix] = result + total_aggregate; } total_aggregate += block_aggregate; } - // return the total sum - return total_aggregate; } template -static __global__ void computeSplitKernel(typename ObjectiveT::BinT* histograms, - IdxT max_n_bins, - IdxT min_samples_split, - IdxT max_leaves, - const Dataset dataset, - const Quantiles quantiles, - const NodeWorkItem* work_items, - IdxT colStart, - const IdxT* column_samples, - int* done_count, - int* mutex, - volatile Split* splits, - ObjectiveT objective, - IdxT treeid, - const WorkloadInfo* workload_info, - uint64_t seed, - bool use_global_memory_histogram) +static __global__ void buildHistogramsKernel(typename ObjectiveT::BinT* histograms, + IdxT max_n_bins, + const Dataset dataset, + const Quantiles quantiles, + const NodeWorkItem* work_items, + IdxT colStart, + const IdxT* column_samples, + ObjectiveT objective, + const WorkloadInfo* workload_info, + bool use_global_memory_histogram) { using BinT = typename ObjectiveT::BinT; - // dynamic shared memory extern __shared__ char smem[]; - constexpr int n_split_warps = (TPB + raft::WarpSize - 1) / raft::WarpSize; - __shared__ __align__(alignof(Split)) unsigned char - split_scratch_storage[sizeof(Split) * n_split_warps]; - auto* split_scratch = reinterpret_cast*>(split_scratch_storage); - // Read workload info for this block WorkloadInfo workload_info_cta = workload_info[blockIdx.x]; IdxT nid = workload_info_cta.nodeid; - IdxT large_nid = workload_info_cta.large_nodeid; const auto work_item = work_items[nid]; auto range_start = work_item.instances.begin; auto range_len = work_item.instances.count; @@ -288,30 +269,23 @@ static __global__ void computeSplitKernel(typename ObjectiveT::BinT* histograms, IdxT offset_blockid = workload_info_cta.offset_blockid; IdxT num_blocks = workload_info_cta.num_blocks; - // obtaining the feature to test split on IdxT colIndex = colStart + blockIdx.y; IdxT col = column_samples[nid * dataset.n_sampled_cols + colIndex]; - - // getting the n_bins for that feature - int n_bins = quantiles.n_bins_array[col]; + int n_bins = quantiles.n_bins_array[col]; auto n_classes = objective.NumClasses(); auto end = range_start + range_len; auto histogram_len = n_bins * n_classes; - auto* histogram = static_cast(nullptr); - auto* shared_done = static_cast(nullptr); + auto histograms_offset = (std::size_t(nid) * gridDim.y + blockIdx.y) * max_n_bins * n_classes; + auto* global_histogram = histograms + histograms_offset; + auto* histogram = global_histogram; auto* quantiles_for_split = quantiles.quantiles_array + std::size_t(max_n_bins) * col; IdxT stride = blockDim.x * num_blocks; IdxT tid = threadIdx.x + offset_blockid * blockDim.x; - if (use_global_memory_histogram) { - auto histograms_offset = (std::size_t(nid) * gridDim.y + blockIdx.y) * max_n_bins * n_classes; - histogram = histograms + histograms_offset; - shared_done = alignPointer(smem); - } else { + if (!use_global_memory_histogram) { histogram = alignPointer(smem); auto* shared_quantiles = alignPointer(histogram + histogram_len); - shared_done = alignPointer(shared_quantiles + n_bins); quantiles_for_split = shared_quantiles; for (IdxT i = threadIdx.x; i < histogram_len; i += blockDim.x) { histogram[i] = BinT(); @@ -319,123 +293,111 @@ static __global__ void computeSplitKernel(typename ObjectiveT::BinT* histograms, for (IdxT b = threadIdx.x; b < n_bins; b += blockDim.x) { shared_quantiles[b] = quantiles.quantiles_array[max_n_bins * col + b]; } + __syncthreads(); } - __syncthreads(); - - // compute pdf histogram for all bins for all classes - - // Must be 64 bit - can easily grow larger than a 32 bit int for (auto i = range_start + tid; i < end; i += stride) { - // each thread works over a data point and strides to the next auto row = dataset.row_ids[i]; auto data = dataset.value(row, col); auto label = dataset.labels[row]; - // `start` is lowest index such that data <= quantiles_for_split[start] IdxT start = lower_bound(quantiles_for_split, n_bins, data); - // ++histogram[start] objective.IncrementHistogram(histogram, n_bins, start, label, dataset, row); } - __syncthreads(); - if (use_global_memory_histogram) { - __threadfence(); // for commit guarantee before the last block scores the split + if (!use_global_memory_histogram) { __syncthreads(); - - bool last = MLCommon::signalDone( - done_count + nid * gridDim.y + blockIdx.y, num_blocks, offset_blockid == 0, shared_done); - if (!last) return; - } else if (num_blocks > 1) { - // Shared-memory histogram path: each block built a partial histogram, so unify those - // partial histograms in global memory before scoring the split. - // update the corresponding global location - auto histograms_offset = - (std::size_t(large_nid) * gridDim.y + blockIdx.y) * max_n_bins * n_classes; for (IdxT i = threadIdx.x; i < histogram_len; i += blockDim.x) { - BinT::AtomicAdd(histograms + histograms_offset + i, histogram[i]); + BinT::AtomicAdd(global_histogram + i, histogram[i]); } + } +} - __threadfence(); // for commit guarantee - __syncthreads(); +template +static __global__ void findBestSplitsKernel(typename ObjectiveT::BinT* histograms, + IdxT max_n_bins, + const Dataset dataset, + const Quantiles quantiles, + const NodeWorkItem* work_items, + IdxT colStart, + const IdxT* column_samples, + int* mutex, + volatile Split* splits, + ObjectiveT objective) +{ + using BinT = typename ObjectiveT::BinT; + constexpr int n_split_warps = (TPB + raft::WarpSize - 1) / raft::WarpSize; + __shared__ __align__(alignof(Split)) unsigned char + split_scratch_storage[sizeof(Split) * n_split_warps]; + auto* split_scratch = reinterpret_cast*>(split_scratch_storage); - // last threadblock will go ahead and compute the best split - bool last = MLCommon::signalDone( - done_count + nid * gridDim.y + blockIdx.y, num_blocks, offset_blockid == 0, shared_done); - // if not the last threadblock, exit - if (!last) return; + IdxT nid = blockIdx.x; + auto work_item = work_items[nid]; + auto range_len = work_item.instances.count; - // store the complete global histogram in shared memory of last block - for (IdxT i = threadIdx.x; i < histogram_len; i += blockDim.x) { - histogram[i] = histograms[histograms_offset + i]; - } + IdxT colIndex = colStart + blockIdx.y; + IdxT col = column_samples[nid * dataset.n_sampled_cols + colIndex]; + int n_bins = quantiles.n_bins_array[col]; - __syncthreads(); - } + auto n_classes = objective.NumClasses(); + auto histograms_offset = (std::size_t(nid) * gridDim.y + blockIdx.y) * max_n_bins * n_classes; + auto* histogram = histograms + histograms_offset; + auto* quantiles_for_split = quantiles.quantiles_array + std::size_t(max_n_bins) * col; - // PDF to CDF inplace in `histogram` for (IdxT c = 0; c < n_classes; ++c) { - // left to right scan operation for scanning - // "lesser-than-or-equal" counts - BinT total_sum = pdf_to_cdf(histogram + n_bins * c, n_bins); - // now, `histogram[n_bins * c + i]` will have count of datapoints of class `c` - // that are less than or equal to `quantiles_for_split[i]`. + pdf_to_cdf(histogram + n_bins * c, n_bins); } __syncthreads(); - // calculate the best candidate bins (one for each thread in the block) in current feature and - // corresponding information gain for splitting Split sp = objective.Gain(histogram, quantiles_for_split, col, range_len, n_bins); __syncthreads(); - // calculate best bins among candidate bins per feature using warp reduce - // then atomically update across features to get best split per node - // (in split[nid]) sp.evalBestSplit(split_scratch, splits + nid, mutex + nid, quantiles_for_split, n_bins); } template -void launchComputeSplitKernel(typename ObjectiveT::BinT* histograms, - IdxT max_n_bins, - IdxT min_samples_split, - IdxT max_leaves, - const Dataset& dataset, - const Quantiles& quantiles, - const NodeWorkItem* work_items, - IdxT colStart, - const IdxT* column_samples, - int* done_count, - int* mutex, - volatile Split* splits, - ObjectiveT& objective, - IdxT treeid, - const WorkloadInfo* workload_info, - uint64_t seed, - bool use_global_memory_histogram, - dim3 grid, - size_t smem_size, - cudaStream_t builder_stream) +void launchComputeSplitKernels(typename ObjectiveT::BinT* histograms, + IdxT max_n_bins, + const Dataset& dataset, + const Quantiles& quantiles, + const NodeWorkItem* work_items, + IdxT colStart, + const IdxT* column_samples, + int* mutex, + volatile Split* splits, + ObjectiveT& objective, + const WorkloadInfo* workload_info, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, + cudaStream_t builder_stream) { - computeSplitKernel - <<>>(histograms, - max_n_bins, - min_samples_split, - max_leaves, - dataset, - quantiles, - work_items, - colStart, - column_samples, - done_count, - mutex, - splits, - objective, - treeid, - workload_info, - seed, - use_global_memory_histogram); + buildHistogramsKernel + <<>>( + histograms, + max_n_bins, + dataset, + quantiles, + work_items, + colStart, + column_samples, + objective, + workload_info, + split_smem_config.use_global_memory_histogram); + + findBestSplitsKernel + <<>>(histograms, + max_n_bins, + dataset, + quantiles, + work_items, + colStart, + column_samples, + mutex, + splits, + objective); } } // namespace DT diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu index 919429375c..8048e3674e 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-double.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -29,26 +29,21 @@ template void launchLeafKernel( cudaStream_t builder_stream); // Explicit instantiations are split across separate .cu files to increase compilation parallelism. -template void launchComputeSplitKernel( +template void launchComputeSplitKernels( BinT* histograms, IdxT n_bins, - IdxT min_samples_split, - IdxT max_leaves, const DatasetT& dataset, const Quantiles& quantiles, const NodeWorkItem* work_items, IdxT colStart, const IdxT* column_samples, - int* done_count, int* mutex, volatile Split* splits, ObjectiveT& objective, - IdxT treeid, const WorkloadInfo* workload_info, - uint64_t seed, - bool use_global_memory_histogram, - dim3 grid, - size_t smem_size, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu index 32560e72cf..3f74f4244b 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/classification-float.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -29,26 +29,21 @@ template void launchLeafKernel( cudaStream_t builder_stream); // Explicit instantiations are split across separate .cu files to increase compilation parallelism. -template void launchComputeSplitKernel( +template void launchComputeSplitKernels( BinT* histograms, IdxT n_bins, - IdxT min_samples_split, - IdxT max_leaves, const DatasetT& dataset, const Quantiles& quantiles, const NodeWorkItem* work_items, IdxT colStart, const IdxT* column_samples, - int* done_count, int* mutex, volatile Split* splits, ObjectiveT& objective, - IdxT treeid, const WorkloadInfo* workload_info, - uint64_t seed, - bool use_global_memory_histogram, - dim3 grid, - size_t smem_size, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu index b9b72a6495..390e4aa8b6 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-double.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -29,26 +29,21 @@ template void launchLeafKernel( cudaStream_t builder_stream); // Explicit instantiations are split across separate .cu files to increase compilation parallelism. -template void launchComputeSplitKernel( +template void launchComputeSplitKernels( BinT* histograms, IdxT n_bins, - IdxT min_samples_split, - IdxT max_leaves, const DatasetT& dataset, const Quantiles& quantiles, const NodeWorkItem* work_items, IdxT colStart, const IdxT* column_samples, - int* done_count, int* mutex, volatile Split* splits, ObjectiveT& objective, - IdxT treeid, const WorkloadInfo* workload_info, - uint64_t seed, - bool use_global_memory_histogram, - dim3 grid, - size_t smem_size, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu index 68d888fd97..bbe20aad0c 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/regression-float.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -29,26 +29,21 @@ template void launchLeafKernel( cudaStream_t builder_stream); // Explicit instantiations are split across separate .cu files to increase compilation parallelism. -template void launchComputeSplitKernel( +template void launchComputeSplitKernels( BinT* histograms, IdxT n_bins, - IdxT min_samples_split, - IdxT max_leaves, const DatasetT& dataset, const Quantiles& quantiles, const NodeWorkItem* work_items, IdxT colStart, const IdxT* column_samples, - int* done_count, int* mutex, volatile Split* splits, ObjectiveT& objective, - IdxT treeid, const WorkloadInfo* workload_info, - uint64_t seed, - bool use_global_memory_histogram, - dim3 grid, - size_t smem_size, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu index 2d9590d097..ccf5e0d6da 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-double.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -29,26 +29,21 @@ template void launchLeafKernel( cudaStream_t builder_stream); // Explicit instantiations are split across separate .cu files to increase compilation parallelism. -template void launchComputeSplitKernel( +template void launchComputeSplitKernels( BinT* histograms, IdxT n_bins, - IdxT min_samples_split, - IdxT max_leaves, const DatasetT& dataset, const Quantiles& quantiles, const NodeWorkItem* work_items, IdxT colStart, const IdxT* column_samples, - int* done_count, int* mutex, volatile Split* splits, ObjectiveT& objective, - IdxT treeid, const WorkloadInfo* workload_info, - uint64_t seed, - bool use_global_memory_histogram, - dim3 grid, - size_t smem_size, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu index f74ff643ac..414b50804b 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-classification-float.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -29,26 +29,21 @@ template void launchLeafKernel( cudaStream_t builder_stream); // Explicit instantiations are split across separate .cu files to increase compilation parallelism. -template void launchComputeSplitKernel( +template void launchComputeSplitKernels( BinT* histograms, IdxT n_bins, - IdxT min_samples_split, - IdxT max_leaves, const DatasetT& dataset, const Quantiles& quantiles, const NodeWorkItem* work_items, IdxT colStart, const IdxT* column_samples, - int* done_count, int* mutex, volatile Split* splits, ObjectiveT& objective, - IdxT treeid, const WorkloadInfo* workload_info, - uint64_t seed, - bool use_global_memory_histogram, - dim3 grid, - size_t smem_size, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu index 442cc9c024..484aeeefaf 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-double.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -29,26 +29,21 @@ template void launchLeafKernel( cudaStream_t builder_stream); // Explicit instantiations are split across separate .cu files to increase compilation parallelism. -template void launchComputeSplitKernel( +template void launchComputeSplitKernels( BinT* histograms, IdxT n_bins, - IdxT min_samples_split, - IdxT max_leaves, const DatasetT& dataset, const Quantiles& quantiles, const NodeWorkItem* work_items, IdxT colStart, const IdxT* column_samples, - int* done_count, int* mutex, volatile Split* splits, ObjectiveT& objective, - IdxT treeid, const WorkloadInfo* workload_info, - uint64_t seed, - bool use_global_memory_histogram, - dim3 grid, - size_t smem_size, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, cudaStream_t builder_stream); } // namespace DT } // namespace ML diff --git a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu index 115d00af22..3353af18e3 100644 --- a/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu +++ b/cpp/src/decisiontree/batched-levelalgo/kernels/weighted-regression-float.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -29,26 +29,21 @@ template void launchLeafKernel( cudaStream_t builder_stream); // Explicit instantiations are split across separate .cu files to increase compilation parallelism. -template void launchComputeSplitKernel( +template void launchComputeSplitKernels( BinT* histograms, IdxT n_bins, - IdxT min_samples_split, - IdxT max_leaves, const DatasetT& dataset, const Quantiles& quantiles, const NodeWorkItem* work_items, IdxT colStart, const IdxT* column_samples, - int* done_count, int* mutex, volatile Split* splits, ObjectiveT& objective, - IdxT treeid, const WorkloadInfo* workload_info, - uint64_t seed, - bool use_global_memory_histogram, - dim3 grid, - size_t smem_size, + dim3 histogram_grid, + dim3 split_grid, + const SharedMemoryConfig& split_smem_config, cudaStream_t builder_stream); } // namespace DT } // namespace ML