diff --git a/cpp/src/randomforest/randomforest.cuh b/cpp/src/randomforest/randomforest.cuh index b03fbed08e..665bdb8b40 100644 --- a/cpp/src/randomforest/randomforest.cuh +++ b/cpp/src/randomforest/randomforest.cuh @@ -20,9 +20,11 @@ #include #include +#include #include #include #include +#include #include #include #include @@ -51,6 +53,11 @@ struct InvalidSampleWeight { __device__ bool operator()(T weight) const { return weight < T(0) || !isfinite(weight); } }; +template +struct NonzeroSampleWeight { + __device__ bool operator()(T weight) const { return weight != T(0); } +}; + // Matches estimator behavior: when bootstrapping is enabled and sample weights exist, // those weights are materialized by drawing bootstrap rows according to them. class RowSampler { @@ -106,34 +113,49 @@ class RowSampler { auto& selected_rows = selected_rows_[stream_id]; + raft::resources stream_resources; + raft::resource::set_cuda_stream(stream_resources, stream); + // Hash these together so per-tree row samples are uncorrelated. auto rs = DT::fnv1a32_basis; rs = DT::fnv1a32(rs, seed_); rs = DT::fnv1a32(rs, tree_id); raft::random::RngState rng_state(rs, raft::random::GenPhilox); - if (bootstrap_) { - raft::resources stream_resources; - raft::resource::set_cuda_stream(stream_resources, stream); - if (use_weighted_bootstrap()) { - auto& weighted_draw_scratch = weighted_draw_scratch_[stream_id]; - raft::random::uniform(stream_resources, - rng_state, - weighted_draw_scratch.data(), - weighted_draw_scratch.size(), - 0.0, - sample_weight_sum_); - thrust::upper_bound(rmm::exec_policy(stream), - sample_weight_cdf_.data(), - sample_weight_cdf_.data() + n_rows_, - weighted_draw_scratch.begin(), - weighted_draw_scratch.end(), - selected_rows.begin()); - } else { - raft::random::uniformInt( - stream_resources, rng_state, selected_rows.data(), selected_rows.size(), 0, n_rows_); - } + if (use_weighted_bootstrap()) { + // Draw bootstrap rows according to sample weights. + auto& weighted_draw_scratch = weighted_draw_scratch_[stream_id]; + raft::random::uniform(stream_resources, + rng_state, + weighted_draw_scratch.data(), + weighted_draw_scratch.size(), + 0.0, + sample_weight_sum_); + thrust::upper_bound(rmm::exec_policy(stream), + sample_weight_cdf_.data(), + sample_weight_cdf_.data() + n_rows_, + weighted_draw_scratch.begin(), + weighted_draw_scratch.end(), + selected_rows.begin()); + } else if (bootstrap_) { + // Draw bootstrap rows uniformly when there are no sample weights. + raft::random::uniformInt( + stream_resources, rng_state, selected_rows.data(), selected_rows.size(), 0, n_rows_); + } else if (sample_weight_ != nullptr) { + // Remove zero-weight rows from the non-bootstrap row set. + selected_rows.resize(n_sampled_rows_, stream); + auto rows_begin = thrust::make_counting_iterator(0); + auto selected_rows_end = thrust::copy_if(rmm::exec_policy(stream), + rows_begin, + rows_begin + n_rows_, + sample_weight_, + selected_rows.begin(), + NonzeroSampleWeight{}); + auto n_selected = selected_rows_end - selected_rows.begin(); + ASSERT(n_selected > 0, "sample_weight values must contain at least one positive value"); + selected_rows.resize(n_selected, stream); } else { + selected_rows.resize(n_sampled_rows_, stream); thrust::sequence(rmm::exec_policy(stream), selected_rows.begin(), selected_rows.end()); } @@ -155,7 +177,7 @@ class RowSampler { thrust::fill(rmm::exec_policy(stream), tree_mask, tree_mask + n_rows_, false); thrust::scatter(rmm::exec_policy(stream), thrust::make_constant_iterator(true), - thrust::make_constant_iterator(true) + n_sampled_rows_, + thrust::make_constant_iterator(true) + selected_rows.size(), selected_rows.data(), tree_mask); } @@ -256,7 +278,8 @@ class RandomForest { * (n_trees * n_rows), only populated if a non-null pointer is provided. * @param[in] sample_weight: optional device pointer to per-row sample weights. With bootstrap * enabled, rows are sampled with probability proportional to these weights and the sampled - * counts drive tree training. Without bootstrap, weights are used for impurity/objective math. + * counts drive tree training. Without bootstrap, zero-weight rows are removed from the tree + * row set and remaining weights are used for impurity/objective math. */ void fit(const raft::handle_t& user_handle, const T* input, @@ -310,7 +333,7 @@ class RandomForest { /* Build individual tree in the forest. - input is a pointer to orig data that have n_cols features and n_rows rows. - - n_sampled_rows: # rows sampled for tree's bootstrap sample. + - n_sampled_rows: # rows sampled or retained for this tree. - sorted_selected_rows: points to a list of row #s (w/ n_sampled_rows elements) used to build the bootstrapped sample. Expectation: Each tree node will contain (a) # n_sampled_rows and diff --git a/cpp/tests/sg/rf_test.cu b/cpp/tests/sg/rf_test.cu index a2a08572d4..dc8b4bab2d 100644 --- a/cpp/tests/sg/rf_test.cu +++ b/cpp/tests/sg/rf_test.cu @@ -1643,7 +1643,7 @@ TEST(RfWeightedTest, RegressionRootLeafUsesWeights) const auto& tree = *forest->trees[0]; ASSERT_EQ(tree.sparsetree.size(), 1); EXPECT_TRUE(tree.sparsetree[0].IsLeaf()); - EXPECT_EQ(tree.sparsetree[0].InstanceCount(), 3); + EXPECT_EQ(tree.sparsetree[0].InstanceCount(), 2); ASSERT_EQ(tree.vector_leaf.size(), 1); EXPECT_NEAR(tree.vector_leaf[0], 7.5f, 1e-6f); } @@ -1718,7 +1718,7 @@ TEST(RfWeightedTest, ZeroWeightSamplesDoNotCreatePositiveWeightSplit) const auto& tree = *forest->trees[0]; ASSERT_EQ(tree.sparsetree.size(), 1); EXPECT_TRUE(tree.sparsetree[0].IsLeaf()); - EXPECT_EQ(tree.sparsetree[0].InstanceCount(), 4); + EXPECT_EQ(tree.sparsetree[0].InstanceCount(), 2); ASSERT_EQ(tree.vector_leaf.size(), 2); EXPECT_NEAR(tree.vector_leaf[0], 0.0f, 1e-6f); EXPECT_NEAR(tree.vector_leaf[1], 1.0f, 1e-6f); diff --git a/python/cuml/cuml/accel/_overrides/sklearn/ensemble.py b/python/cuml/cuml/accel/_overrides/sklearn/ensemble.py index 72dcb353ee..67a6f7abfb 100644 --- a/python/cuml/cuml/accel/_overrides/sklearn/ensemble.py +++ b/python/cuml/cuml/accel/_overrides/sklearn/ensemble.py @@ -25,9 +25,6 @@ def _check_inputs(self, X, y=None, sample_weight=None): ) from None raise - if sample_weight is not None: - raise UnsupportedOnGPU("`sample_weight` is not supported") - if y is not None: y = check_array( y, @@ -48,7 +45,7 @@ def _check_inputs(self, X, y=None, sample_weight=None): def _gpu_fit(self, X, y, sample_weight=None): self._check_inputs(X, y, sample_weight=sample_weight) - return self._gpu.fit(X, y) + return self._gpu.fit(X, y, sample_weight=sample_weight) def _gpu_predict(self, X): self._check_inputs(X) @@ -56,7 +53,7 @@ def _gpu_predict(self, X): def _gpu_score(self, X, y, sample_weight=None): self._check_inputs(X, y, sample_weight=sample_weight) - return self._gpu.score(X, y) + return self._gpu.score(X, y, sample_weight=sample_weight) class RandomForestRegressor(ProxyBase, _RandomForestMixin): diff --git a/python/cuml/cuml/common/classification.py b/python/cuml/cuml/common/classification.py index ff71baf2e2..05c3e93251 100644 --- a/python/cuml/cuml/common/classification.py +++ b/python/cuml/cuml/common/classification.py @@ -1,5 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 +from collections.abc import Mapping + import cudf import cupy as cp import numpy as np @@ -111,6 +113,20 @@ def decode_labels(y_encoded, classes, output_type="cupy", index=None): ) +def validate_class_weight(class_weight): + if class_weight is None: + return + if isinstance(class_weight, str) and class_weight == "balanced": + return + if isinstance(class_weight, Mapping): + return + + raise ValueError( + "class_weight must be a dict, 'balanced', or None; " + f"got {class_weight!r}" + ) + + def process_class_weight( classes, y_ind, @@ -152,6 +168,8 @@ def process_class_weight( sample_weight: cp.ndarray or None The resulting sample weights, or None if uniformly weighted. """ + validate_class_weight(class_weight) + n_classes = len(classes) if dtype is None: dtype = getattr(sample_weight, "dtype", np.float32) diff --git a/python/cuml/cuml/dask/ensemble/randomforestclassifier.py b/python/cuml/cuml/dask/ensemble/randomforestclassifier.py index 3dc1a7e3e4..3825bf5136 100755 --- a/python/cuml/cuml/dask/ensemble/randomforestclassifier.py +++ b/python/cuml/cuml/dask/ensemble/randomforestclassifier.py @@ -155,7 +155,13 @@ def _construct_rf(n_estimators, random_state, **kwargs): n_estimators=n_estimators, random_state=random_state, **kwargs ) - def fit(self, X, y, convert_dtype="deprecated", broadcast_data=False): + def fit( + self, + X, + y, + convert_dtype="deprecated", + broadcast_data=False, + ): """ Fit the input data with a Random Forest classifier @@ -207,7 +213,6 @@ def fit(self, X, y, convert_dtype="deprecated", broadcast_data=False): When set to True, the whole dataset is broadcasted to train the workers, otherwise each worker is trained on its partition - """ # Handle both Dask Arrays and Dask Series/DataFrames if isinstance(y, dask.array.Array): diff --git a/python/cuml/cuml/dask/ensemble/randomforestregressor.py b/python/cuml/cuml/dask/ensemble/randomforestregressor.py index 99b16d6df0..4d6d369ed8 100755 --- a/python/cuml/cuml/dask/ensemble/randomforestregressor.py +++ b/python/cuml/cuml/dask/ensemble/randomforestregressor.py @@ -138,7 +138,13 @@ def _construct_rf(n_estimators, random_state, **kwargs): n_estimators=n_estimators, random_state=random_state, **kwargs ) - def fit(self, X, y, convert_dtype="deprecated", broadcast_data=False): + def fit( + self, + X, + y, + convert_dtype="deprecated", + broadcast_data=False, + ): """ Fit the input data with a Random Forest regression model @@ -186,7 +192,6 @@ def fit(self, X, y, convert_dtype="deprecated", broadcast_data=False): When set to True, the whole dataset is broadcasted to train the workers, otherwise each worker is trained on its partition - """ self.internal_model = None self._fit( diff --git a/python/cuml/cuml/ensemble/randomforest_common.pyx b/python/cuml/cuml/ensemble/randomforest_common.pyx index 14083ff2e3..2f9683e7d8 100644 --- a/python/cuml/cuml/ensemble/randomforest_common.pyx +++ b/python/cuml/cuml/ensemble/randomforest_common.pyx @@ -80,7 +80,8 @@ cdef extern from "cuml/ensemble/randomforest.hpp" namespace "ML" nogil: RF_params params, bool* bootstrap_masks, T* feature_importances, - level_enum verbosity + level_enum verbosity, + const double* sample_weight ) except + cdef void fit_treelite[T, L]( @@ -93,7 +94,8 @@ cdef extern from "cuml/ensemble/randomforest.hpp" namespace "ML" nogil: RF_params params, bool* bootstrap_masks, T* feature_importances, - level_enum verbosity + level_enum verbosity, + const double* sample_weight ) except + @@ -455,12 +457,15 @@ class BaseRandomForestModel(InteropMixin, Base): handle=get_handle(), ) - def _fit_forest(self, X, y): + def _fit_forest(self, X, y, sample_weight=None): cdef bool is_classifier = self._estimator_type == "classifier" cdef bool is_float32 = X.dtype == np.float32 cdef uintptr_t X_ptr = X.data.ptr cdef uintptr_t y_ptr = y.data.ptr + cdef uintptr_t sample_weight_ptr = ( + 0 if sample_weight is None else sample_weight.data.ptr + ) cdef int n_rows = X.shape[0] cdef int n_cols = X.shape[1] cdef level_enum verbose = self._verbose_level @@ -578,7 +583,8 @@ class BaseRandomForestModel(InteropMixin, Base): params, bootstrap_masks_ptr, feature_importances_ptr, - verbose + verbose, + sample_weight_ptr ) else: fit_treelite( @@ -592,7 +598,8 @@ class BaseRandomForestModel(InteropMixin, Base): params, bootstrap_masks_ptr, feature_importances_ptr, - verbose + verbose, + sample_weight_ptr ) else: if is_float32: @@ -606,7 +613,8 @@ class BaseRandomForestModel(InteropMixin, Base): params, bootstrap_masks_ptr, feature_importances_ptr, - verbose + verbose, + sample_weight_ptr ) else: fit_treelite( @@ -619,7 +627,8 @@ class BaseRandomForestModel(InteropMixin, Base): params, bootstrap_masks_ptr, feature_importances_ptr, - verbose + verbose, + sample_weight_ptr ) # XXX: Theoretically we could wrap `tl_handle` with `treelite.Model` to diff --git a/python/cuml/cuml/ensemble/randomforestclassifier.py b/python/cuml/cuml/ensemble/randomforestclassifier.py index 8861c25722..6fdec188e5 100644 --- a/python/cuml/cuml/ensemble/randomforestclassifier.py +++ b/python/cuml/cuml/ensemble/randomforestclassifier.py @@ -6,7 +6,11 @@ import cuml.internals import cuml.internals.nvtx as nvtx from cuml.common.array_descriptor import CumlArrayDescriptor -from cuml.common.classification import decode_labels +from cuml.common.classification import ( + decode_labels, + process_class_weight, + validate_class_weight, +) from cuml.common.doc_utils import generate_docstring, insert_into_docstring from cuml.ensemble.randomforest_common import BaseRandomForestModel from cuml.internals.array import CumlArray @@ -122,6 +126,9 @@ class RandomForestClassifier(ClassifierMixin, BaseRandomForestModel): accuracy. Only available if ``bootstrap=True``. The out-of-bag estimate provides a way to evaluate the model without requiring a separate validation set. The OOB score is computed using accuracy. + class_weight : dict, 'balanced', or None, default=None + Weights associated with classes. If ``'balanced'``, class weights are + computed from the training labels. verbose : int or boolean, default=False Sets logging level. It must be one of `cuml.common.logger.level_*`. See :ref:`verbosity-levels` for more info. @@ -165,11 +172,26 @@ class RandomForestClassifier(ClassifierMixin, BaseRandomForestModel): _cpu_class_path = "sklearn.ensemble.RandomForestClassifier" + @classmethod + def _get_param_names(cls): + return [*super()._get_param_names(), "class_weight"] + @classmethod def _params_from_cpu(cls, model): - if model.class_weight is not None: - raise UnsupportedOnGPU("`class_weight` is not supported") - return super()._params_from_cpu(model) + if model.class_weight == "balanced_subsample": + raise UnsupportedOnGPU( + "`class_weight='balanced_subsample'` is not supported" + ) + return { + "class_weight": model.class_weight, + **super()._params_from_cpu(model), + } + + def _params_to_cpu(self): + return { + **super()._params_to_cpu(), + "class_weight": self.class_weight, + } def _attrs_from_cpu(self, model): return { @@ -210,6 +232,7 @@ def __init__( random_state=None, n_streams=4, oob_score=False, + class_weight=None, verbose=False, output_type=None, ): @@ -232,6 +255,7 @@ def __init__( verbose=verbose, output_type=output_type, ) + self.class_weight = class_weight @nvtx.annotate( message="fit RF-Classifier @randomforestclassifier.pyx", @@ -240,25 +264,36 @@ def __init__( @generate_docstring(y="dense_intdtype") @cuml.internals.reflect(reset=True) def fit( - self, X, y, *, convert_dtype="deprecated" + self, X, y, sample_weight=None, *, convert_dtype="deprecated" ) -> "RandomForestClassifier": """ Perform Random Forest Classification on the input data """ - X, y, classes = check_inputs( + validate_class_weight(self.class_weight) + + X, y, sample_weight, classes = check_inputs( self, X, y, + sample_weight, dtype=("float32", "float64"), convert_dtype=convert_dtype, order="F", y_dtype="int32", + sample_weight_dtype="float64", return_classes=True, reset=True, ) self.classes_ = classes self.n_classes_ = len(classes) - return self._fit_forest(X, y) + _, sample_weight = process_class_weight( + classes, + y, + class_weight=self.class_weight, + sample_weight=sample_weight, + dtype=np.float64, + ) + return self._fit_forest(X, y, sample_weight=sample_weight) @nvtx.annotate( message="predict RF-Classifier @randomforestclassifier.pyx", @@ -459,6 +494,7 @@ def score( self, X, y, + sample_weight=None, *, threshold=0.5, convert_dtype="deprecated", @@ -473,6 +509,8 @@ def score( ---------- X : {} y : {} + sample_weight : array-like, shape=(n_samples,), default=None + Sample weights for weighted mean accuracy. threshold : float (default = 0.5) Threshold used for classification predictions convert_dtype : bool, default="deprecated" @@ -508,4 +546,4 @@ def score( default_chunk_size=default_chunk_size, align_bytes=align_bytes, ) - return accuracy_score(y, y_pred) + return accuracy_score(y, y_pred, sample_weight=sample_weight) diff --git a/python/cuml/cuml/ensemble/randomforestregressor.py b/python/cuml/cuml/ensemble/randomforestregressor.py index 7a13116026..d7c1fa1f98 100644 --- a/python/cuml/cuml/ensemble/randomforestregressor.py +++ b/python/cuml/cuml/ensemble/randomforestregressor.py @@ -202,22 +202,24 @@ def __init__( @generate_docstring() @reflect(reset=True) def fit( - self, X, y, *, convert_dtype="deprecated" + self, X, y, sample_weight=None, *, convert_dtype="deprecated" ) -> "RandomForestRegressor": """ Perform Random Forest Regression on the input data """ - X, y = check_inputs( + X, y, sample_weight = check_inputs( self, X, y, + sample_weight, dtype=("float32", "float64"), convert_dtype=convert_dtype, order="F", + sample_weight_dtype="float64", reset=True, ) - return self._fit_forest(X, y) + return self._fit_forest(X, y, sample_weight=sample_weight) @nvtx.annotate( message="predict RF-Regressor @randomforestclassifier.pyx", @@ -304,6 +306,7 @@ def score( self, X, y, + sample_weight=None, *, convert_dtype="deprecated", layout="depth_first", @@ -317,6 +320,8 @@ def score( ---------- X : {} y : {} + sample_weight : array-like, shape=(n_samples,), default=None + Sample weights for weighted R^2. convert_dtype : bool, default="deprecated" .. deprecated:: 26.08 `convert_dtype` was deprecated in version 26.08 and will be @@ -348,4 +353,4 @@ def score( default_chunk_size=default_chunk_size, align_bytes=align_bytes, ) - return r2_score(y, y_pred) + return r2_score(y, y_pred, sample_weight=sample_weight) diff --git a/python/cuml/cuml_accel_tests/integration/test_rf_classifier.py b/python/cuml/cuml_accel_tests/integration/test_rf_classifier.py index c0ab9f0863..efdc246f9c 100644 --- a/python/cuml/cuml_accel_tests/integration/test_rf_classifier.py +++ b/python/cuml/cuml_accel_tests/integration/test_rf_classifier.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # @@ -168,6 +168,33 @@ def test_rf_class_weight(classification_data, class_weight): _ = accuracy_score(y, clf.predict(X)) +@pytest.mark.parametrize("bootstrap", [False, True]) +def test_rf_sample_weight(bootstrap): + zero_weight_X = np.linspace(-3.0, -1.0, 72).reshape(-1, 1) + one_weight_X = np.linspace(1.0, 3.0, 72).reshape(-1, 1) + X = np.vstack([zero_weight_X, one_weight_X]) + y = np.array([0] * len(zero_weight_X) + [1] * len(one_weight_X)) + sample_weight = np.array( + [0.0] * len(zero_weight_X) + [1.0] * len(one_weight_X) + ) + probe = np.linspace(-3.5, 3.5, 17).reshape(-1, 1) + + clf = RandomForestClassifier( + n_estimators=3, + bootstrap=bootstrap, + max_depth=3, + max_features=1.0, + random_state=42, + ) + clf.fit(X, y, sample_weight=sample_weight) + + expected = np.ones(probe.shape[0], dtype=y.dtype) + np.testing.assert_array_equal(clf.predict(probe), expected) + assert clf.score(X, y, sample_weight=sample_weight) == pytest.approx( + accuracy_score(y, clf.predict(X), sample_weight=sample_weight) + ) + + @pytest.mark.parametrize("ccp_alpha", [0.0, 0.1]) def test_rf_ccp_alpha(classification_data, ccp_alpha): X, y = classification_data diff --git a/python/cuml/cuml_accel_tests/integration/test_rf_regressor.py b/python/cuml/cuml_accel_tests/integration/test_rf_regressor.py index 4b62a4ce46..ef4f2a09c3 100644 --- a/python/cuml/cuml_accel_tests/integration/test_rf_regressor.py +++ b/python/cuml/cuml_accel_tests/integration/test_rf_regressor.py @@ -1,8 +1,9 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # +import numpy as np import pytest from sklearn.datasets import make_regression from sklearn.ensemble import RandomForestRegressor @@ -73,6 +74,33 @@ def test_rf_min_samples_leaf_reg(regression_data, min_samples_leaf): _ = r2_score(y, reg.predict(X)) +@pytest.mark.parametrize("bootstrap", [False, True]) +def test_rf_sample_weight_reg(bootstrap): + zero_weight_X = np.linspace(-3.0, -1.0, 72).reshape(-1, 1) + one_weight_X = np.linspace(1.0, 3.0, 72).reshape(-1, 1) + X = np.vstack([zero_weight_X, one_weight_X]) + y = np.array([-100.0] * len(zero_weight_X) + [4.0] * len(one_weight_X)) + sample_weight = np.array( + [0.0] * len(zero_weight_X) + [1.0] * len(one_weight_X) + ) + probe = np.linspace(-3.5, 3.5, 17).reshape(-1, 1) + + reg = RandomForestRegressor( + n_estimators=3, + bootstrap=bootstrap, + max_depth=3, + max_features=1.0, + random_state=42, + ) + reg.fit(X, y, sample_weight=sample_weight) + + expected = np.full(probe.shape[0], 4.0) + np.testing.assert_allclose(reg.predict(probe), expected) + assert reg.score(X, y, sample_weight=sample_weight) == pytest.approx( + r2_score(y, reg.predict(X), sample_weight=sample_weight) + ) + + @pytest.mark.parametrize("min_weight_fraction_leaf", [0.0, 0.1]) def test_rf_min_weight_fraction_leaf_reg( regression_data, min_weight_fraction_leaf diff --git a/python/cuml/tests/test_random_forest.py b/python/cuml/tests/test_random_forest.py index b1577b474a..3659ecd85f 100644 --- a/python/cuml/tests/test_random_forest.py +++ b/python/cuml/tests/test_random_forest.py @@ -28,7 +28,9 @@ mean_squared_error, mean_tweedie_deviance, ) +from sklearn.metrics import r2_score as sk_r2_score from sklearn.model_selection import train_test_split +from sklearn.utils.class_weight import compute_sample_weight import cuml from cuml.ensemble import RandomForestClassifier as curfc @@ -219,6 +221,7 @@ def test_default_parameters(): for name in ["max_features", "split_criterion"]: reg_params.pop(name) clf_params.pop(name) + clf_params.pop("class_weight") # The rest are the same assert reg_params == clf_params @@ -249,6 +252,216 @@ def test_rf_invalid_n_streams(estimator, n_streams, error_type): estimator(n_streams=n_streams).fit(X, y) +def _zero_weight_two_cluster_data(datatype, n_zero=8, n_one=8): + zero_weight_X = np.linspace(-3.0, -1.0, n_zero, dtype=datatype).reshape( + -1, 1 + ) + one_weight_X = np.linspace(1.0, 3.0, n_one, dtype=datatype).reshape(-1, 1) + X = np.vstack([zero_weight_X, one_weight_X]) + y = np.array( + [0] * len(zero_weight_X) + [1] * len(one_weight_X), + dtype=np.int32, + ) + sample_weight = y.astype(np.float64) + probe = np.linspace(-3.5, 3.5, 17, dtype=datatype).reshape(-1, 1) + return X, y, sample_weight, probe + + +def _sample_weight_rf_params(X, bootstrap): + return dict( + n_estimators=3, + bootstrap=bootstrap, + max_depth=3, + max_features=1.0, + n_bins=X.shape[0], + min_samples_leaf=1, + min_samples_split=2, + random_state=0, + n_streams=1, + ) + + +def _cuml_preds(model, X): + return cp.asnumpy(cp.asarray(model.predict(X))) + + +def _sklearn_fit_params(cuml_model): + params = cuml_model._params_to_cpu() + if params["max_samples"] == 1.0: + params["max_samples"] = None + return params + + +@pytest.mark.parametrize("bootstrap", [False, True]) +@pytest.mark.parametrize("datatype", [np.float32, np.float64]) +def test_rf_classifier_sample_weight_zero_distribution_matches_sklearn( + datatype, bootstrap +): + X, y, sample_weight, probe = _zero_weight_two_cluster_data(datatype) + cuml_model = curfc(**_sample_weight_rf_params(X, bootstrap)) + sk_model = skrfc(**_sklearn_fit_params(cuml_model)) + + cuml_model.fit(X, y, sample_weight=sample_weight) + sk_model.fit(X, y, sample_weight=sample_weight) + + cuml_preds = _cuml_preds(cuml_model, probe) + np.testing.assert_array_equal(cuml_preds, sk_model.predict(probe)) + np.testing.assert_array_equal( + cuml_preds, np.ones(probe.shape[0], dtype=np.int32) + ) + + score = cuml_model.score(X, y, sample_weight=sample_weight) + assert score == pytest.approx( + accuracy_score(y, sk_model.predict(X), sample_weight=sample_weight) + ) + + +@pytest.mark.parametrize("bootstrap", [False, True]) +@pytest.mark.parametrize("datatype", [np.float32, np.float64]) +def test_rf_regressor_sample_weight_zero_distribution_matches_sklearn( + datatype, bootstrap +): + X, y, sample_weight, probe = _zero_weight_two_cluster_data(datatype) + cuml_model = curfr(**_sample_weight_rf_params(X, bootstrap)) + sk_model = skrfr(**_sklearn_fit_params(cuml_model)) + + cuml_model.fit(X, y, sample_weight=sample_weight) + sk_model.fit(X, y, sample_weight=sample_weight) + + cuml_preds = _cuml_preds(cuml_model, probe) + np.testing.assert_allclose( + cuml_preds, sk_model.predict(probe), rtol=1e-6, atol=1e-6 + ) + np.testing.assert_allclose( + cuml_preds, + np.ones(probe.shape[0]), + rtol=1e-6, + atol=1e-6, + ) + + score = cuml_model.score(X, y, sample_weight=sample_weight) + assert score == pytest.approx( + sk_r2_score(y, sk_model.predict(X), sample_weight=sample_weight), + abs=1e-6, + ) + + +@pytest.mark.parametrize("datatype", [np.float32, np.float64]) +def test_rf_regressor_sample_weight_min_samples_leaf_matches_sklearn(datatype): + X = np.array([[0.0], [1.0], [2.0]], dtype=datatype) + y = np.array([0.0, 0.0, 10.0], dtype=datatype) + sample_weight = np.array([1.0, 1.0, 1000.0], dtype=np.float64) + + cuml_model = curfr( + n_estimators=1, + bootstrap=False, + max_depth=2, + max_features=1.0, + n_bins=3, + min_samples_leaf=2, + min_samples_split=2, + random_state=0, + n_streams=1, + ) + sk_model = skrfr(**_sklearn_fit_params(cuml_model)) + + cuml_model.fit(X, y, sample_weight=sample_weight) + sk_model.fit(X, y, sample_weight=sample_weight) + + np.testing.assert_allclose( + _cuml_preds(cuml_model, X), sk_model.predict(X), rtol=1e-6, atol=1e-6 + ) + + +def test_rf_classifier_balanced_subsample_rejected_before_fit_state(): + X = np.array([[0.0], [1.0]], dtype=np.float32) + y = np.array([0, 1], dtype=np.int32) + clf = curfc( + n_estimators=1, + max_depth=1, + n_bins=2, + n_streams=1, + class_weight="balanced_subsample", + ) + + with pytest.raises(ValueError, match="class_weight"): + clf.fit(X, y) + + assert not hasattr(clf, "classes_") + assert not hasattr(clf, "n_classes_") + assert not hasattr(clf, "n_features_in_") + + +@pytest.mark.parametrize( + "class_weight,match", + [ + pytest.param("not-balanced", "class_weight", id="invalid-string"), + pytest.param( + {0: 1.0, 2: 1.0}, "not in class_weight", id="missing-class" + ), + ], +) +def test_rf_classifier_invalid_class_weight_raises(class_weight, match): + X = np.array([[0.0], [1.0]], dtype=np.float32) + y = np.array([0, 1], dtype=np.int32) + clf = curfc( + n_estimators=1, + max_depth=1, + n_bins=2, + n_streams=1, + class_weight=class_weight, + ) + + with pytest.raises(ValueError, match=match): + clf.fit(X, y) + + +@pytest.mark.parametrize( + "class_weight", + [ + pytest.param({0: 0.0, 1: 1.0}, id="zero-class-zero"), + pytest.param("balanced", id="balanced"), + ], +) +@pytest.mark.parametrize("bootstrap", [False, True]) +@pytest.mark.parametrize("datatype", [np.float32, np.float64]) +def test_rf_classifier_class_weight_matches_sklearn( + datatype, bootstrap, class_weight +): + n_zero, n_one = (12, 4) if class_weight == "balanced" else (8, 8) + X, y, _, probe = _zero_weight_two_cluster_data( + datatype, n_zero=n_zero, n_one=n_one + ) + sample_weight = compute_sample_weight(class_weight, y).astype(np.float64) + params = _sample_weight_rf_params(X, bootstrap) + + class_weight_model = curfc(**params, class_weight=class_weight) + sample_weight_model = curfc(**params) + sk_model = skrfc(**_sklearn_fit_params(class_weight_model)) + + class_weight_model.fit(X, y) + sample_weight_model.fit(X, y, sample_weight=sample_weight) + sk_model.fit(X, y) + + class_weight_preds = _cuml_preds(class_weight_model, probe) + np.testing.assert_array_equal( + class_weight_preds, _cuml_preds(sample_weight_model, probe) + ) + + if class_weight == "balanced": + assert np.unique(sample_weight).size == 2 + np.testing.assert_array_equal( + _cuml_preds(class_weight_model, X), sk_model.predict(X) + ) + else: + np.testing.assert_array_equal( + class_weight_preds, sk_model.predict(probe) + ) + np.testing.assert_array_equal( + class_weight_preds, np.ones(probe.shape[0], dtype=np.int32) + ) + + @pytest.mark.parametrize("max_depth", [2, 4]) @pytest.mark.parametrize( "split_criterion", ["poisson", "gamma", "inverse_gaussian"] @@ -937,7 +1150,6 @@ def test_rf_regression_with_identical_labels(): # with only the root node. model = curfr( max_features=1.0, - max_samples=1.0, n_bins=5, bootstrap=False, split_criterion="mse", diff --git a/python/cuml/tests/test_sklearn_compatibility.py b/python/cuml/tests/test_sklearn_compatibility.py index dc8715fca8..5709ce8ac4 100644 --- a/python/cuml/tests/test_sklearn_compatibility.py +++ b/python/cuml/tests/test_sklearn_compatibility.py @@ -190,8 +190,24 @@ def _all_cuml_estimators(): Ridge: { "check_non_transformer_estimators_n_iter": "Ridge `n_iter_` may be `None`", }, + RandomForestClassifier: { + "check_sample_weight_equivalence_on_dense_data": ( + "RandomForest uses quantile-binned splits, so sample weighting is " + "not equivalent to duplicating rows" + ), + "check_sample_weight_equivalence_on_sparse_data": ( + "RandomForestClassifier does not handle sparse data" + ), + }, RandomForestRegressor: { "check_regressor_data_not_an_array": "RandomForestRegressor does not handle non-array data", + "check_sample_weight_equivalence_on_dense_data": ( + "RandomForest uses quantile-binned splits, so sample weighting is " + "not equivalent to duplicating rows" + ), + "check_sample_weight_equivalence_on_sparse_data": ( + "RandomForestRegressor does not handle sparse data" + ), }, KNeighborsRegressor: { "check_supervised_y_2d": "KNeighborsRegressor does not handle 2D y",