From de5b388dc61bd0545b13799f7c9e2eaadae9b4d1 Mon Sep 17 00:00:00 2001 From: Divye Gala Date: Thu, 7 May 2026 00:14:54 +0000 Subject: [PATCH] add dim checker --- python/cuml/cuml/cluster/agglomerative.pyx | 6 ++ python/cuml/cuml/cluster/dbscan.pyx | 7 ++ python/cuml/cuml/cluster/hdbscan/hdbscan.pyx | 19 +++++ python/cuml/cuml/cluster/kmeans.pyx | 45 +++++++--- .../cuml/cuml/cluster/spectral_clustering.pyx | 15 +++- python/cuml/cuml/common/opg_data_utils_mg.pyx | 27 ++++-- python/cuml/cuml/datasets/arima.pyx | 13 +++ python/cuml/cuml/datasets/regression.pyx | 10 +++ python/cuml/cuml/decomposition/pca.pyx | 21 +++++ python/cuml/cuml/decomposition/pca_mg.pyx | 6 ++ python/cuml/cuml/decomposition/tsvd.pyx | 21 +++++ python/cuml/cuml/decomposition/tsvd_mg.pyx | 10 +++ .../cuml/ensemble/randomforest_common.pyx | 6 ++ .../cuml/experimental/linear_model/lars.pyx | 11 +++ python/cuml/cuml/explainer/base.pyx | 4 + python/cuml/cuml/explainer/kernel_shap.pyx | 12 +++ .../cuml/cuml/explainer/permutation_shap.pyx | 7 ++ python/cuml/cuml/explainer/tree_shap.pyx | 8 ++ python/cuml/cuml/fil/fil.pyx | 6 ++ .../cuml/cuml/internals/dimension_limits.py | 82 +++++++++++++++++++ python/cuml/cuml/linear_model/base_mg.pyx | 7 ++ .../cuml/linear_model/linear_regression.pyx | 4 + .../linear_model/logistic_regression_mg.pyx | 4 + python/cuml/cuml/linear_model/ridge.pyx | 2 + .../cuml/cuml/manifold/spectral_embedding.pyx | 13 ++- python/cuml/cuml/manifold/t_sne.pyx | 26 +++++- python/cuml/cuml/manifold/umap/umap.pyx | 53 ++++++++++++ .../metrics/cluster/adjusted_rand_index.pyx | 2 + python/cuml/cuml/metrics/cluster/entropy.pyx | 9 ++ .../cuml/metrics/cluster/silhouette_score.pyx | 16 +++- python/cuml/cuml/metrics/cluster/utils.py | 3 + python/cuml/cuml/metrics/kl_divergence.pyx | 2 + .../cuml/cuml/metrics/pairwise_distances.pyx | 21 ++++- python/cuml/cuml/metrics/trustworthiness.pyx | 9 ++ .../cuml/neighbors/kneighbors_classifier.pyx | 16 +++- .../neighbors/kneighbors_classifier_mg.pyx | 35 ++++++-- .../cuml/neighbors/kneighbors_regressor.pyx | 11 ++- .../neighbors/kneighbors_regressor_mg.pyx | 9 ++ .../cuml/cuml/neighbors/nearest_neighbors.pyx | 34 ++++++++ .../cuml/neighbors/nearest_neighbors_mg.pyx | 11 +++ python/cuml/cuml/solvers/cd.pyx | 5 ++ python/cuml/cuml/solvers/cd_mg.pyx | 2 + python/cuml/cuml/solvers/qn.pyx | 8 ++ python/cuml/cuml/solvers/sgd.pyx | 11 +++ python/cuml/cuml/svm/linear.pyx | 17 +++- python/cuml/cuml/svm/svm_base.pyx | 30 +++++-- python/cuml/cuml/tsa/arima.pyx | 30 +++++++ python/cuml/cuml/tsa/auto_arima.pyx | 17 ++++ python/cuml/cuml/tsa/holtwinters.pyx | 10 +++ python/cuml/cuml/tsa/stationarity.pyx | 9 ++ 50 files changed, 711 insertions(+), 51 deletions(-) create mode 100644 python/cuml/cuml/internals/dimension_limits.py diff --git a/python/cuml/cuml/cluster/agglomerative.pyx b/python/cuml/cuml/cluster/agglomerative.pyx index e8f1f6a77c..94e9fe74e7 100644 --- a/python/cuml/cuml/cluster/agglomerative.pyx +++ b/python/cuml/cuml/cluster/agglomerative.pyx @@ -8,6 +8,10 @@ from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.common.doc_utils import generate_docstring from cuml.internals.array import CumlArray from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import ( + dims_within_int_limits, + dims_within_size_t_limits, +) from cuml.internals.mixins import ClusterMixin, CMajorInputTagMixin from cuml.internals.outputs import reflect from cuml.internals.validation import check_inputs @@ -153,6 +157,8 @@ class AgglomerativeClustering(Base, ClusterMixin, CMajorInputTagMixin): ensure_min_samples=2, reset=True, ) + dims_within_int_limits(n_rows=X.shape[0], n_cols=X.shape[1]) + dims_within_size_t_limits(n_clusters=self.n_clusters) cdef int n_rows = X.shape[0] cdef int n_cols = X.shape[1] diff --git a/python/cuml/cuml/cluster/dbscan.pyx b/python/cuml/cuml/cluster/dbscan.pyx index f5bcd1e9cf..6440cf6d5f 100644 --- a/python/cuml/cuml/cluster/dbscan.pyx +++ b/python/cuml/cuml/cluster/dbscan.pyx @@ -9,6 +9,10 @@ from cuml.common.doc_utils import generate_docstring from cuml.internals import logger, reflect from cuml.internals.array import CumlArray from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import ( + dims_within_int_limits, + dims_within_size_t_limits, +) from cuml.internals.interop import ( InteropMixin, UnsupportedOnGPU, @@ -336,6 +340,9 @@ class DBSCAN(Base, cdef int64_t n_rows = X.shape[0] cdef int64_t n_cols = X.shape[1] + dims_within_size_t_limits(n_rows=n_rows, n_cols=n_cols) + dims_within_int_limits(min_samples=self.min_samples) + if out_dtype not in (cp.dtype("int32"), cp.dtype("int64")): raise ValueError( f"Expected out_dtype to be one of ['int32', 'int64'], got {out_dtype!s}" diff --git a/python/cuml/cuml/cluster/hdbscan/hdbscan.pyx b/python/cuml/cuml/cluster/hdbscan/hdbscan.pyx index e8b77d519f..879513ef0e 100644 --- a/python/cuml/cuml/cluster/hdbscan/hdbscan.pyx +++ b/python/cuml/cuml/cluster/hdbscan/hdbscan.pyx @@ -9,6 +9,10 @@ from cuml.common.doc_utils import generate_docstring from cuml.internals import logger, reflect from cuml.internals.array import CumlArray from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import ( + dims_within_int_limits, + dims_within_size_t_limits, +) from cuml.internals.interop import ( InteropMixin, UnsupportedOnGPU, @@ -145,6 +149,8 @@ cdef class _HDBSCANState: lambdas = np.ascontiguousarray(tree["lambda_val"], dtype=np.float32) sizes = np.ascontiguousarray(tree["child_size"], dtype=np.int64) + dims_within_int_limits(n_edges=len(tree)) + dims_within_size_t_limits(n_leaves=n_leaves) cdef int n_edges = len(tree) cdef handle_t *handle_ = handle.getHandle() self.condensed_tree = new lib.CondensedHierarchy[int64_t, float]( @@ -181,6 +187,7 @@ cdef class _HDBSCANState: cdef _HDBSCANState self = _HDBSCANState.__new__(_HDBSCANState) + dims_within_int_limits(n_rows=X.shape[0], n_cols=X.shape[1]) cdef int n_rows = X.shape[0] cdef int n_cols = X.shape[1] @@ -262,6 +269,8 @@ cdef class _HDBSCANState: sizes = cp.asarray(dendrogram[:, 3], order="C", dtype="int64") cdef size_t n_leaves = dendrogram.shape[0] + 1 + dims_within_size_t_limits(n_leaves=n_leaves, dendrogram_rows=dendrogram.shape[0]) + dims_within_int_limits(min_cluster_size=min_cluster_size) handle = get_handle() cdef handle_t *handle_ = handle.getHandle() @@ -293,6 +302,7 @@ cdef class _HDBSCANState: """Initialize internal state from a new `fit`""" cdef _HDBSCANState self = _HDBSCANState.__new__(_HDBSCANState) + dims_within_int_limits(n_rows=X.shape[0], n_cols=X.shape[1]) cdef int n_rows = X.shape[0] cdef int n_cols = X.shape[1] @@ -404,6 +414,7 @@ cdef class _HDBSCANState: handle = get_handle() + dims_within_int_limits(n_rows=X.shape[0], n_cols=X.shape[1]) cdef int n_rows = X.shape[0] cdef int n_cols = X.shape[1] cdef int64_t* labels_ptr = labels.ptr @@ -1227,6 +1238,11 @@ def membership_vector(clusterer, points_to_predict, int batch_size=4096, convert order="C", return_index=True, ) + dims_within_int_limits( + n_prediction_points=points_to_predict.shape[0], + n_clusters=clusterer.n_clusters_, + batch_size=batch_size, + ) cdef int n_prediction_points = points_to_predict.shape[0] membership_vec = cp.empty( @@ -1312,6 +1328,7 @@ def approximate_predict(clusterer, points_to_predict, convert_dtype=True): order="C", return_index=True, ) + dims_within_int_limits(n_prediction_points=points_to_predict.shape[0]) cdef int n_prediction_points = points_to_predict.shape[0] prediction_labels = cp.empty(n_prediction_points, dtype="int64") @@ -1391,6 +1408,8 @@ def _extract_clusters( Exposed for testing only""" cdef size_t n_leaves = condensed_tree["parent"].min() cdef int n_edges = len(condensed_tree) + dims_within_int_limits(n_edges=n_edges) + dims_within_size_t_limits(n_leaves=n_leaves) parents = cp.asarray(condensed_tree["parent"], order="C", dtype="int64") children = cp.asarray(condensed_tree["child"], order="C", dtype="int64") diff --git a/python/cuml/cuml/cluster/kmeans.pyx b/python/cuml/cuml/cluster/kmeans.pyx index 0d0c86c510..c8b96aee4a 100644 --- a/python/cuml/cuml/cluster/kmeans.pyx +++ b/python/cuml/cuml/cluster/kmeans.pyx @@ -8,6 +8,7 @@ from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.common.doc_utils import generate_docstring from cuml.internals.array import CumlArray from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import INT32_MAX, dims_within_int_limits from cuml.internals.interop import ( InteropMixin, UnsupportedOnGPU, @@ -35,6 +36,12 @@ cdef _kmeans_init_params(kmeans, lib.KMeansParams& params): """Initialize a passed KMeansParams instance from a KMeans instance.""" cdef bool multi_gpu = kmeans._multi_gpu + dims_within_int_limits( + n_clusters=kmeans.n_clusters, + max_samples_per_batch=kmeans.max_samples_per_batch, + max_iter=kmeans.max_iter, + ) + params.n_clusters = kmeans.n_clusters params.max_iter = kmeans.max_iter params.tol = kmeans.tol @@ -78,6 +85,8 @@ cdef _kmeans_init_params(kmeans, lib.KMeansParams& params): else: params.n_init = kmeans.n_init + dims_within_int_limits(n_init=params.n_init) + cdef _kmeans_fit( handle_t& handle, @@ -91,7 +100,11 @@ cdef _kmeans_fit( cdef int64_t n_cols = X.shape[1] cdef bool values_f32 = X.dtype == cp.float32 - cdef bool indices_i32 = (n_rows * n_cols) < (2**31 - 1) + cdef bool use_i32_dims = ( + n_rows <= INT32_MAX + and n_cols <= INT32_MAX + and n_rows * n_cols < INT32_MAX + ) cdef uintptr_t X_ptr = X.data.ptr cdef uintptr_t centers_ptr = centers.data.ptr @@ -104,7 +117,7 @@ cdef _kmeans_fit( with nogil: if values_f32: - if indices_i32: + if use_i32_dims: lib.fit( handle, params, @@ -129,7 +142,7 @@ cdef _kmeans_fit( n_iter_64, ) else: - if indices_i32: + if use_i32_dims: lib.fit( handle, params, @@ -153,7 +166,7 @@ cdef _kmeans_fit( inertia_64, n_iter_64, ) - return n_iter_32 if indices_i32 else n_iter_64 + return n_iter_32 if use_i32_dims else n_iter_64 cdef _kmeans_predict( @@ -170,9 +183,15 @@ cdef _kmeans_predict( cdef int64_t n_rows = X.shape[0] cdef int64_t n_cols = X.shape[1] + cdef bool use_i32_dims = ( + n_rows <= INT32_MAX + and n_cols <= INT32_MAX + and n_rows * n_cols < INT32_MAX + ) + labels = cp.zeros( shape=n_rows, - dtype=(cp.int32 if n_rows * n_cols < 2**31 - 1 else cp.int64), + dtype=(cp.int32 if use_i32_dims else cp.int64), ) cdef uintptr_t X_ptr = X.data.ptr @@ -181,14 +200,13 @@ cdef _kmeans_predict( cdef uintptr_t labels_ptr = labels.data.ptr cdef bool values_f32 = X.dtype == cp.float32 - cdef bool indices_i32 = labels.dtype == cp.int32 cdef float inertia_f32 = 0 cdef double inertia_f64 = 0 with nogil: if values_f32: - if indices_i32: + if use_i32_dims: lib.predict( handle, params, @@ -215,7 +233,7 @@ cdef _kmeans_predict( inertia_f32, ) else: - if indices_i32: + if use_i32_dims: lib.predict( handle, params, @@ -687,6 +705,12 @@ class KMeans(Base, cdef int64_t n_rows = X.shape[0] cdef int64_t n_cols = X.shape[1] + cdef bool use_i32_dims = ( + n_rows <= INT32_MAX + and n_cols <= INT32_MAX + and n_rows * n_cols < INT32_MAX + ) + out = cp.zeros( shape=(n_rows, self.n_clusters), dtype=X.dtype, order="C", ) @@ -701,11 +725,10 @@ class KMeans(Base, _kmeans_init_params(self, params) cdef bool values_f32 = X.dtype == cp.float32 - cdef bool indices_i32 = self.labels_.dtype == cp.int32 with nogil: if values_f32: - if indices_i32: + if use_i32_dims: lib.transform( handle_[0], params, @@ -726,7 +749,7 @@ class KMeans(Base, out_ptr, ) else: - if indices_i32: + if use_i32_dims: lib.transform( handle_[0], params, diff --git a/python/cuml/cuml/cluster/spectral_clustering.pyx b/python/cuml/cuml/cluster/spectral_clustering.pyx index 84331befeb..5521792a74 100644 --- a/python/cuml/cuml/cluster/spectral_clustering.pyx +++ b/python/cuml/cuml/cluster/spectral_clustering.pyx @@ -9,6 +9,7 @@ import cuml from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.internals.array import CumlArray from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.interop import ( InteropMixin, UnsupportedOnGPU, @@ -303,8 +304,10 @@ class SpectralClustering(Base, ensure_min_samples=2, reset=True, ) - cdef int n_samples, n_features - n_samples, n_features = X.shape + n_samples_py, n_features_py = map(int, X.shape) + dims_within_int_limits(n_samples=n_samples_py, n_features=n_features_py) + cdef int n_samples = n_samples_py + cdef int n_features = n_features_py cdef float* affinity_data_ptr = NULL cdef int* affinity_rows_ptr = NULL @@ -351,6 +354,14 @@ class SpectralClustering(Base, config.n_components = max(1, min(effective_n_components, (n_samples - 1) // 3)) config.n_neighbors = min(self.n_neighbors, n_samples - 1) config.n_init = self.n_init + if precomputed: + dims_within_int_limits(affinity_nnz=int(affinity_nnz)) + dims_within_int_limits( + n_clusters=self.n_clusters, + n_init=self.n_init, + n_components=int(config.n_components), + n_neighbors=int(config.n_neighbors), + ) if self.eigen_tol == "auto": config.eigen_tol = 0.0 else: diff --git a/python/cuml/cuml/common/opg_data_utils_mg.pyx b/python/cuml/cuml/common/opg_data_utils_mg.pyx index 9ae50661ac..13f89c0e0c 100644 --- a/python/cuml/cuml/common/opg_data_utils_mg.pyx +++ b/python/cuml/cuml/common/opg_data_utils_mg.pyx @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # @@ -7,6 +7,10 @@ import numpy as np from cuml.common import input_to_cuml_array from cuml.internals.array import CumlArray +from cuml.internals.dimension_limits import ( + dims_within_int_limits, + dims_within_size_t_limits, +) from cython.operator cimport dereference as deref from libc.stdint cimport uintptr_t @@ -112,10 +116,12 @@ def build_rank_size_pair(parts_to_sizes, rank): cdef vector[RankSizePair*] *rsp_vec = new vector[RankSizePair*]() for idx, rankToSize in enumerate(parts_to_sizes): - rank, size = rankToSize + part_rank, part_size = rankToSize + dims_within_int_limits(rank=part_rank) + dims_within_size_t_limits(partition_size=part_size) rsp = malloc(sizeof(RankSizePair)) - rsp.rank = rank - rsp.size = size + rsp.rank = part_rank + rsp.size = part_size rsp_vec.push_back(rsp) @@ -157,6 +163,9 @@ def build_part_descriptor(m, n, rank_size_t, rank): -------- ptr: PartDescriptor object """ + dims_within_size_t_limits(total_rows=m, n_cols=n) + dims_within_int_limits(rank=rank) + cdef uintptr_t rank_size_ptr = rank_size_t cdef vector[RankSizePair *] *rsp_vec \ @@ -200,6 +209,8 @@ def _build_part_inputs(cuda_arr_ifaces, parts_to_ranks, m, n, local_rank, convert_dtype): + dims_within_size_t_limits(total_rows=m, n_cols=n) + dims_within_int_limits(local_rank=local_rank) cuml_arr_ifaces = [] for arr in cuda_arr_ifaces: @@ -220,10 +231,12 @@ def _build_part_inputs(cuda_arr_ifaces, cdef vector[RankSizePair*] partsToRanks for idx, rankToSize in enumerate(parts_to_ranks): - rank, size = rankToSize + part_rank, part_size = rankToSize + dims_within_int_limits(rank=part_rank) + dims_within_size_t_limits(partition_size=part_size) rsp = malloc(sizeof(RankSizePair)) - rsp.rank = rank - rsp.size = size + rsp.rank = part_rank + rsp.size = part_size partsToRanks.push_back(rsp) cdef PartDescriptor *descriptor = \ diff --git a/python/cuml/cuml/datasets/arima.pyx b/python/cuml/cuml/datasets/arima.pyx index 662c23b946..a7b16ae245 100644 --- a/python/cuml/cuml/datasets/arima.pyx +++ b/python/cuml/cuml/datasets/arima.pyx @@ -8,6 +8,7 @@ import numpy as np from cuml.internals import get_handle, reflect from cuml.internals.array import CumlArray as cumlArray +from cuml.internals.dimension_limits import dims_within_int_limits from libc.stdint cimport uint64_t, uintptr_t from pylibraft.common.handle cimport handle_t @@ -94,6 +95,18 @@ def make_arima(batch_size=1000, n_obs=100, order=(1, 1, 1), cpp_order.k = intercept cpp_order.n_exog = 0 + dims_within_int_limits( + batch_size=batch_size, + n_obs=n_obs, + arima_p=order[0], + arima_d=order[1], + arima_q=order[2], + seasonal_P=seasonal_order[0], + seasonal_D=seasonal_order[1], + seasonal_Q=seasonal_order[2], + seasonal_s=seasonal_order[3], + ) + # Define some parameters based on the order scale = 1.0 noise_scale = 0.2 diff --git a/python/cuml/cuml/datasets/regression.pyx b/python/cuml/cuml/datasets/regression.pyx index fa47ee07aa..aad401c183 100644 --- a/python/cuml/cuml/datasets/regression.pyx +++ b/python/cuml/cuml/datasets/regression.pyx @@ -10,6 +10,7 @@ import numpy as np import cuml.internals.nvtx as nvtx from cuml.internals import get_handle, reflect from cuml.internals.array import CumlArray +from cuml.internals.dimension_limits import dims_within_size_t_limits from libc.stdint cimport uint64_t, uintptr_t from libcpp cimport bool @@ -158,6 +159,15 @@ def make_regression( if effective_rank is None: effective_rank = -1 + dims_within_size_t_limits( + n_samples=n_samples, + n_features=n_features, + n_informative=n_informative, + n_targets=n_targets, + ) + if effective_rank >= 0: + dims_within_size_t_limits(effective_rank=effective_rank) + handle = get_handle() cdef handle_t* handle_ = handle.getHandle() diff --git a/python/cuml/cuml/decomposition/pca.pyx b/python/cuml/cuml/decomposition/pca.pyx index c8c47396f9..9beb529a7a 100644 --- a/python/cuml/cuml/decomposition/pca.pyx +++ b/python/cuml/cuml/decomposition/pca.pyx @@ -12,6 +12,10 @@ from cuml.common.doc_utils import generate_docstring from cuml.common.sparse_utils import is_sparse from cuml.internals.array import CumlArray from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import ( + dims_within_size_t_limits, + dims_within_uint32_limits, +) from cuml.internals.interop import ( InteropMixin, UnsupportedOnGPU, @@ -507,6 +511,13 @@ class PCA(Base, else: self.n_components_ = self.n_components + dims_within_size_t_limits( + n_rows=n_rows, + n_cols=n_cols, + n_components=self.n_components_, + ) + dims_within_uint32_limits(n_iterations=self.iterated_power) + if is_sparse(X): self._fit_sparse(X) else: @@ -546,6 +557,11 @@ class PCA(Base, def _inverse_transform_dense(self, X, *, index=None): dtype = X.dtype n_rows = X.shape[0] + dims_within_size_t_limits( + n_rows=n_rows, + n_cols=self.n_features_in_, + n_components=self.n_components_, + ) out = cp.zeros((n_rows, self.n_features_in_), dtype=dtype, order="F") @@ -642,6 +658,11 @@ class PCA(Base, def _transform_dense(self, X, *, index=None): dtype = X.dtype n_rows, n_cols = X.shape + dims_within_size_t_limits( + n_rows=n_rows, + n_cols=n_cols, + n_components=self.n_components_, + ) out = cp.zeros((n_rows, self.n_components_), dtype=dtype, order="F") diff --git a/python/cuml/cuml/decomposition/pca_mg.pyx b/python/cuml/cuml/decomposition/pca_mg.pyx index 34cc37fc02..7d67bd8f63 100644 --- a/python/cuml/cuml/decomposition/pca_mg.pyx +++ b/python/cuml/cuml/decomposition/pca_mg.pyx @@ -8,6 +8,7 @@ from cuml.decomposition import PCA from cuml.decomposition.base_mg import BaseDecompositionMG from cuml.internals import run_in_internal_context from cuml.internals.array import CumlArray +from cuml.internals.dimension_limits import dims_within_size_t_limits from cython.operator cimport dereference as deref from libc.stdint cimport uintptr_t @@ -55,6 +56,11 @@ cdef extern from "cuml/decomposition/pca_mg.hpp" namespace "ML::PCA::opg" nogil: class PCAMG(BaseDecompositionMG, PCA): @run_in_internal_context def _mg_fit(self, X_ptr, n_rows, n_cols, dtype, input_desc_ptr): + dims_within_size_t_limits( + n_rows=n_rows, + n_cols=n_cols, + n_components=self.n_components_, + ) # Validate and initialize parameters cdef paramsPCAMG params params.n_components = self.n_components_ diff --git a/python/cuml/cuml/decomposition/tsvd.pyx b/python/cuml/cuml/decomposition/tsvd.pyx index 8d257738d3..149b19871a 100644 --- a/python/cuml/cuml/decomposition/tsvd.pyx +++ b/python/cuml/cuml/decomposition/tsvd.pyx @@ -11,6 +11,10 @@ from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.common.doc_utils import generate_docstring from cuml.internals.array import CumlArray from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import ( + dims_within_size_t_limits, + dims_within_uint32_limits, +) from cuml.internals.interop import InteropMixin, to_cpu, to_gpu from cuml.internals.mixins import FMajorInputTagMixin from cuml.internals.validation import ( @@ -329,6 +333,13 @@ class TruncatedSVD(Base, f"number of features in X ({n_cols})" ) + dims_within_size_t_limits( + n_rows=n_rows, + n_cols=n_cols, + n_components=self.n_components, + ) + dims_within_uint32_limits(n_iterations=self.n_iter) + cdef paramsTSVD params cdef bool flip_signs_based_on_U = self._u_based_sign_flip params.n_components = self.n_components @@ -427,6 +438,11 @@ class TruncatedSVD(Base, n_rows = X.shape[0] dtype = X.dtype + dims_within_size_t_limits( + n_rows=n_rows, + n_cols=self.n_features_in_, + n_components=self.n_components, + ) cdef paramsTSVD params params.n_components = self.n_components @@ -486,6 +502,11 @@ class TruncatedSVD(Base, n_rows = X.shape[0] dtype = X.dtype + dims_within_size_t_limits( + n_rows=n_rows, + n_cols=self.n_features_in_, + n_components=self.n_components, + ) cdef paramsTSVD params params.n_components = self.n_components diff --git a/python/cuml/cuml/decomposition/tsvd_mg.pyx b/python/cuml/cuml/decomposition/tsvd_mg.pyx index dc2db18102..75162261ec 100644 --- a/python/cuml/cuml/decomposition/tsvd_mg.pyx +++ b/python/cuml/cuml/decomposition/tsvd_mg.pyx @@ -8,6 +8,10 @@ from cuml.decomposition import TruncatedSVD from cuml.decomposition.base_mg import BaseDecompositionMG from cuml.internals import run_in_internal_context from cuml.internals.array import CumlArray +from cuml.internals.dimension_limits import ( + dims_within_size_t_limits, + dims_within_uint32_limits, +) from cython.operator cimport dereference as deref from libc.stdint cimport uintptr_t @@ -57,6 +61,12 @@ class TSVDMG(BaseDecompositionMG, TruncatedSVD): def _mg_fit_transform( self, X_ptr, n_rows, n_cols, dtype, trans_ptr, input_desc_ptr, trans_desc_ptr ): + dims_within_size_t_limits( + n_rows=n_rows, + n_cols=n_cols, + n_components=self.n_components_, + ) + dims_within_uint32_limits(n_iterations=self.n_iter) # Validate and initialize parameters cdef paramsTSVDMG params params.n_components = self.n_components_ diff --git a/python/cuml/cuml/ensemble/randomforest_common.pyx b/python/cuml/cuml/ensemble/randomforest_common.pyx index a886aeb4b7..b0c86a9834 100644 --- a/python/cuml/cuml/ensemble/randomforest_common.pyx +++ b/python/cuml/cuml/ensemble/randomforest_common.pyx @@ -14,6 +14,7 @@ import treelite.sklearn from cuml.fil.fil import ForestInference from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.interop import ( InteropMixin, UnsupportedOnCPU, @@ -417,6 +418,11 @@ class BaseRandomForestModel(Base, InteropMixin): cdef uintptr_t X_ptr = X.data.ptr cdef uintptr_t y_ptr = y.data.ptr + dims_within_int_limits( + n_rows=X.shape[0], + n_cols=X.shape[1], + n_classes=(self.n_classes_ if is_classifier else 0), + ) cdef int n_rows = X.shape[0] cdef int n_cols = X.shape[1] cdef level_enum verbose = self._verbose_level diff --git a/python/cuml/cuml/experimental/linear_model/lars.pyx b/python/cuml/cuml/experimental/linear_model/lars.pyx index 49aacf1e8c..a90646de71 100644 --- a/python/cuml/cuml/experimental/linear_model/lars.pyx +++ b/python/cuml/cuml/experimental/linear_model/lars.pyx @@ -9,6 +9,7 @@ from cuml.common.doc_utils import generate_docstring from cuml.internals import logger, reflect from cuml.internals.array import CumlArray, cuda_ptr from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.mixins import RegressorMixin from cuml.internals.validation import ( check_array, @@ -252,6 +253,11 @@ class Lars(Base, RegressorMixin): cdef uintptr_t X_ptr = X.data.ptr cdef uintptr_t y_ptr = y.data.ptr cdef uintptr_t gram_ptr = NULL if gram is None else gram.data.ptr + dims_within_int_limits( + n_rows=X.shape[0], + n_cols=X.shape[1], + max_iter=max_iter, + ) cdef int n_rows = X.shape[0] cdef int n_cols = X.shape[1] cdef uintptr_t beta_ptr = beta.data.ptr @@ -351,6 +357,11 @@ class Lars(Base, RegressorMixin): order="F", return_index=True, ) + dims_within_int_limits( + n_rows=X.shape[0], + n_cols=X.shape[1], + n_active=self.active_.shape[0], + ) cdef int n_rows = X.shape[0] cdef int n_cols = X.shape[1] preds = cp.zeros(n_rows, dtype=X.dtype) diff --git a/python/cuml/cuml/explainer/base.pyx b/python/cuml/cuml/explainer/base.pyx index 272b27d127..b9ccc7eba1 100644 --- a/python/cuml/cuml/explainer/base.pyx +++ b/python/cuml/cuml/explainer/base.pyx @@ -16,6 +16,7 @@ from cuml.explainer.common import ( output_list_shap_values, ) from cuml.internals.base import get_handle +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.validation import check_array from libc.stdint cimport uintptr_t @@ -133,6 +134,7 @@ class SHAPBase(): background, order=self.order, dtype=self.dtype, ensure_all_finite=False ) self.nrows, self.ncols = self.background.shape + dims_within_int_limits(nrows=self.nrows, ncols=self.ncols) self.random_state = random_state @@ -344,6 +346,8 @@ class SHAPBase(): idx_ptr = get_cai_ptr(inds) row_major = self.masker.order == "C" + dims_within_int_limits(nrows=self.nrows, ncols=self.ncols) + cdef uintptr_t masked_ptr_f32 cdef uintptr_t bg_ptr_f32 cdef uintptr_t row_ptr_f32 diff --git a/python/cuml/cuml/explainer/kernel_shap.pyx b/python/cuml/cuml/explainer/kernel_shap.pyx index e817baaad4..8487f1da49 100644 --- a/python/cuml/cuml/explainer/kernel_shap.pyx +++ b/python/cuml/cuml/explainer/kernel_shap.pyx @@ -13,6 +13,7 @@ import numpy as np from cuml.explainer.base import SHAPBase from cuml.explainer.common import get_cai_ptr, model_func_call from cuml.internals import get_handle +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.validation import check_array from cuml.linear_model import Lasso, LinearRegression @@ -197,6 +198,8 @@ class KernelExplainer(SHAPBase): # all possible samples to check for need for l1 self.ratio_evaluated = self.nsamples / max_samples + dims_within_int_limits(nsamples=self.nsamples) + self.nsamples_exact, self.nsamples_random, self.randind = \ _get_number_of_exact_random_samples(ncols=self.ncols, nsamples=self.nsamples) @@ -292,6 +295,7 @@ class KernelExplainer(SHAPBase): ds_ptr = get_cai_ptr(self._synth_data) if self.nsamples_random > 0: smp_ptr = get_cai_ptr(samples) + maxsample = int(self.nsamples_random / 2) else: smp_ptr = NULL maxsample = 0 @@ -301,6 +305,14 @@ class KernelExplainer(SHAPBase): if self.random_state is None: self.random_state = randint(0, 10**18) + dims_within_int_limits( + mask_rows=self._mask.shape[0], + mask_cols=self._mask.shape[1], + background_rows=self.background.shape[0], + nsamples_random=self.nsamples_random, + maxsample=maxsample, + ) + cdef uintptr_t bg_ptr_f32 cdef uintptr_t ds_ptr_f32 cdef uintptr_t row_ptr_f32 diff --git a/python/cuml/cuml/explainer/permutation_shap.pyx b/python/cuml/cuml/explainer/permutation_shap.pyx index 57ec12d65e..ce77f7f249 100644 --- a/python/cuml/cuml/explainer/permutation_shap.pyx +++ b/python/cuml/cuml/explainer/permutation_shap.pyx @@ -10,6 +10,7 @@ import numpy as np from cuml.explainer.base import SHAPBase from cuml.explainer.common import get_cai_ptr, model_func_call from cuml.internals import get_handle +from cuml.internals.dimension_limits import dims_within_int_limits from libc.stdint cimport uintptr_t from libcpp cimport bool @@ -236,6 +237,12 @@ class PermutationExplainer(SHAPBase): if self.random_state is not None: cp.random.seed(seed=self.random_state) + dims_within_int_limits( + nrows=self.nrows, + ncols=self.ncols, + npermutations=npermutations, + ) + for _ in range(npermutations): if not testing: diff --git a/python/cuml/cuml/explainer/tree_shap.pyx b/python/cuml/cuml/explainer/tree_shap.pyx index 6efd36b31d..1dca66bb9b 100644 --- a/python/cuml/cuml/explainer/tree_shap.pyx +++ b/python/cuml/cuml/explainer/tree_shap.pyx @@ -11,6 +11,7 @@ import pandas as pd import treelite import cuml +from cuml.internals.dimension_limits import dims_within_size_t_limits from cuml.internals.treelite import safe_treelite_call from cuml.internals.validation import check_array @@ -245,6 +246,12 @@ cdef class TreeExplainer: n_rows, n_cols = X.shape dtype = X.dtype + dims_within_size_t_limits(n_rows=n_rows, n_cols=n_cols) + if self.data is not None: + dims_within_size_t_limits( + background_n_rows=self.data.shape[0], + background_n_cols=self.data.shape[1], + ) preds = cp.empty( (n_rows, self.num_class[0] * (n_cols + 1)), @@ -324,6 +331,7 @@ cdef class TreeExplainer: n_rows, n_cols = X.shape dtype = X.dtype + dims_within_size_t_limits(n_rows=n_rows, n_cols=n_cols) preds = cp.empty( (n_rows, self.num_class[0] * (n_cols + 1)**2), diff --git a/python/cuml/cuml/fil/fil.pyx b/python/cuml/cuml/fil/fil.pyx index af12a3736f..022da7f038 100644 --- a/python/cuml/cuml/fil/fil.pyx +++ b/python/cuml/cuml/fil/fil.pyx @@ -15,6 +15,10 @@ import cuml.internals.nvtx as nvtx from cuml.internals.array import CumlArray from cuml.internals.base import Base, get_handle from cuml.internals.device_type import DeviceType, DeviceTypeError +from cuml.internals.dimension_limits import ( + dims_within_size_t_limits, + dims_within_uint32_limits, +) from cuml.internals.global_settings import GlobalSettings from cuml.internals.mem_type import MemoryType from cuml.internals.mixins import CMajorInputTagMixin @@ -267,6 +271,7 @@ cdef class ForestInference_impl(): input_name="X", ) n_rows = X.shape[0] + dims_within_size_t_limits(n_rows=n_rows) cdef raft_proto_device_t in_dev = get_fil_raft_proto_device_type(X) cdef uintptr_t in_ptr = ( @@ -310,6 +315,7 @@ cdef class ForestInference_impl(): if chunk_size is None: chunk_specification = nullopt else: + dims_within_uint32_limits(chunk_size=chunk_size) chunk_specification = chunk_size if model_dtype == np.float32: diff --git a/python/cuml/cuml/internals/dimension_limits.py b/python/cuml/cuml/internals/dimension_limits.py new file mode 100644 index 0000000000..78392c6960 --- /dev/null +++ b/python/cuml/cuml/internals/dimension_limits.py @@ -0,0 +1,82 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 +# +"""Guards for Python bindings that pass array dimensions to C/CUDA as 32-bit types.""" + +from __future__ import annotations + +import numpy as np + +# Signed 32-bit int maximum (typical C ``int`` on cuML-supported platforms). +INT32_MAX = 2_147_483_647 +INT32_MIN = -INT32_MAX - 1 +UINT32_MAX = 4_294_967_295 + + +def values_fit_int32(**values: int) -> None: + """ + Verify each scalar fits a C signed ``int``; raise ``ValueError`` if not. + + Unlike ``dims_within_int_limits``, negative values are allowed (e.g. label + ranges passed to legacy kernels as ``int``). + """ + for name, value in values.items(): + v = int(value) + if v < INT32_MIN or v > INT32_MAX: + raise ValueError( + f"{name}={value!r} is outside the range representable as a " + f"32-bit signed integer [{INT32_MIN}, {INT32_MAX}]; the binding " + "would truncate when passing this value to native code." + ) + + +def dims_within_int_limits(**dims: int) -> None: + """ + Verify dimensions fit a C ``int``; raise ``ValueError`` if any do not. + + Several legacy CUDA entry points take row counts, column counts, or lengths + as C ``int``. Values outside ``[0, INT32_MAX]`` are silently truncated in + Cython, which corrupts kernel launches. + """ + for name, value in dims.items(): + v = int(value) + if v < 0: + raise ValueError(f"{name} must be non-negative, got {value!r}") + if v > INT32_MAX: + raise ValueError( + f"{name}={value!r} exceeds the maximum value supported by this " + f"binding (<= {INT32_MAX}); larger inputs would be truncated when " + "passed to native code as a 32-bit signed integer." + ) + + +def dims_within_uint32_limits(**dims: int) -> None: + """Verify dimensions fit ``uint32_t``; raise ``ValueError`` if any do not.""" + for name, value in dims.items(): + v = int(value) + if v < 0: + raise ValueError(f"{name} must be non-negative, got {value!r}") + if v > UINT32_MAX: + raise ValueError( + f"{name}={value!r} exceeds uint32_t maximum ({UINT32_MAX})." + ) + + +def dims_within_size_t_limits(**dims: int) -> None: + """ + Verify values fit ``size_t`` on this platform; raise ``ValueError`` if not. + + Uses ``numpy.uintp`` range (same width as ``size_t`` / ``uintptr_t`` on + supported builds) for the upper bound. + """ + maxv = int(np.iinfo(np.uintp).max) + for name, value in dims.items(): + v = int(value) + if v < 0: + raise ValueError(f"{name} must be non-negative, got {value!r}") + if v > maxv: + raise ValueError( + f"{name}={value!r} exceeds the maximum value representable as " + f"size_t on this platform ({maxv})." + ) diff --git a/python/cuml/cuml/linear_model/base_mg.pyx b/python/cuml/cuml/linear_model/base_mg.pyx index f7f748fe9f..fe9edd447a 100644 --- a/python/cuml/cuml/linear_model/base_mg.pyx +++ b/python/cuml/cuml/linear_model/base_mg.pyx @@ -9,6 +9,10 @@ from cuml.common.sparse_utils import is_sparse from cuml.internals import run_in_internal_context from cuml.internals.array import CumlArray from cuml.internals.array_sparse import SparseCumlArray +from cuml.internals.dimension_limits import ( + dims_within_int_limits, + dims_within_size_t_limits, +) from cuml.internals.input_utils import input_to_cuml_array from cuml.internals.validation import check_features @@ -44,6 +48,8 @@ class MGFitMixin: :return: self """ + dims_within_size_t_limits(n_rows=n_rows, n_cols=n_cols) + self._set_output_type(input_data[0][0]) check_features(self, input_data[0][0], reset=True) sparse_input = is_sparse(input_data[0][0]) @@ -113,6 +119,7 @@ class MGFitMixin: X_cols = X_arys[0].indices.ptr X_row_ids = X_arys[0].indptr.ptr X_nnz = sum([x.nnz for x in X_arys]) + dims_within_int_limits(X_nnz=X_nnz) # call inheriting class _fit that does all cython pointers and calls self._fit(X=[X_arg, X_cols, X_row_ids, X_nnz], diff --git a/python/cuml/cuml/linear_model/linear_regression.pyx b/python/cuml/cuml/linear_model/linear_regression.pyx index 450db871a1..aa54303889 100644 --- a/python/cuml/cuml/linear_model/linear_regression.pyx +++ b/python/cuml/cuml/linear_model/linear_regression.pyx @@ -11,6 +11,7 @@ from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.common.doc_utils import generate_docstring from cuml.internals.array import CumlArray, cuda_ptr from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import dims_within_size_t_limits from cuml.internals.interop import ( InteropMixin, UnsupportedOnGPU, @@ -254,6 +255,9 @@ class LinearRegression(Base, cdef size_t n_rows = X.shape[0] cdef size_t n_cols = X.shape[1] + dims_within_size_t_limits(n_rows=n_rows, n_cols=n_cols) + if y.ndim == 2: + dims_within_size_t_limits(n_targets=y.shape[1]) cdef uintptr_t X_ptr = X.data.ptr cdef uintptr_t y_ptr = y.data.ptr cdef uintptr_t sample_weight_ptr = ( diff --git a/python/cuml/cuml/linear_model/logistic_regression_mg.pyx b/python/cuml/cuml/linear_model/logistic_regression_mg.pyx index 16f37e526d..e919c4614d 100644 --- a/python/cuml/cuml/linear_model/logistic_regression_mg.pyx +++ b/python/cuml/cuml/linear_model/logistic_regression_mg.pyx @@ -6,6 +6,7 @@ import numpy as np from cuml.internals import run_in_internal_context from cuml.internals.array import CumlArray +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.linear_model import LogisticRegression from cuml.linear_model.base_mg import MGFitMixin @@ -169,6 +170,7 @@ class LogisticRegressionMG(MGFitMixin, LogisticRegression): ) cdef int n_classes = len(self.classes_) + dims_within_int_limits(n_classes=n_classes, n_cols=self.n_cols) # Validate and initialize parameters l1_strength, l2_strength = self._get_l1_l2_strength() @@ -205,6 +207,8 @@ class LogisticRegressionMG(MGFitMixin, LogisticRegression): else: X_ptr, X_cols_ptr, X_rows_ptr, X_nnz = X X_index_is_i32 = self.index_dtype == np.int32 + if X_index_is_i32: + dims_within_int_limits(X_nnz=X_nnz) coef_ptr = coef.ptr cdef bool standardization = self.standardization diff --git a/python/cuml/cuml/linear_model/ridge.pyx b/python/cuml/cuml/linear_model/ridge.pyx index c9e196cbfd..356a589d52 100644 --- a/python/cuml/cuml/linear_model/ridge.pyx +++ b/python/cuml/cuml/linear_model/ridge.pyx @@ -10,6 +10,7 @@ from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.common.doc_utils import generate_docstring from cuml.internals.array import CumlArray, cuda_ptr from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.interop import ( InteropMixin, UnsupportedOnGPU, @@ -279,6 +280,7 @@ class Ridge(Base, may_mutate_sample_weight, ): """Fit a Ridge regression using the Eig solver.""" + dims_within_int_limits(n_rows=X.shape[0], n_cols=X.shape[1]) cdef int n_rows = X.shape[0] cdef int n_cols = X.shape[1] diff --git a/python/cuml/cuml/manifold/spectral_embedding.pyx b/python/cuml/cuml/manifold/spectral_embedding.pyx index 69589e87a5..9abd296bdd 100644 --- a/python/cuml/cuml/manifold/spectral_embedding.pyx +++ b/python/cuml/cuml/manifold/spectral_embedding.pyx @@ -10,6 +10,7 @@ import scipy.sparse as sp from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.internals.array import CumlArray from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.input_utils import input_to_cupy_array from cuml.internals.interop import ( InteropMixin, @@ -182,8 +183,12 @@ def spectral_embedding( "['nearest_neighbors', 'precomputed']" ) - cdef int n_samples, n_features - n_samples, n_features = A.shape + n_samples_py, n_features_py = map(int, A.shape) + dims_within_int_limits(n_samples=n_samples_py, n_features=n_features_py) + if affinity == "precomputed": + dims_within_int_limits(affinity_nnz=int(affinity_nnz)) + cdef int n_samples = n_samples_py + cdef int n_features = n_features_py if not isfinite: raise ValueError( @@ -222,6 +227,10 @@ def spectral_embedding( if n_neighbors is not None else max(int(A.shape[0] / 10), 1) ) + dims_within_int_limits( + n_components=int(config.n_components), + n_neighbors=int(config.n_neighbors), + ) cdef float* eigenvectors_ptr = eigenvectors.ptr cdef bool precomputed = affinity == "precomputed" handle = get_handle() diff --git a/python/cuml/cuml/manifold/t_sne.pyx b/python/cuml/cuml/manifold/t_sne.pyx index 660a5e874b..65b0d960e9 100644 --- a/python/cuml/cuml/manifold/t_sne.pyx +++ b/python/cuml/cuml/manifold/t_sne.pyx @@ -15,6 +15,7 @@ from cuml.common.sparsefuncs import extract_knn_graph from cuml.internals.array import CumlArray from cuml.internals.array_sparse import SparseCumlArray from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.interop import ( InteropMixin, UnsupportedOnGPU, @@ -224,6 +225,14 @@ cdef _init_params(self, int n_samples, TSNEParams ¶ms): else check_random_seed(self.random_state) ) + dims_within_int_limits( + dim=n_components, + n_neighbors=n_neighbors, + perplexity_max_iter=perplexity_max_iter, + exaggeration_iter=exaggeration_iter, + max_iter=max_iter, + ) + params.dim = n_components params.n_neighbors = n_neighbors params.theta = angle @@ -582,28 +591,37 @@ class TSNE(Base, should match the metric used to train the TSNE embeedings. Takes precedence over the precomputed_knn parameter. """ - cdef int n_samples, n_features cdef uintptr_t X_ptr = 0 cdef uintptr_t X_indptr_ptr = 0 cdef uintptr_t X_indices_ptr = 0 cdef int X_nnz = 0 cdef bool sparse_fit = is_sparse(X) + cdef int n_samples + cdef int n_features # Normalize input X if sparse_fit: X_m = SparseCumlArray(X, convert_to_dtype=cupy.float32) - n_samples, n_features = X_m.shape + n_s, n_f = map(int, X_m.shape) + X_nnz = int(X_m.nnz) X_ptr = X_m.data.ptr X_indptr_ptr = X_m.indptr.ptr X_indices_ptr = X_m.indices.ptr - X_nnz = X_m.nnz else: - X_m, n_samples, n_features, _ = input_to_cuml_array( + X_m, n_s, n_f, _ = input_to_cuml_array( X, order='F', check_dtype=np.float32, convert_to_dtype=(np.float32 if convert_dtype else None) ) + n_s, n_f = int(n_s), int(n_f) X_ptr = X_m.ptr + dims_within_int_limits(n_samples=n_s, n_features=n_f) + if sparse_fit: + dims_within_int_limits(X_nnz=X_nnz, csr_indptr_len=n_s + 1) + + n_samples = n_s + n_features = n_f + # Initialize TSNEParams cdef TSNEParams params _init_params(self, n_samples, params) diff --git a/python/cuml/cuml/manifold/umap/umap.pyx b/python/cuml/cuml/manifold/umap/umap.pyx index bf3ca6d293..dd848fdb31 100644 --- a/python/cuml/cuml/manifold/umap/umap.pyx +++ b/python/cuml/cuml/manifold/umap/umap.pyx @@ -21,6 +21,10 @@ from cuml.internals import logger, reflect from cuml.internals.array import CumlArray from cuml.internals.array_sparse import SparseCumlArray from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import ( + dims_within_int_limits, + dims_within_size_t_limits, +) from cuml.internals.input_utils import input_to_cuml_array, is_array_like from cuml.internals.interop import ( InteropMixin, @@ -314,6 +318,8 @@ cdef class RaftCOO: cdef RaftCOO self = RaftCOO.__new__(RaftCOO) cdef handle_t* handle_ = handle.getHandle() + dims_within_size_t_limits(nnz=arr.nnz) + dims_within_int_limits(n_rows=arr.shape[0]) cdef lib.COO* coo = new lib.COO(handle_.get_stream()) self.ptr.reset(coo) coo.allocate(arr.nnz, arr.shape[0], False, handle_.get_stream()) @@ -523,6 +529,16 @@ cdef init_params(self, lib.UMAPParams ¶ms, n_rows, is_sparse=False, is_fit=T warnings.warn("build_algo='nn_descent' is not deterministic. Please use " "build_algo='brute_force_knn' instead with random_state set.") + dims_within_int_limits( + n_rows=n_rows, + n_components=self.n_components, + n_neighbors=self._n_neighbors, + n_epochs=self.n_epochs or 0, + negative_sample_rate=self.negative_sample_rate, + transform_queue_size=self.transform_queue_size, + target_n_neighbors=self.target_n_neighbors, + ) + params.n_neighbors = self._n_neighbors params.n_components = self.n_components params.n_epochs = self.n_epochs or 0 @@ -583,6 +599,8 @@ cdef init_params(self, lib.UMAPParams ¶ms, n_rows, is_sparse=False, is_fit=T f"knn_overlap_factor ({overlap_factor})`" ) + dims_within_int_limits(knn_n_clusters=n_clusters) + all_neighbors_supported_metrics = [ 'l2', 'euclidean', 'sqeuclidean', 'cosine', 'inner_product' ] @@ -631,6 +649,12 @@ cdef init_params(self, lib.UMAPParams ¶ms, n_rows, is_sparse=False, is_fit=T params.build_params.nnd.graph_degree = graph_degree params.build_params.nnd.intermediate_graph_degree = intermediate_graph_degree + dims_within_int_limits( + nnd_max_iterations=max_iterations, + nnd_graph_degree=graph_degree, + nnd_intermediate_graph_degree=intermediate_graph_degree, + ) + class UMAP(Base, InteropMixin, CMajorInputTagMixin, SparseInputTagMixin): """Uniform Manifold Approximation and Projection @@ -1180,6 +1204,7 @@ class UMAP(Base, InteropMixin, CMajorInputTagMixin, SparseInputTagMixin): if len(X.shape) != 2: raise ValueError("Reshape your data: data should be two dimensional") + dims_within_int_limits(n_rows=X.shape[0], n_dims=X.shape[1]) cdef int n_rows = X.shape[0] cdef int n_dims = X.shape[1] @@ -1216,6 +1241,7 @@ class UMAP(Base, InteropMixin, CMajorInputTagMixin, SparseInputTagMixin): X_indices_ptr = X_m.indices.ptr X_indptr_ptr = X_m.indptr.ptr X_nnz = X_m.nnz + dims_within_size_t_limits(X_nnz=X_nnz) else: X_m = input_to_cuml_array( X, @@ -1467,6 +1493,11 @@ class UMAP(Base, InteropMixin, CMajorInputTagMixin, SparseInputTagMixin): ).array cdef bool X_is_sparse = self._sparse_data + dims_within_int_limits( + n_rows=X.shape[0], + n_cols=X.shape[1], + orig_n_rows=self._raw_data.shape[0], + ) cdef int n_rows = X.shape[0] cdef int n_cols = X.shape[1] cdef int orig_n_rows = self._raw_data.shape[0] @@ -1503,6 +1534,7 @@ class UMAP(Base, InteropMixin, CMajorInputTagMixin, SparseInputTagMixin): orig_indices_ptr = self._raw_data.indices.ptr orig_ptr = self._raw_data.data.ptr orig_nnz = self._raw_data.nnz + dims_within_size_t_limits(X_nnz=X_nnz, orig_nnz=orig_nnz) else: X_ptr = X.ptr orig_ptr = self._raw_data.ptr @@ -1621,6 +1653,13 @@ class UMAP(Base, InteropMixin, CMajorInputTagMixin, SparseInputTagMixin): # Ensure C-contiguous layout for CUDA kernels inv_transformed_gpu = cp.ascontiguousarray(inv_transformed_gpu) + dims_within_int_limits( + c_n_samples=n_samples, + c_n_features=raw_data_np.shape[1], + c_orig_n=raw_data_np.shape[0], + c_nnz=vals_gpu.shape[0], + n_epochs_inv=n_epochs_inv, + ) cdef int c_n_samples = n_samples cdef int c_n_features = raw_data_np.shape[1] cdef int c_orig_n = raw_data_np.shape[0] @@ -1742,6 +1781,11 @@ def fuzzy_simplicial_set( convert_to_dtype=np.float32 ).array + dims_within_int_limits( + n_rows=X_m.shape[0], + n_cols=X_m.shape[1], + n_neighbors=n_neighbors, + ) cdef int n_rows = X_m.shape[0] cdef int n_cols = X_m.shape[1] @@ -1895,6 +1939,13 @@ def simplicial_set_embedding( check_dtype=np.float32, ).array + dims_within_int_limits( + n_rows=X.shape[0], + n_cols=X.shape[1], + n_components=n_components, + negative_sample_rate=negative_sample_rate, + n_epochs=(n_epochs or 0), + ) cdef int n_rows = X.shape[0] cdef int n_cols = X.shape[1] @@ -1952,6 +2003,8 @@ def simplicial_set_embedding( if not isinstance(graph, cupyx.scipy.sparse.coo_matrix): graph = cupyx.scipy.sparse.coo_matrix(graph) + dims_within_size_t_limits(graph_nnz=graph.nnz) + handle = get_handle() cdef handle_t* handle_ = handle.getHandle() cdef RaftCOO fss_graph = RaftCOO.from_cupy_coo(handle, graph) diff --git a/python/cuml/cuml/metrics/cluster/adjusted_rand_index.pyx b/python/cuml/cuml/metrics/cluster/adjusted_rand_index.pyx index ac0993d22e..bee74e8cd4 100644 --- a/python/cuml/cuml/metrics/cluster/adjusted_rand_index.pyx +++ b/python/cuml/cuml/metrics/cluster/adjusted_rand_index.pyx @@ -5,6 +5,7 @@ import numpy as np from cuml.internals import get_handle +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.validation import check_array, check_consistent_length from libc.stdint cimport uintptr_t @@ -62,6 +63,7 @@ def adjusted_rand_score(labels_true, labels_pred, convert_dtype=True) -> float: f"{labels_true.shape} and {labels_pred.shape}" ) check_consistent_length(labels_true, labels_pred) + dims_within_int_limits(n_rows=labels_true.shape[0]) cdef int n_rows = labels_true.shape[0] rand_score = adjusted_rand_index(handle_[0], diff --git a/python/cuml/cuml/metrics/cluster/entropy.pyx b/python/cuml/cuml/metrics/cluster/entropy.pyx index 9aa020905c..4f8863502b 100644 --- a/python/cuml/cuml/metrics/cluster/entropy.pyx +++ b/python/cuml/cuml/metrics/cluster/entropy.pyx @@ -8,6 +8,10 @@ import cupy as cp import numpy as np from cuml.internals import get_handle +from cuml.internals.dimension_limits import ( + dims_within_int_limits, + values_fit_int32, +) from cuml.internals.validation import check_array from libc.stdint cimport uintptr_t @@ -57,9 +61,14 @@ def cython_entropy(clustering, base=None) -> float: f"{clustering.shape}" ) clustering = clustering.ravel() + dims_within_int_limits(n_rows=clustering.shape[0]) cdef int n_rows = clustering.shape[0] lower_class_range = cp.min(clustering).item() upper_class_range = cp.max(clustering).item() + values_fit_int32( + lower_class_range=lower_class_range, + upper_class_range=upper_class_range, + ) cdef uintptr_t clustering_ptr = clustering.data.ptr diff --git a/python/cuml/cuml/metrics/cluster/silhouette_score.pyx b/python/cuml/cuml/metrics/cluster/silhouette_score.pyx index ecef4d959f..88ddb36ea3 100644 --- a/python/cuml/cuml/metrics/cluster/silhouette_score.pyx +++ b/python/cuml/cuml/metrics/cluster/silhouette_score.pyx @@ -6,6 +6,7 @@ import cupy as cp import numpy as np from cuml.internals import get_handle +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.validation import check_array, check_consistent_length from cuml.metrics.pairwise_distances import _determine_metric @@ -79,8 +80,6 @@ def _silhouette_coeff( convert_dtype=convert_dtype, input_name='X', ) - cdef int n_rows = data.shape[0] - cdef int n_cols = data.shape[1] dtype = data.dtype labels = check_array( @@ -98,9 +97,20 @@ def _silhouette_coeff( # Use cp.unique with return_inverse to get monotonic labels efficiently. unique_labels, inverse = cp.unique(labels, return_inverse=True) - cdef int n_labels = unique_labels.shape[0] mono_labels = cp.ascontiguousarray(inverse, dtype=np.int32) + n_rows_py, n_cols_py = int(data.shape[0]), int(data.shape[1]) + n_labels_py = int(unique_labels.shape[0]) + dims_within_int_limits( + n_rows=n_rows_py, + n_cols=n_cols_py, + n_labels=n_labels_py, + chunksize=chunksize, + ) + cdef int n_rows = n_rows_py + cdef int n_cols = n_cols_py + cdef int n_labels = n_labels_py + cdef uintptr_t scores_ptr if sil_scores is None: scores_ptr = NULL diff --git a/python/cuml/cuml/metrics/cluster/utils.py b/python/cuml/cuml/metrics/cluster/utils.py index 714b1a1552..61dcacbb71 100644 --- a/python/cuml/cuml/metrics/cluster/utils.py +++ b/python/cuml/cuml/metrics/cluster/utils.py @@ -4,6 +4,7 @@ # import numpy as np +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.validation import check_array, check_consistent_length from cuml.metrics.utils import sorted_unique_labels from cuml.prims.label import make_monotonic @@ -54,4 +55,6 @@ def prepare_cluster_metric_inputs(labels_true, labels_pred): lower_class_range = 0 upper_class_range = len(classes) - 1 + dims_within_int_limits(n_rows=n_rows, upper_class_range=upper_class_range) + return y_true, y_pred, n_rows, lower_class_range, upper_class_range diff --git a/python/cuml/cuml/metrics/kl_divergence.pyx b/python/cuml/cuml/metrics/kl_divergence.pyx index 26d4e6697e..b7520c5c38 100644 --- a/python/cuml/cuml/metrics/kl_divergence.pyx +++ b/python/cuml/cuml/metrics/kl_divergence.pyx @@ -5,6 +5,7 @@ import numpy as np from cuml.internals import get_handle +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.validation import check_array from libc.stdint cimport uintptr_t @@ -88,6 +89,7 @@ def kl_divergence(P, Q, convert_dtype=True): ) Q_m = Q_m.ravel() + dims_within_int_limits(n_features=P_m.shape[0]) cdef int n_features_p = P_m.shape[0] if Q_m.shape[0] != n_features_p: raise ValueError( diff --git a/python/cuml/cuml/metrics/pairwise_distances.pyx b/python/cuml/cuml/metrics/pairwise_distances.pyx index 617dcdaee7..394022ed73 100644 --- a/python/cuml/cuml/metrics/pairwise_distances.pyx +++ b/python/cuml/cuml/metrics/pairwise_distances.pyx @@ -14,6 +14,7 @@ from cuml.common import CumlArray, input_to_cuml_array from cuml.common.sparse_utils import is_sparse from cuml.internals import get_handle, reflect from cuml.internals.array_sparse import SparseCumlArray +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.input_utils import sparse_scipy_to_cp from cuml.thirdparty_adapters import _get_mask @@ -397,6 +398,12 @@ def pairwise_distances( X.shape[1] == {} while Y.shape[1] == {}" .format(n_features_x, n_features_y)) + dims_within_int_limits( + n_samples_X=n_samples_x, + n_samples_Y=n_samples_y, + n_features=n_features_x, + ) + # Get the metric string to int metric_val = _determine_metric(metric) @@ -574,8 +581,14 @@ def sparse_pairwise_distances( # Get the metric string to a distance enum metric_val = _determine_metric(metric, is_sparse_=True) - x_nrows, y_nrows = X_m.indptr.shape[0] - 1, Y_m.indptr.shape[0] - 1 - dest_m = CumlArray.zeros((x_nrows, y_nrows), dtype=dtype_x) + dims_within_int_limits( + n_samples_x=n_samples_x, + n_samples_y=n_samples_y, + n_features=n_features_x, + X_nnz=X_m.nnz, + Y_nnz=Y_m.nnz, + ) + dest_m = CumlArray.zeros((n_samples_x, n_samples_y), dtype=dtype_x) cdef uintptr_t d_dest_ptr = dest_m.ptr cdef uintptr_t d_X_ptr = X_m.data.ptr @@ -591,8 +604,8 @@ def sparse_pairwise_distances( d_X_ptr, d_Y_ptr, d_dest_ptr, - x_nrows, - y_nrows, + n_samples_x, + n_samples_y, n_features_x, X_m.nnz, Y_m.nnz, diff --git a/python/cuml/cuml/metrics/trustworthiness.pyx b/python/cuml/cuml/metrics/trustworthiness.pyx index baa450ab0f..54487890c1 100644 --- a/python/cuml/cuml/metrics/trustworthiness.pyx +++ b/python/cuml/cuml/metrics/trustworthiness.pyx @@ -5,6 +5,7 @@ import numpy as np from cuml.internals import get_handle +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.input_utils import input_to_cuml_array from libc.stdint cimport uintptr_t @@ -99,6 +100,14 @@ def trustworthiness( else None)) d_X_embedded_ptr = X_m2.ptr + dims_within_int_limits( + n_samples=n_samples, + n_features=n_features, + n_components=n_components, + n_neighbors=n_neighbors, + batch_size=batch_size, + ) + handle = get_handle() cdef handle_t* handle_ = handle.getHandle() diff --git a/python/cuml/cuml/neighbors/kneighbors_classifier.pyx b/python/cuml/cuml/neighbors/kneighbors_classifier.pyx index 2f528fa334..8f902d53ff 100644 --- a/python/cuml/cuml/neighbors/kneighbors_classifier.pyx +++ b/python/cuml/cuml/neighbors/kneighbors_classifier.pyx @@ -12,6 +12,10 @@ from cuml.common.classification import decode_labels from cuml.common.doc_utils import generate_docstring from cuml.internals import get_handle from cuml.internals.array import CumlArray +from cuml.internals.dimension_limits import ( + dims_within_int_limits, + dims_within_size_t_limits, +) from cuml.internals.interop import UnsupportedOnGPU from cuml.internals.mixins import ClassifierMixin, FMajorInputTagMixin from cuml.internals.outputs import reflect, run_in_internal_context @@ -215,9 +219,14 @@ class KNeighborsClassifier(ClassifierMixin, FMajorInputTagMixin, NeighborsBase): ) dists_cp = knn_distances.to_output("cupy") cdef size_t n_rows = inds_cp.shape[0] + out_cols = self._y.shape[1] if self._y.ndim == 2 else 1 + dims_within_size_t_limits( + n_query_rows=n_rows, + n_index_rows=self._y.shape[0], + ) + dims_within_int_limits(n_neighbors=self.n_neighbors, n_output_cols=out_cols) # Allocate array for predictions - out_cols = self._y.shape[1] if self._y.ndim == 2 else 1 out_shape = (n_rows, out_cols) if out_cols > 1 else n_rows out = cp.empty(out_shape, dtype=np.int32, order="C") cdef int* out_ptr = out.data.ptr @@ -286,6 +295,11 @@ class KNeighborsClassifier(ClassifierMixin, FMajorInputTagMixin, NeighborsBase): dists_cp = knn_distances.to_output("cupy") cdef size_t n_rows = inds_cp.shape[0] index = knn_indices.index + dims_within_size_t_limits( + n_query_rows=n_rows, + n_index_rows=self._y.shape[0], + ) + dims_within_int_limits(n_neighbors=self.n_neighbors) if self._y.ndim == 1 or self._y.shape[1] == 1: n_classes = [len(self.classes_)] diff --git a/python/cuml/cuml/neighbors/kneighbors_classifier_mg.pyx b/python/cuml/cuml/neighbors/kneighbors_classifier_mg.pyx index 7bb613320a..87764b387d 100644 --- a/python/cuml/cuml/neighbors/kneighbors_classifier_mg.pyx +++ b/python/cuml/cuml/neighbors/kneighbors_classifier_mg.pyx @@ -7,6 +7,10 @@ import typing from cuml.common import input_to_cuml_array from cuml.internals import logger, reflect from cuml.internals.array import CumlArray +from cuml.internals.dimension_limits import ( + dims_within_int_limits, + dims_within_size_t_limits, +) from cuml.neighbors.nearest_neighbors_mg import NearestNeighborsMG from cython.operator cimport dereference as deref @@ -107,6 +111,15 @@ class KNeighborsClassifierMG(NearestNeighborsMG): uniq_labels_d, _, _, _ = \ input_to_cuml_array(uniq_labels, order='C', check_dtype='int32', convert_to_dtype='int32') + n_outputs = len(n_unique) + dims_within_int_limits( + n_neighbors=self.n_neighbors, + uniq_label_rows=uniq_labels_d.shape[0], + uniq_label_stride=uniq_labels_d.shape[1], + n_outputs=n_outputs, + ) + dims_within_size_t_limits(batch_size=self.batch_size) + cdef int* ptr = uniq_labels_d.ptr cdef vector[int*] *uniq_labels_vec = new vector[int*]() for i in range(uniq_labels_d.shape[0]): @@ -119,8 +132,6 @@ class KNeighborsClassifierMG(NearestNeighborsMG): for uniq_label in n_unique: n_unique_vec.push_back(uniq_label) - n_outputs = len(n_unique) - # Build labels output array for native code interfacing cdef vector[intData_t*] *out_result_local_parts \ = new vector[intData_t*]() @@ -213,6 +224,20 @@ class KNeighborsClassifierMG(NearestNeighborsMG): uniq_labels_d, _, _, _ = \ input_to_cuml_array(uniq_labels, order='C', check_dtype='int32', convert_to_dtype='int32') + + query_cais = input['cais']['query'] + local_query_rows = list(map(lambda x: x.shape[0], query_cais)) + n_local_queries = len(local_query_rows) + n_outputs = len(n_unique) + dims_within_int_limits( + n_neighbors=self.n_neighbors, + uniq_label_rows=uniq_labels_d.shape[0], + uniq_label_stride=uniq_labels_d.shape[1], + n_outputs=n_outputs, + n_local_query_partitions=n_local_queries, + ) + dims_within_size_t_limits(batch_size=self.batch_size) + cdef int* ptr = uniq_labels_d.ptr cdef vector[int*] *uniq_labels_vec = new vector[int*]() for i in range(uniq_labels_d.shape[0]): @@ -225,15 +250,9 @@ class KNeighborsClassifierMG(NearestNeighborsMG): for uniq_label in n_unique: n_unique_vec.push_back(uniq_label) - query_cais = input['cais']['query'] - local_query_rows = list(map(lambda x: x.shape[0], query_cais)) - n_local_queries = len(local_query_rows) - cdef vector[float_ptr_vector] *probas_local_parts \ = new vector[float_ptr_vector](n_local_queries) - n_outputs = len(n_unique) - # Build probas output array for native code interfacing proba_cais = [[] for i in range(n_outputs)] for query_idx, n_rows in enumerate(local_query_rows): diff --git a/python/cuml/cuml/neighbors/kneighbors_regressor.pyx b/python/cuml/cuml/neighbors/kneighbors_regressor.pyx index 648f6b8fa4..8436634ce7 100644 --- a/python/cuml/cuml/neighbors/kneighbors_regressor.pyx +++ b/python/cuml/cuml/neighbors/kneighbors_regressor.pyx @@ -8,6 +8,10 @@ import numpy as np from cuml.common.doc_utils import generate_docstring from cuml.internals import get_handle, reflect from cuml.internals.array import CumlArray +from cuml.internals.dimension_limits import ( + dims_within_int_limits, + dims_within_size_t_limits, +) from cuml.internals.interop import UnsupportedOnGPU from cuml.internals.mixins import FMajorInputTagMixin, RegressorMixin from cuml.internals.validation import check_consistent_length, check_y @@ -211,9 +215,14 @@ class KNeighborsRegressor(RegressorMixin, FMajorInputTagMixin, NeighborsBase): ) dists_cp = knn_distances.to_output("cupy") cdef size_t n_rows = inds_cp.shape[0] + res_cols = 1 if self._y.ndim == 1 else self._y.shape[1] + dims_within_size_t_limits( + n_query_rows=n_rows, + n_index_rows=self._y.shape[0], + ) + dims_within_int_limits(n_neighbors=self.n_neighbors, n_output_cols=res_cols) cdef int64_t* inds_ctype = inds_cp.data.ptr - res_cols = 1 if self._y.ndim == 1 else self._y.shape[1] res_shape = n_rows if res_cols == 1 else (n_rows, res_cols) out = CumlArray.zeros( diff --git a/python/cuml/cuml/neighbors/kneighbors_regressor_mg.pyx b/python/cuml/cuml/neighbors/kneighbors_regressor_mg.pyx index ab2af8bd9e..e795d122c6 100644 --- a/python/cuml/cuml/neighbors/kneighbors_regressor_mg.pyx +++ b/python/cuml/cuml/neighbors/kneighbors_regressor_mg.pyx @@ -6,6 +6,10 @@ import typing from cuml.internals import logger, reflect from cuml.internals.array import CumlArray +from cuml.internals.dimension_limits import ( + dims_within_int_limits, + dims_within_size_t_limits, +) from cuml.neighbors.nearest_neighbors_mg import NearestNeighborsMG from cython.operator cimport dereference as deref @@ -95,6 +99,11 @@ class KNeighborsRegressorMG(NearestNeighborsMG): query_cais = input['cais']['query'] local_query_rows = list(map(lambda x: x.shape[0], query_cais)) + dims_within_int_limits( + n_neighbors=self.n_neighbors, + n_outputs=n_outputs, + ) + dims_within_size_t_limits(batch_size=self.batch_size) # Build labels output array for native code interfacing cdef vector[floatData_t*] *out_result_local_parts \ diff --git a/python/cuml/cuml/neighbors/nearest_neighbors.pyx b/python/cuml/cuml/neighbors/nearest_neighbors.pyx index 43af59d66b..4b7ec43ee4 100644 --- a/python/cuml/cuml/neighbors/nearest_neighbors.pyx +++ b/python/cuml/cuml/neighbors/nearest_neighbors.pyx @@ -17,6 +17,11 @@ from cuml.common.sparse_utils import is_dense, is_sparse from cuml.internals.array import CumlArray from cuml.internals.array_sparse import SparseCumlArray from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import ( + dims_within_int_limits, + dims_within_size_t_limits, + dims_within_uint32_limits, +) from cuml.internals.interop import InteropMixin, UnsupportedOnGPU, to_gpu from cuml.internals.mixins import CMajorInputTagMixin, SparseInputTagMixin from cuml.internals.outputs import reflect @@ -198,6 +203,7 @@ void swap_kernel(long long int* I, float* D, int n_rows, int n_cols) { def _drop_self_edges(distances_cp, indices_cp): """Drop edges between a point and itself in the knn graph""" rows, cols = indices_cp.shape + dims_within_int_limits(n_rows=rows, n_cols=cols) # Launch config threads_per_block = 32 @@ -284,6 +290,7 @@ cdef class RBCIndex: cdef float* X_ptr = X.data.ptr cdef int64_t n_rows = X.shape[0] cdef int64_t n_cols = X.shape[1] + dims_within_size_t_limits(n_rows=n_rows, n_cols=n_cols) cdef DistanceType distance_type = _metric_to_distance_type(metric) with nogil: @@ -313,6 +320,7 @@ cdef class RBCIndex: cdef float* X_ptr = X.data.ptr cdef int64_t n_rows = X.shape[0] cdef int64_t n_cols = X.shape[1] + dims_within_size_t_limits(n_query=n_query, n_rows=n_rows, n_cols=n_cols) cdef int64_t* indptr_ptr = indptr.data.ptr with nogil: @@ -329,6 +337,7 @@ cdef class RBCIndex: ) cdef int64_t nnz = indptr[-1].item() + dims_within_size_t_limits(nnz=nnz) indices = cp.empty(nnz, dtype=np.int64) cdef int64_t* indices_ptr = indices.data.ptr @@ -357,6 +366,10 @@ cdef class RBCIndex: raise ValueError( "The rbc algorithm is not supported for >3 dimensions currently." ) + dims_within_uint32_limits( + n_query_rows=X.shape[0], + n_neighbors=n_neighbors, + ) distances_cp = cp.empty((X.shape[0], n_neighbors), dtype=np.float32, order="C") indices_cp = cp.empty((X.shape[0], n_neighbors), dtype=np.int64, order="C") @@ -435,6 +448,7 @@ cdef class ApproxIndex: cdef DistanceType distance_type = _metric_to_distance_type(metric) cdef handle_t* handle_ = handle.getHandle() cdef float* X_ptr = X.data.ptr + dims_within_int_limits(n_rows=X.shape[0], n_cols=X.shape[1]) cdef int n_rows = X.shape[0] cdef int n_cols = X.shape[1] @@ -456,6 +470,7 @@ cdef class ApproxIndex: def kneighbors(ApproxIndex self, X, int n_neighbors): """Query the index for the k nearest neighbors.""" + dims_within_int_limits(n_rows=X.shape[0], n_neighbors=n_neighbors) distances_cp = cp.empty((X.shape[0], n_neighbors), dtype=np.float32, order="C") indices_cp = cp.empty((X.shape[0], n_neighbors), dtype=np.int64, order="C") @@ -807,6 +822,12 @@ class NeighborsBase(Base, InteropMixin, CMajorInputTagMixin, SparseInputTagMixin ) if index is None: # Special case if X is a CumlArray (self._fit_X forwarded) index = getattr(X, "index", None) + dims_within_int_limits( + n_rows=X_cp.shape[0], + n_cols=X_cp.shape[1], + n_samples_fit=self.n_samples_fit_, + n_neighbors=n_neighbors, + ) cdef int n_rows = X_cp.shape[0] cdef int n_cols = X_cp.shape[1] @@ -906,6 +927,19 @@ class NeighborsBase(Base, InteropMixin, CMajorInputTagMixin, SparseInputTagMixin convert_dtype=True, input_name="X", ) + dims_within_int_limits( + X_n_rows=X_cp.shape[0], + X_n_cols=X_cp.shape[1], + idx_n_rows=self._fit_X.shape[0], + idx_n_cols=self._fit_X.shape[1], + n_neighbors=n_neighbors, + ) + dims_within_size_t_limits( + idx_nnz=self._fit_X.nnz, + X_nnz=X_cp.nnz, + batch_size_index=batch_size_index, + batch_size_query=batch_size_query, + ) cdef int* X_indptr = X_cp.indptr.data.ptr cdef int* X_indices = X_cp.indices.data.ptr cdef float* X_data = X_cp.data.data.ptr diff --git a/python/cuml/cuml/neighbors/nearest_neighbors_mg.pyx b/python/cuml/cuml/neighbors/nearest_neighbors_mg.pyx index 161778450e..80fef5cc57 100644 --- a/python/cuml/cuml/neighbors/nearest_neighbors_mg.pyx +++ b/python/cuml/cuml/neighbors/nearest_neighbors_mg.pyx @@ -8,6 +8,10 @@ from cuml.common import input_to_cuml_array from cuml.common.opg_data_utils_mg import _build_part_inputs from cuml.internals import logger, reflect from cuml.internals.array import CumlArray +from cuml.internals.dimension_limits import ( + dims_within_int_limits, + dims_within_size_t_limits, +) from cuml.neighbors import NearestNeighbors from cython.operator cimport dereference as deref @@ -95,6 +99,8 @@ class NearestNeighborsMG(NearestNeighbors): self.n_neighbors = self.n_neighbors if n_neighbors is None \ else n_neighbors + dims_within_int_limits(n_neighbors=self.n_neighbors) + dims_within_size_t_limits(batch_size=self.batch_size) # Build input arrays and descriptors for native code interfacing input = type(self).gen_local_input( @@ -177,6 +183,7 @@ class NearestNeighborsMG(NearestNeighbors): outputs = [d[1] for d in index] n_out = len(outputs) + dims_within_int_limits(n_label_partitions=n_out) if dtype == 'int32': out_local_parts_i32 = new vector[int_ptr_vector](n_out) @@ -218,6 +225,10 @@ class NearestNeighborsMG(NearestNeighbors): @staticmethod def alloc_local_output(local_query_rows, n_neighbors): + dims_within_int_limits(n_neighbors=n_neighbors) + for n_rows in local_query_rows: + dims_within_int_limits(partition_query_rows=n_rows) + cdef vector[int64Data_t*] *indices_local_parts \ = new vector[int64Data_t*]() cdef vector[floatData_t*] *dist_local_parts \ diff --git a/python/cuml/cuml/solvers/cd.pyx b/python/cuml/cuml/solvers/cd.pyx index 925fbe83d1..0b66b8976b 100644 --- a/python/cuml/cuml/solvers/cd.pyx +++ b/python/cuml/cuml/solvers/cd.pyx @@ -8,6 +8,7 @@ from cuml.common import CumlArray from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.common.doc_utils import generate_docstring from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.mixins import FMajorInputTagMixin from cuml.internals.outputs import reflect from cuml.internals.validation import check_inputs, check_is_fitted @@ -142,6 +143,8 @@ def fit_cd( # Allocate outputs coef = cp.zeros(X.shape[1], dtype=X.dtype) + dims_within_int_limits(n_rows=X.shape[0], n_cols=X.shape[1]) + cdef int n_rows = X.shape[0] cdef int n_cols = X.shape[1] cdef uintptr_t X_ptr = X.data.ptr @@ -356,6 +359,8 @@ class CD(Base, FMajorInputTagMixin): ) preds = cp.zeros(X.shape[0], dtype=self.coef_.dtype, order="F") + dims_within_int_limits(n_rows=X.shape[0], n_cols=X.shape[1]) + cdef int n_rows = X.shape[0] cdef int n_cols = X.shape[1] cdef uintptr_t X_ptr = X.data.ptr diff --git a/python/cuml/cuml/solvers/cd_mg.pyx b/python/cuml/cuml/solvers/cd_mg.pyx index 285593ede9..930e4919de 100644 --- a/python/cuml/cuml/solvers/cd_mg.pyx +++ b/python/cuml/cuml/solvers/cd_mg.pyx @@ -5,6 +5,7 @@ import numpy as np from cuml.internals import run_in_internal_context +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.linear_model.base_mg import MGFitMixin from cuml.solvers import CD @@ -64,6 +65,7 @@ class CDMG(MGFitMixin, CD): cdef handle_t* handle_ = self.handle.getHandle() cdef bool use_f32 = self.dtype == np.float32 cdef bool fit_intercept = self.fit_intercept + dims_within_int_limits(max_iter=self.max_iter) cdef int max_iter = self.max_iter cdef double alpha = ( self.alpha if np.isscalar(self.alpha) else self.alpha.item() diff --git a/python/cuml/cuml/solvers/qn.pyx b/python/cuml/cuml/solvers/qn.pyx index b70c611e27..668675d45f 100644 --- a/python/cuml/cuml/solvers/qn.pyx +++ b/python/cuml/cuml/solvers/qn.pyx @@ -10,6 +10,7 @@ from cuml.common.classification import process_class_weight from cuml.common.doc_utils import generate_docstring from cuml.internals.array import CumlArray from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.outputs import reflect, run_in_internal_context from cuml.internals.validation import ( check_array, @@ -248,6 +249,13 @@ def fit_qn( raise ValueError(f"Expected coef.shape == ({coef_shape}), got {coef.shape}") cdef bool sparse_X = sp.issparse(X) + dims_within_int_limits( + n_rows=X.shape[0], + n_cols=X.shape[1], + n_classes=n_classes or 0, + ) + if sparse_X: + dims_within_int_limits(X_nnz=X.nnz) cdef int n_rows = X.shape[0] cdef int n_cols = X.shape[1] cdef uintptr_t X_ptr, X_indices_ptr, X_indptr_ptr diff --git a/python/cuml/cuml/solvers/sgd.pyx b/python/cuml/cuml/solvers/sgd.pyx index 5d06e7d641..bd404b1ea4 100644 --- a/python/cuml/cuml/solvers/sgd.pyx +++ b/python/cuml/cuml/solvers/sgd.pyx @@ -7,6 +7,7 @@ from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.common.doc_utils import generate_docstring from cuml.internals.array import CumlArray from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.mixins import FMajorInputTagMixin from cuml.internals.outputs import reflect from cuml.internals.validation import check_inputs @@ -194,6 +195,14 @@ def fit_sgd( # Allocate outputs coef = cp.zeros(X.shape[1], dtype=X.dtype) + dims_within_int_limits( + n_rows=X.shape[0], + n_cols=X.shape[1], + batch_size=batch_size, + epochs=epochs, + n_iter_no_change=n_iter_no_change, + ) + # Perform fit handle = get_handle() cdef handle_t* handle_ = handle.getHandle() @@ -462,6 +471,8 @@ class SGD(Base, FMajorInputTagMixin): preds = cp.zeros(X.shape[0], dtype=self.coef_.dtype) + dims_within_int_limits(n_rows=X.shape[0], n_cols=X.shape[1]) + handle = get_handle() cdef handle_t* handle_ = handle.getHandle() cdef int loss_code = _LOSSES[self.loss] diff --git a/python/cuml/cuml/svm/linear.pyx b/python/cuml/cuml/svm/linear.pyx index 0bce9a0977..64b7a5ab2a 100644 --- a/python/cuml/cuml/svm/linear.pyx +++ b/python/cuml/cuml/svm/linear.pyx @@ -5,6 +5,10 @@ import cupy as cp from cuml.common.classification import process_class_weight from cuml.internals.base import get_handle +from cuml.internals.dimension_limits import ( + dims_within_int_limits, + dims_within_size_t_limits, +) from cuml.internals.validation import check_inputs from libc.stdint cimport uintptr_t @@ -187,6 +191,7 @@ def fit( # Extract dimensions cdef size_t n_rows = out[0].shape[0] cdef size_t n_cols = out[0].shape[1] + dims_within_size_t_limits(n_rows=n_rows, n_cols=n_cols) cdef int n_classes n_coefs = n_cols + int(fit_intercept) @@ -214,6 +219,8 @@ def fit( classes = class_codes = None w_shape = n_coefs + dims_within_int_limits(n_classes=n_classes) + # Allocate output arrays w = cp.empty(shape=w_shape, dtype=X.dtype, order="F") if probability and is_classifier: @@ -300,9 +307,13 @@ def compute_probabilities(scores, prob_scale, n_streams): prob_scale = cp.asarray(prob_scale, order="F") scores = cp.asarray(scores, order="C", dtype=prob_scale.dtype) - # Extract dimensions - cdef size_t n_rows = scores.shape[0] - cdef int n_classes = prob_scale.shape[0] + # Extract dimensions (validate before narrowing to native widths) + n_rows_py = int(scores.shape[0]) + n_classes_py = int(prob_scale.shape[0]) + dims_within_size_t_limits(n_rows=n_rows_py) + dims_within_int_limits(n_classes=n_classes_py) + cdef size_t n_rows = n_rows_py + cdef int n_classes = n_classes_py # Allocate outputs out = cp.empty((n_rows, n_classes), dtype=scores.dtype, order="C") diff --git a/python/cuml/cuml/svm/svm_base.pyx b/python/cuml/cuml/svm/svm_base.pyx index c22d9417b9..a84a42c01a 100644 --- a/python/cuml/cuml/svm/svm_base.pyx +++ b/python/cuml/cuml/svm/svm_base.pyx @@ -11,6 +11,7 @@ from cuml.common.sparse_utils import is_sparse from cuml.internals.array import CumlArray from cuml.internals.array_sparse import SparseCumlArray from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.interop import ( InteropMixin, UnsupportedOnGPU, @@ -509,8 +510,26 @@ class SVMBase(Base, cdef bool sparse_X = cupyx.scipy.sparse.issparse(X) cdef bool is_float32 = X.dtype == np.float32 + n_cols_fit_py = int(self.shape_fit_[1]) + n_support_py = int(self.support_.shape[0]) + X_rows_py, X_cols_py = map(int, X.shape) + + dims_within_int_limits( + n_cols_fit=n_cols_fit_py, + n_support=n_support_py, + X_rows=X_rows_py, + X_cols=X_cols_py, + ) + if sparse_X: + dims_within_int_limits(X_nnz=int(X.nnz)) + # Extract support_vectors_ - cdef int support_nnz = support_vectors.nnz if sparse_model else -1 + cdef int support_nnz + if sparse_model: + support_nnz = int(support_vectors.nnz) + dims_within_int_limits(support_nnz=support_nnz) + else: + support_nnz = -1 cdef int* support_indptr = ( support_vectors.indptr.ptr if sparse_model else 0 ) @@ -524,11 +543,11 @@ class SVMBase(Base, # Setup SvmModel of proper type # Use shape_fit_[1] for n_cols to handle the no-support-vectors case correctly # (support_vectors.shape[1] would be 0 when there are no support vectors) - cdef int n_cols_fit = self.shape_fit_[1] + cdef int n_cols_fit = n_cols_fit_py cdef lib.SvmModel[float] model_f cdef lib.SvmModel[double] model_d if is_float32: - model_f.n_support = self.support_.shape[0] + model_f.n_support = n_support_py model_f.n_cols = n_cols_fit model_f.b = self.intercept_.item() model_f.dual_coefs = self.dual_coef_.ptr @@ -538,7 +557,7 @@ class SVMBase(Base, model_f.support_matrix.indices = support_indices model_f.support_matrix.data = support_data_ptr else: - model_d.n_support = self.support_.shape[0] + model_d.n_support = n_support_py model_d.n_cols = n_cols_fit model_d.b = self.intercept_.item() model_d.dual_coefs = self.dual_coef_.ptr @@ -560,7 +579,8 @@ class SVMBase(Base, cdef int *X_indices cdef uintptr_t X_data_ptr cdef int X_rows, X_cols, X_nnz - X_rows, X_cols = X.shape + X_rows = X_rows_py + X_cols = X_cols_py if sparse_X: X_indptr = X.indptr.data.ptr X_indices = X.indices.data.ptr diff --git a/python/cuml/cuml/tsa/arima.pyx b/python/cuml/cuml/tsa/arima.pyx index c20106ea81..986cbd97f0 100644 --- a/python/cuml/cuml/tsa/arima.pyx +++ b/python/cuml/cuml/tsa/arima.pyx @@ -10,6 +10,7 @@ from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.internals import logger, nvtx, reflect, run_in_internal_context from cuml.internals.array import CumlArray from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import INT32_MAX, dims_within_int_limits from cuml.internals.input_utils import input_to_cuml_array from cuml.tsa.batched_lbfgs import batched_fmin_lbfgs_b @@ -278,6 +279,21 @@ class ARIMA(Base): sma_ = CumlArrayDescriptor() sigma2_ = CumlArrayDescriptor() + def _assert_arima_core_dims(self): + """Ensure batch/observation counts fit libcuml ``int`` parameters.""" + dims_within_int_limits( + batch_size=self.batch_size, + n_obs=self.n_obs, + n_obs_diff=self.n_obs_diff, + ) + prod = int(self.batch_size) * int(self.n_obs) + if prod > INT32_MAX: + raise ValueError( + f"batch_size * n_obs ({prod}) exceeds maximum value ({INT32_MAX}) " + "supported by this binding when passed to native code as a 32-bit " + "signed integer." + ) + def __init__(self, endog, *, @@ -351,6 +367,15 @@ class ARIMA(Base): cpp_order.n_exog = n_exog self.order = cpp_order + if n_exog > 0: + prod_be = int(self.batch_size) * int(n_exog) + if prod_be > INT32_MAX: + raise ValueError( + f"batch_size * n_exog ({prod_be}) exceeds maximum value ({INT32_MAX}) " + "supported by this binding when passed to native code as a 32-bit " + "signed integer." + ) + self.simple_differencing = simple_differencing self._d_y_diff = CumlArray.empty( @@ -362,6 +387,8 @@ class ARIMA(Base): self.n_obs_diff = self.n_obs - d - D * s + self._assert_arima_core_dims() + # Allocate temporary storage temp_mem_size = ARIMAMemory[double].compute_size( cpp_order, self.batch_size, self.n_obs) @@ -658,6 +685,7 @@ class ARIMA(Base): handle = get_handle() cdef handle_t* handle_ = handle.getHandle() predict_size = end - start + dims_within_int_limits(start=start, end=end, predict_size=predict_size) # Future values of the exogenous variables cdef uintptr_t d_exog_fut_ptr = NULL @@ -939,6 +967,7 @@ class ARIMA(Base): loglike : numpy.ndarray Batched log-likelihood. Shape: (batch_size,) """ + dims_within_int_limits(truncate=int(truncate)) cdef vector[double] vec_loglike vec_loglike.resize(self.batch_size) @@ -1014,6 +1043,7 @@ class ARIMA(Base): Batched log-likelihood gradient. Shape: (n_params * batch_size,) where n_params is the complexity of the model """ + dims_within_int_limits(truncate=int(truncate)) N = self.complexity assert len(x) == N * self.batch_size diff --git a/python/cuml/cuml/tsa/auto_arima.pyx b/python/cuml/cuml/tsa/auto_arima.pyx index 34d4e059d8..d591b34c86 100644 --- a/python/cuml/cuml/tsa/auto_arima.pyx +++ b/python/cuml/cuml/tsa/auto_arima.pyx @@ -13,6 +13,7 @@ from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.internals import logger, reflect, run_in_internal_context from cuml.internals.array import CumlArray from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import INT32_MAX, dims_within_int_limits from cuml.tsa.arima import ARIMA from cuml.tsa.seasonality import seas_test from cuml.tsa.stationarity import kpss_test @@ -175,6 +176,14 @@ class AutoARIMA(Base): self.simple_differencing = simple_differencing + dims_within_int_limits(batch_size=self.batch_size, n_obs=self.n_obs) + if int(self.batch_size) * int(self.n_obs) > INT32_MAX: + raise ValueError( + f"batch_size * n_obs ({int(self.batch_size) * int(self.n_obs)}) exceeds " + f"maximum value ({INT32_MAX}) supported by this binding when passed to " + "native code as a 32-bit signed integer." + ) + self._initial_calc() @run_in_internal_context @@ -589,6 +598,8 @@ def _divide_by_mask(original, mask, batch_id): n_obs = original.shape[0] batch_size = original.shape[1] if len(original.shape) > 1 else 1 + dims_within_int_limits(n_obs=n_obs, batch_size=batch_size) + handle = get_handle() cdef handle_t* handle_ = handle.getHandle() @@ -701,6 +712,8 @@ def _divide_by_min(original, metrics, batch_id): n_sub = metrics.shape[1] batch_size = original.shape[1] if len(original.shape) > 1 else 1 + dims_within_int_limits(n_obs=n_obs, batch_size=batch_size, n_sub=n_sub) + handle = get_handle() cdef handle_t* handle_ = handle.getHandle() @@ -812,6 +825,8 @@ def _build_division_map(id_tracker, batch_size): n_sub = len(id_tracker) + dims_within_int_limits(batch_size=batch_size, n_sub=n_sub) + id_to_pos = CumlArray.empty(batch_size, np.int32) id_to_model = CumlArray.empty(batch_size, np.int32) @@ -864,6 +879,8 @@ def _merge_series(data_in, id_to_sub, id_to_pos, batch_size): n_obs = data_in[0].shape[0] n_sub = len(data_in) + dims_within_int_limits(n_obs=n_obs, batch_size=batch_size, n_sub=n_sub) + handle = get_handle() cdef handle_t* handle_ = handle.getHandle() diff --git a/python/cuml/cuml/tsa/holtwinters.pyx b/python/cuml/cuml/tsa/holtwinters.pyx index f2a30a7dd2..02ec878bf9 100644 --- a/python/cuml/cuml/tsa/holtwinters.pyx +++ b/python/cuml/cuml/tsa/holtwinters.pyx @@ -9,6 +9,7 @@ from cuml.common import using_output_type from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.internals.array import CumlArray from cuml.internals.base import Base, get_handle +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.input_utils import input_to_cupy_array from cuml.internals.outputs import run_in_internal_context @@ -273,6 +274,13 @@ class ExponentialSmoothing(Base): raise ValueError("Time series must contain at least 1 value." " Given: " + str(self.n)) + dims_within_int_limits( + n=self.n, + ts_num=self.ts_num, + seasonal_periods=self.seasonal_periods, + start_periods=self.start_periods, + ) + cdef uintptr_t input_ptr cdef int leveltrend_seed_len, season_seed_len, components_len cdef int leveltrend_coef_offset, season_coef_offset @@ -373,6 +381,8 @@ class ExponentialSmoothing(Base): if h <= 0: raise ValueError("h must be > 0. Currently: " + str(h)) + dims_within_int_limits(h=h) + if h > self.h: self.h = h self.forecasted_points = CumlArray.zeros(self.ts_num*h, diff --git a/python/cuml/cuml/tsa/stationarity.pyx b/python/cuml/cuml/tsa/stationarity.pyx index b1ec77ff9f..99b130d3de 100644 --- a/python/cuml/cuml/tsa/stationarity.pyx +++ b/python/cuml/cuml/tsa/stationarity.pyx @@ -4,6 +4,7 @@ import numpy as np from cuml.internals import get_handle, reflect from cuml.internals.array import CumlArray +from cuml.internals.dimension_limits import dims_within_int_limits from cuml.internals.input_utils import input_to_cuml_array from libc.stdint cimport uintptr_t @@ -64,6 +65,14 @@ def kpss_test(y, d=0, D=0, s=0, pval_threshold=0.05, convert_dtype=True) -> Cuml check_dtype=[np.float32, np.float64]) cdef uintptr_t d_y_ptr = d_y.ptr + dims_within_int_limits( + batch_size=batch_size, + n_obs=n_obs, + d=d, + D=D, + s=s, + ) + handle = get_handle() cdef handle_t* handle_ = handle.getHandle()