diff --git a/python/cuml/cuml/metrics/pairwise_distances.pyx b/python/cuml/cuml/metrics/pairwise_distances.pyx index 617dcdaee7..6e7108bc6f 100644 --- a/python/cuml/cuml/metrics/pairwise_distances.pyx +++ b/python/cuml/cuml/metrics/pairwise_distances.pyx @@ -10,11 +10,12 @@ import numpy as np import pandas as pd import scipy.sparse -from cuml.common import CumlArray, input_to_cuml_array +from cuml.common import CumlArray from cuml.common.sparse_utils import is_sparse from cuml.internals import get_handle, reflect from cuml.internals.array_sparse import SparseCumlArray from cuml.internals.input_utils import sparse_scipy_to_cp +from cuml.internals.validation import check_array from cuml.thirdparty_adapters import _get_mask from libc.stdint cimport uintptr_t @@ -147,13 +148,15 @@ def nan_euclidean_distances( Parameters ---------- - X : Dense matrix of shape (n_samples_X, n_features) - Acceptable formats: cuDF DataFrame, Pandas DataFrame, NumPy ndarray, - cuda array interface compliant array like CuPy. + X : array-like (device or host) of shape (n_samples_X, n_features) + Acceptable formats: cuDF DataFrame, NumPy ndarray, Numba device + ndarray, cuda array interface compliant array like CuPy. - Y : Dense matrix of shape (n_samples_Y, n_features), default=None - Acceptable formats: cuDF DataFrame, Pandas DataFrame, NumPy ndarray, - cuda array interface compliant array like CuPy. + Y : array-like (device or host) of shape (n_samples_Y, n_features), \ + default=None + A second feature array. If ``None``, ``Y`` is assumed to be ``X``. + Acceptable formats: cuDF DataFrame, NumPy ndarray, Numba device + ndarray, cuda array interface compliant array like CuPy. squared : bool, default=False Return squared Euclidean distances. @@ -161,11 +164,16 @@ def nan_euclidean_distances( missing_values : np.nan or int, default=np.nan Representation of missing value. + convert_dtype : bool, optional (default = True) + When set to True, the method will, when necessary, convert ``X`` + to a supported floating-point dtype and convert ``Y`` to match + ``X``'s dtype. This will increase memory used for the method. + Returns ------- - distances : ndarray of shape (n_samples_X, n_samples_Y) - Returns the distances between the row vectors of `X` - and the row vectors of `Y`. + distances : array of shape (n_samples_X, n_samples_Y) + Returns the distances between the row vectors of ``X`` + and the row vectors of ``Y``. """ if isinstance(X, cudf.DataFrame) or isinstance(X, pd.DataFrame): @@ -176,23 +184,27 @@ def nan_euclidean_distances( if (Y.isnull().any()).any(): Y.fillna(0, inplace=True) - X_m, _n_samples_x, _n_features_x, dtype_x = \ - input_to_cuml_array(X, - order="K", - convert_to_dtype=(np.float32 if convert_dtype - else None), - check_dtype=[np.float32, np.float64]) + X_m = check_array( + X, + order="A", + dtype=[np.float32, np.float64], + convert_dtype=convert_dtype, + ensure_all_finite=False, + input_name="X", + ) + dtype_x = X_m.dtype if Y is None: - Y = X_m - - Y_m, _n_samples_y, _n_features_y, _dtype_y = \ - input_to_cuml_array( - Y, order=X_m.order, convert_to_dtype=dtype_x, - check_dtype=[dtype_x]) - - X_m = cp.asarray(X_m) - Y_m = cp.asarray(Y_m) + Y_m = X_m + else: + Y_m = check_array( + Y, + order="F" if X_m.flags.f_contiguous else "C", + dtype=[dtype_x], + convert_dtype=convert_dtype, + ensure_all_finite=False, + input_name="Y", + ) # Get missing mask for X missing_X = _get_mask(X_m, missing_values) @@ -269,16 +281,17 @@ def pairwise_distances( Parameters ---------- - X : Dense or sparse matrix (device or host) of shape + X : {array-like, sparse matrix} (device or host) of shape \ (n_samples_x, n_features) Acceptable formats: cuDF DataFrame, NumPy ndarray, Numba device ndarray, cuda array interface compliant array like CuPy, or - cupyx.scipy.sparse for sparse input + cupyx.scipy.sparse for sparse input. - Y : array-like (device or host) of shape (n_samples_y, n_features),\ - optional + Y : array-like (device or host) of shape (n_samples_y, n_features), \ + default=None + A second feature array. If ``None``, ``Y`` is assumed to be ``X``. Acceptable formats: cuDF DataFrame, NumPy ndarray, Numba device - ndarray, cuda array interface compliant array like CuPy + ndarray, cuda array interface compliant array like CuPy. metric : {"cityblock", "cosine", "euclidean", "l1", "l2", "manhattan", \ "sqeuclidean"} @@ -347,49 +360,54 @@ def pairwise_distances( X = np.where(X != 0., 1.0, 0.0) # Get the input arrays, preserve order and type where possible - X_m, n_samples_x, n_features_x, dtype_x = \ - input_to_cuml_array(X, - order="K", - convert_to_dtype=(np.float32 if convert_dtype - else None), - check_dtype=[np.float32, np.float64]) - - # Get the order from the CumlArray - input_order = X_m.order + X_m = check_array( + X, + order="A", + dtype=[np.float32, np.float64], + convert_dtype=convert_dtype, + input_name="X", + ) + cdef int n_samples_x = X_m.shape[0] + cdef int n_features_x = X_m.shape[1] + dtype_x = X_m.dtype cdef uintptr_t d_X_ptr cdef uintptr_t d_Y_ptr cdef uintptr_t d_dest_ptr + cdef bint is_row_major = X_m.flags.c_contiguous + cdef int n_samples_y = n_samples_x + cdef int n_features_y = n_features_x - if (Y is not None): - - # Check for the odd case where one dimension of X is 1. In this case, - # CumlArray always returns order=="C" so instead get the order from Y - if (n_samples_x == 1 or n_features_x == 1): - input_order = "K" - + if Y is not None: if metric in ['russellrao'] and not np.all(Y.data == 1.): warnings.warn("Y was converted to boolean for metric {}" .format(metric)) Y = np.where(Y != 0., 1.0, 0.0) - Y_m, n_samples_y, n_features_y, dtype_y = \ - input_to_cuml_array(Y, order=input_order, - convert_to_dtype=(dtype_x if convert_dtype - else None), - check_dtype=[dtype_x]) - # Get the order from Y if necessary (It's possible to set order="F" in - # input_to_cuml_array and have Y_m.order=="C") - if (input_order == "K"): - input_order = Y_m.order + if n_samples_x == 1 or n_features_x == 1: + # X is degenerate (both C- and F-contiguous); let Y choose the + # layout and propagate it. + Y_m = check_array( + Y, + order="A", + dtype=[dtype_x], + convert_dtype=convert_dtype, + input_name="Y", + ) + is_row_major = Y_m.flags.c_contiguous + else: + # X is the authority; force Y's layout to match X's. + Y_m = check_array( + Y, + order="C" if is_row_major else "F", + dtype=[dtype_x], + convert_dtype=convert_dtype, + input_name="Y", + ) + n_samples_y = Y_m.shape[0] + n_features_y = Y_m.shape[1] else: - # Shallow copy X variables Y_m = X_m - n_samples_y = n_samples_x - n_features_y = n_features_x - dtype_y = dtype_x - - is_row_major = input_order == "C" # Check feature sizes are equal if (n_features_x != n_features_y): @@ -402,10 +420,10 @@ def pairwise_distances( # Create the output array dest_m = CumlArray.zeros((n_samples_x, n_samples_y), dtype=dtype_x, - order=input_order) + order="C" if is_row_major else "F") - d_X_ptr = X_m.ptr - d_Y_ptr = Y_m.ptr + d_X_ptr = X_m.data.ptr + d_Y_ptr = Y_m.data.ptr d_dest_ptr = dest_m.ptr # Now execute the functions @@ -414,9 +432,9 @@ def pairwise_distances( d_X_ptr, d_Y_ptr, d_dest_ptr, - n_samples_x, - n_samples_y, - n_features_x, + n_samples_x, + n_samples_y, + n_features_x, metric_val, is_row_major, metric_arg) @@ -425,9 +443,9 @@ def pairwise_distances( d_X_ptr, d_Y_ptr, d_dest_ptr, - n_samples_x, - n_samples_y, - n_features_x, + n_samples_x, + n_samples_y, + n_features_x, metric_val, is_row_major, metric_arg) diff --git a/python/cuml/cuml/metrics/pairwise_kernels.py b/python/cuml/cuml/metrics/pairwise_kernels.py index 5e19b0457a..0caa589dd3 100644 --- a/python/cuml/cuml/metrics/pairwise_kernels.py +++ b/python/cuml/cuml/metrics/pairwise_kernels.py @@ -202,18 +202,18 @@ def pairwise_kernels( Parameters ---------- - X : Dense matrix (device or host) of shape (n_samples_X, n_samples_X) or \ + X : array-like (device or host) of shape (n_samples_X, n_samples_X) or \ (n_samples_X, n_features) Array of pairwise kernels between samples, or a feature array. The shape of the array should be (n_samples_X, n_samples_X) if metric == "precomputed" and (n_samples_X, n_features) otherwise. Acceptable formats: cuDF DataFrame, NumPy ndarray, Numba device - ndarray, cuda array interface compliant array like CuPy - Y : Dense matrix (device or host) of shape (n_samples_Y, n_features), \ + ndarray, cuda array interface compliant array like CuPy. + Y : array-like (device or host) of shape (n_samples_Y, n_features), \ default=None A second feature array only if X has shape (n_samples_X, n_features). Acceptable formats: cuDF DataFrame, NumPy ndarray, Numba device - ndarray, cuda array interface compliant array like CuPy + ndarray, cuda array interface compliant array like CuPy. metric : str or callable (numba device function), default="linear" The metric to use when calculating kernel between instances in a feature array. diff --git a/python/cuml/tests/test_metrics.py b/python/cuml/tests/test_metrics.py index 6e135f01ec..4ea5981910 100644 --- a/python/cuml/tests/test_metrics.py +++ b/python/cuml/tests/test_metrics.py @@ -52,6 +52,7 @@ from cuml.metrics import kl_divergence as cu_kl_divergence from cuml.metrics import ( log_loss, + nan_euclidean_distances, pairwise_distances, precision_recall_curve, roc_auc_score, @@ -1470,7 +1471,7 @@ def test_pairwise_distances_exceptions(): pairwise_distances(X_double, X_int, metric="euclidean") # Test sending different types with convert_dtype=False - with pytest.raises(TypeError): + with pytest.raises(ValueError, match="dtype"): pairwise_distances( X_double, X_float, metric="euclidean", convert_dtype=False ) @@ -1487,6 +1488,56 @@ def test_pairwise_distances_exceptions(): pairwise_distances(X, Y, metric="euclidean") +@pytest.mark.parametrize("bad_value", [np.nan, np.inf, -np.inf]) +@pytest.mark.parametrize("position", ["X", "Y"]) +def test_pairwise_distances_rejects_non_finite(bad_value, position): + rng = np.random.RandomState(0) + X = rng.random_sample((5, 4)).astype(np.float64) + Y = rng.random_sample((6, 4)).astype(np.float64) + if position == "X": + X[0, 0] = bad_value + else: + Y[0, 0] = bad_value + with pytest.raises(ValueError): + pairwise_distances(X, Y, metric="euclidean") + + +def test_nan_euclidean_distances_allows_nan(): + rng = np.random.RandomState(0) + X = rng.random_sample((5, 4)).astype(np.float64) + X[0, 0] = np.nan + S = pairwise_distances(X, metric="nan_euclidean") + S_ref = sklearn_pairwise_distances(X, metric="nan_euclidean") + cp.testing.assert_array_almost_equal(cp.asnumpy(S), S_ref, decimal=4) + + +def test_nan_euclidean_distances_y_none_diagonal_zero(): + rng = np.random.RandomState(0) + X = rng.random_sample((6, 4)).astype(np.float64) + X[0, 0] = np.nan + S = cp.asnumpy(nan_euclidean_distances(X)) + np.testing.assert_array_equal(np.diag(S), np.zeros(X.shape[0])) + # And the off-diagonal values should still match sklearn. + S_ref = sklearn_pairwise_distances(X, metric="nan_euclidean") + np.testing.assert_array_almost_equal(S, S_ref, decimal=4) + + +@pytest.mark.parametrize( + "x_order,y_order", + [("C", "C"), ("C", "F"), ("F", "C"), ("F", "F")], +) +def test_pairwise_distances_degenerate_x_layout(x_order, y_order): + # When X has a degenerate shape (1 sample), it is both C- and + # F-contiguous, so the implementation lets Y choose the layout. + # Verify all four input layout combinations match sklearn. + rng = np.random.RandomState(0) + X = np.asarray(rng.random_sample((1, 4)), order=x_order, dtype=np.float64) + Y = np.asarray(rng.random_sample((10, 4)), order=y_order, dtype=np.float64) + S = cp.asnumpy(pairwise_distances(X, Y, metric="euclidean")) + S_ref = sklearn_pairwise_distances(X, Y, metric="euclidean") + np.testing.assert_array_almost_equal(S, S_ref, decimal=12) + + @pytest.mark.parametrize("input_type", ["cudf", "numpy", "cupy"]) @pytest.mark.parametrize("output_type", ["cudf", "numpy", "cupy"]) @pytest.mark.parametrize("use_global", [True, False])