Support recent hdbscan

conda/environments/all_cuda-129_arch-aarch64.yaml

@@ -25,7 +25,7 @@ dependencies:
 - doxygen=1.9.1
 - gcc_linux-aarch64=14.*
 - graphviz
-- hdbscan>=0.8.39,<0.8.40
+- hdbscan>=0.8.39
 - hypothesis>=6.0,<7
 - ipykernel
 - ipython

conda/environments/all_cuda-129_arch-x86_64.yaml

@@ -25,7 +25,7 @@ dependencies:
 - doxygen=1.9.1
 - gcc_linux-64=14.*
 - graphviz
-- hdbscan>=0.8.39,<0.8.40
+- hdbscan>=0.8.39
 - hypothesis>=6.0,<7
 - ipykernel
 - ipython

conda/environments/all_cuda-131_arch-aarch64.yaml

@@ -25,7 +25,7 @@ dependencies:
 - doxygen=1.9.1
 - gcc_linux-aarch64=14.*
 - graphviz
-- hdbscan>=0.8.39,<0.8.40
+- hdbscan>=0.8.39
 - hypothesis>=6.0,<7
 - ipykernel
 - ipython

conda/environments/all_cuda-131_arch-x86_64.yaml

@@ -25,7 +25,7 @@ dependencies:
 - doxygen=1.9.1
 - gcc_linux-64=14.*
 - graphviz
-- hdbscan>=0.8.39,<0.8.40
+- hdbscan>=0.8.39
 - hypothesis>=6.0,<7
 - ipykernel
 - ipython

dependencies.yaml

@@ -520,18 +520,21 @@ dependencies:
             packages:
               - scikit-learn==1.5.0
               - umap-learn==0.5.7
+              - hdbscan==0.8.39
           - matrix: {dependencies: "intermediate"}
             packages:
               - scikit-learn==1.7.2
               - umap-learn==0.5.8
               - ipython==8.10.0
+              - hdbscan==0.8.41
           - matrix: {dependencies: "nightly"}
             packages:
               # Pre-release specifier tells pip to accept dev/alpha/beta versions
               - scikit-learn>=1.5.0a0
           - matrix:
             packages:
               - *scikit_learn
+              - hdbscan>=0.8.39
               - ipython>=7.32.0
       # ipython for "oldest" is split: conda uses 8.x (7.x has no Python 3.11+ packages),
       # pip uses 7.34.0 from PyPI (which supports Python 3.11+)
@@ -582,7 +585,6 @@ dependencies:
         packages:
           - certifi
           - *cython
-          - hdbscan>=0.8.39,<0.8.40
           - hypothesis>=6.0,<7
           - nltk
           # upstream sklearn docstring tests require numpydoc<1.9

python/cuml/cuml/cluster/hdbscan/hdbscan.pyx

@@ -42,6 +42,16 @@ def import_hdbscan():
         ) from exc
 
 
+def hdbscan_at_least(min_version):
+    """Get the version of hdbscan installed"""
+    # hdbscan doesn't have `hdbscan.__version__`, so we have to
+    # check the package metadata
+    from importlib.metadata import version
+
+    from packaging.version import Version
+    return Version(version("hdbscan")) >= Version(min_version)
+
+
 _metrics_mapping = {
     "l2": DistanceType.L2SqrtExpanded,
     "euclidean": DistanceType.L2SqrtExpanded,
@@ -855,11 +865,16 @@ class HDBSCAN(Base, InteropMixin, ClusterMixin, CMajorInputTagMixin):
     def condensed_tree_(self):
         if self._state is not None:
             hdbscan = import_hdbscan()
-            return hdbscan.plots.CondensedTree(
-                self._condensed_tree,
-                self.cluster_selection_method,
-                self.allow_single_cluster
-            )
+            if hdbscan_at_least("0.8.41"):
+                with cuml.using_output_type("numpy"):
+                    labels = self.labels_
+                return hdbscan.plots.CondensedTree(self._condensed_tree, labels)
+            else:
+                return hdbscan.plots.CondensedTree(
+                    self._condensed_tree,
+                    self.cluster_selection_method,
+                    self.allow_single_cluster
+                )
         raise AttributeError("No condensed tree was generated; try running fit first.")
 
     @property

python/cuml/pyproject.toml

@@ -108,7 +108,7 @@ classifiers = [
 test = [
     "certifi",
     "cython>=3.2.2",
-    "hdbscan>=0.8.39,<0.8.40",
+    "hdbscan>=0.8.39",
     "hypothesis>=6.0,<7",
     "ipython>=7.32.0",
     "nltk",

python/cuml/tests/test_hdbscan.py

@@ -1,6 +1,8 @@
 # SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION.
 # SPDX-License-Identifier: Apache-2.0
 #
+import importlib.metadata
+
 import cupy as cp
 import hdbscan
 import numpy as np
@@ -24,10 +26,13 @@
 from cuml.testing.datasets import make_pattern
 from cuml.testing.utils import array_equal
 
-if Version(sklearn.__version__) >= Version("1.8.0.dev0"):
-    pytest.skip(
-        "hdbscan requires sklearn < 1.8.0.dev0", allow_module_level=True
-    )
+if Version(sklearn.__version__) >= Version("1.8.0"):
+    if Version(importlib.metadata.version("hdbscan")) < Version("0.8.41"):
+        pytest.skip(
+            "hdbscan < 0.8.41 requires sklearn < 1.8.0",
+            allow_module_level=True,
+        )
+
 
 # Ignore FutureWarning from third-party hdbscan package calling
 # sklearn.utils.validation.check_array with deprecated 'force_all_finite'
@@ -802,70 +807,6 @@ def test_approximate_predict_moons(
         assert array_equal(cu_probs, sk_probs, unit_tol=0.05, total_tol=0.005)
 
 
-@pytest.mark.parametrize("nrows", [1000])
-@pytest.mark.parametrize("n_points_to_predict", [50])
-@pytest.mark.parametrize("min_samples", [5, 15])
-@pytest.mark.parametrize("cluster_selection_epsilon", [0.0, 0.5])
-@pytest.mark.parametrize("min_cluster_size", [50, 100])
-@pytest.mark.parametrize("allow_single_cluster", [True, False])
-@pytest.mark.parametrize("max_cluster_size", [0])
-@pytest.mark.parametrize("cluster_selection_method", ["eom", "leaf"])
-def test_approximate_predict_circles(
-    nrows,
-    n_points_to_predict,
-    min_samples,
-    cluster_selection_epsilon,
-    min_cluster_size,
-    allow_single_cluster,
-    max_cluster_size,
-    cluster_selection_method,
-):
-    X, y = datasets.make_circles(
-        n_samples=nrows + n_points_to_predict,
-        factor=0.8,
-        noise=0.05,
-        random_state=42,
-    )
-
-    X_train = X[:nrows]
-    X_test = X[nrows:]
-
-    cuml_agg = HDBSCAN(
-        allow_single_cluster=allow_single_cluster,
-        min_samples=min_samples,
-        max_cluster_size=max_cluster_size,
-        min_cluster_size=min_cluster_size,
-        cluster_selection_epsilon=cluster_selection_epsilon,
-        cluster_selection_method=cluster_selection_method,
-        prediction_data=True,
-    )
-
-    cuml_agg.fit(X_train)
-
-    sk_agg = hdbscan.HDBSCAN(
-        allow_single_cluster=allow_single_cluster,
-        approx_min_span_tree=False,
-        gen_min_span_tree=True,
-        min_samples=min_samples,
-        min_cluster_size=min_cluster_size,
-        cluster_selection_epsilon=cluster_selection_epsilon,
-        cluster_selection_method=cluster_selection_method,
-        algorithm="generic",
-        prediction_data=True,
-    )
-
-    sk_agg.fit(cp.asnumpy(X_train))
-
-    cu_labels, cu_probs = approximate_predict(cuml_agg, X_test)
-    sk_labels, sk_probs = hdbscan.approximate_predict(sk_agg, X_test)
-
-    sk_unique = np.unique(sk_labels)
-    cu_unique = np.unique(cu_labels)
-    if len(sk_unique) == len(cu_unique):
-        assert adjusted_rand_score(cu_labels, sk_labels) >= 0.99
-        assert array_equal(cu_probs, sk_probs, unit_tol=0.05, total_tol=0.005)
-
-
 @pytest.mark.parametrize("n_points_to_predict", [200])
 @pytest.mark.parametrize("min_samples", [15])
 @pytest.mark.parametrize("cluster_selection_epsilon", [0.5])