Metric bootstrapper

CHANGELOG.md

@@ -26,6 +26,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Added multilabel support to `ROC` metric ([#114](https://github.com/PyTorchLightning/metrics/pull/114))
 
+
+- Added `BootStrapper` to easely calculate confidence intervals for metrics ([#101](https://github.com/PyTorchLightning/metrics/pull/101))
+
+
 ### Changed
 
 - Changed `ExplainedVariance` from storing all preds/targets to tracking 5 statistics ([#68](https://github.com/PyTorchLightning/metrics/pull/68))

docs/source/references/modules.rst

@@ -269,4 +269,14 @@ R2Score
 ~~~~~~~
 
 .. autoclass:: torchmetrics.R2Score
-    :noindex:
+    :noindex:
+
+********
+Wrappers
+********
+
+Modular wrapper metrics are not metrics in themself, but instead take a metric and alter the internal logic 
+of the base metric.
+
+.. autoclass:: torchmetrics.BootStrapper
+    :noindex:

tests/classification/test_roc.py

@@ -74,22 +74,38 @@ def _sk_roc_multidim_multiclass_prob(preds, target, num_classes=1):
 def _sk_roc_multilabel_prob(preds, target, num_classes=1):
     sk_preds = preds.numpy()
     sk_target = target.numpy()
-    return _sk_roc_curve(y_true=sk_target, probas_pred=sk_preds, num_classes=num_classes, multilabel=True)
+    return _sk_roc_curve(
+        y_true=sk_target,
+        probas_pred=sk_preds,
+        num_classes=num_classes,
+        multilabel=True
+    )
 
 
 def _sk_roc_multilabel_multidim_prob(preds, target, num_classes=1):
     sk_preds = preds.transpose(0, 1).reshape(num_classes, -1).transpose(0, 1).numpy()
     sk_target = target.transpose(0, 1).reshape(num_classes, -1).transpose(0, 1).numpy()
-    return _sk_roc_curve(y_true=sk_target, probas_pred=sk_preds, num_classes=num_classes, multilabel=True)
+    return _sk_roc_curve(
+        y_true=sk_target,
+        probas_pred=sk_preds,
+        num_classes=num_classes,
+        multilabel=True
+    )
 
 
 @pytest.mark.parametrize(
-    "preds, target, sk_metric, num_classes",
-    [(_input_binary_prob.preds, _input_binary_prob.target, _sk_roc_binary_prob, 1),
-     (_input_mcls_prob.preds, _input_mcls_prob.target, _sk_roc_multiclass_prob, NUM_CLASSES),
-     (_input_mdmc_prob.preds, _input_mdmc_prob.target, _sk_roc_multidim_multiclass_prob, NUM_CLASSES),
-     (_input_mlb_prob.preds, _input_mlb_prob.target, _sk_roc_multilabel_prob, NUM_CLASSES),
-     (_input_mlmd_prob.preds, _input_mlmd_prob.target, _sk_roc_multilabel_multidim_prob, NUM_CLASSES)]
+    "preds, target, sk_metric, num_classes", [
+        (_input_binary_prob.preds, _input_binary_prob.target, _sk_roc_binary_prob, 1),
+        (_input_mcls_prob.preds, _input_mcls_prob.target, _sk_roc_multiclass_prob, NUM_CLASSES),
+        (_input_mdmc_prob.preds, _input_mdmc_prob.target, _sk_roc_multidim_multiclass_prob, NUM_CLASSES),
+        (_input_multilabel_prob.preds, _input_multilabel_prob.target, _sk_roc_multilabel_prob, NUM_CLASSES),
+        (
+            _input_multilabel_multidim_prob.preds,
+            _input_multilabel_multidim_prob.target,
+            _sk_roc_multilabel_multidim_prob,
+            NUM_CLASSES
+        )
+    ]
 )
 class TestROC(MetricTester):
 

tests/wrappers/test_bootstrapping.py

@@ -0,0 +1,113 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import numpy as np
+import pytest
+import torch
+from torch import Tensor
+from sklearn.metrics import precision_score, recall_score
+
+from torchmetrics.classification import Precision, Recall
+from torchmetrics.utilities import apply_to_collection
+from torchmetrics.utilities.imports import _TORCH_GREATER_EQUAL_1_7
+from torchmetrics.wrappers.bootstrapping import BootStrapper, _bootstrap_sampler
+
+_preds = torch.randint(10, (10, 32))
+_target = torch.randint(10, (10, 32))
+
+
+class TestBootStrapper(BootStrapper):
+    """ For testing purpose, we subclass the bootstrapper class so we can get the exact permutation
+        the class is creating
+    """
+    def update(self, *args) -> None:
+        self.out = []
+        for idx in range(self.num_bootstraps):
+            size = len(args[0])
+            sample_idx = _bootstrap_sampler(size, sampling_strategy=self.sampling_strategy)
+            new_args = apply_to_collection(args, Tensor, torch.index_select, dim=0, index=sample_idx)
+            self.metrics[idx].update(*new_args)
+            self.out.append(new_args)
+
+
+@pytest.mark.parametrize("sampling_strategy", ['poisson', 'multinomial'])
+def test_bootstrap_sampler(sampling_strategy):
+    """ make sure that the bootstrap sampler works as intended """
+    old_samples = torch.randn(10, 2)
+
+    # make sure that the new samples are only made up of old samples
+    idx = _bootstrap_sampler(10, sampling_strategy=sampling_strategy)
+    new_samples = old_samples[idx]
+    for ns in new_samples:
+        assert ns in old_samples
+
+    # make sure some samples are also sampled twice
+    found_one = False
+    for os in old_samples:
+        cond = os == new_samples
+        if cond.sum() > 2:
+            found_one = True
+            break
+
+    assert found_one, "resampling did not work because no samples were sampled twice"
+
+    # make sure some samples are never sampled
+    found_zero = False
+    for os in old_samples:
+        cond = os != new_samples
+        if cond.sum() > 0:
+            found_zero = True
+            break
+
+    assert found_zero, "resampling did not work because all samples were atleast sampled once"
+
+
+@pytest.mark.parametrize("sampling_strategy", ['poisson', 'multinomial'])
+@pytest.mark.parametrize(
+    "metric, sk_metric", [[Precision(average='micro'), precision_score], [Recall(average='micro'), recall_score]]
+)
+def test_bootstrap(sampling_strategy, metric, sk_metric):
+    """ Test that the different bootstraps gets updated as we expected and that the compute method works """
+    _kwargs = {'base_metric': metric, 'mean': True, 'std': True, 'raw': True, 'sampling_strategy': sampling_strategy}
+    if _TORCH_GREATER_EQUAL_1_7:
+        bootstrapper = TestBootStrapper(**_kwargs, quantile=torch.tensor([0.05, 0.95]))
+    else:
+        bootstrapper = TestBootStrapper(**_kwargs)
+
+    collected_preds = [[] for _ in range(10)]
+    collected_target = [[] for _ in range(10)]
+    for p, t in zip(_preds, _target):
+        bootstrapper.update(p, t)
+
+        for i, o in enumerate(bootstrapper.out):
+
+            collected_preds[i].append(o[0])
+            collected_target[i].append(o[1])
+
+    collected_preds = [torch.cat(cp) for cp in collected_preds]
+    collected_target = [torch.cat(ct) for ct in collected_target]
+
+    sk_scores = [sk_metric(ct, cp, average='micro') for ct, cp in zip(collected_target, collected_preds)]
+
+    output = bootstrapper.compute()
+    # quantile only avaible for pytorch v1.7 and forward
+    if _TORCH_GREATER_EQUAL_1_7:
+        pl_mean, pl_std, pl_quantile, pl_raw = output
+        assert np.allclose(pl_quantile[0], np.quantile(sk_scores, 0.05))
+        assert np.allclose(pl_quantile[1], np.quantile(sk_scores, 0.95))
+    else:
+        pl_mean, pl_std, pl_raw = output
+
+    assert np.allclose(pl_mean, np.mean(sk_scores))
+    assert np.allclose(pl_std, np.std(sk_scores, ddof=1))
+    assert np.allclose(pl_raw, sk_scores)

torchmetrics/__init__.py

@@ -49,3 +49,4 @@
     R2Score,
 )
 from torchmetrics.retrieval import RetrievalMAP  # noqa: F401 E402
+from torchmetrics.wrappers import BootStrapper  # noqa: F401 E402

torchmetrics/utilities/imports.py

@@ -1,7 +1,47 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Import utilities"""
+import importlib
+import operator
 from distutils.version import LooseVersion
 
-import torch
+from pkg_resources import DistributionNotFound
 
-_TORCH_LOWER_1_4 = LooseVersion(torch.__version__) < LooseVersion("1.4.0")
-_TORCH_LOWER_1_5 = LooseVersion(torch.__version__) < LooseVersion("1.5.0")
-_TORCH_LOWER_1_6 = LooseVersion(torch.__version__) < LooseVersion("1.6.0")
+
+def _compare_version(package: str, op, version) -> bool:
+    """
+    Compare package version with some requirements
+
+    >>> _compare_version("torch", operator.ge, "0.1")
+    True
+    """
+    try:
+        pkg = importlib.import_module(package)
+    except (ModuleNotFoundError, DistributionNotFound):
+        return False
+    try:
+        pkg_version = LooseVersion(pkg.__version__)
+    except AttributeError:
+        return False
+    if not (hasattr(pkg_version, "vstring") and hasattr(pkg_version, "version")):
+        # this is mock by sphinx, so it shall return True ro generate all summaries
+        return True
+    return op(pkg_version, LooseVersion(version))
+
+
+_TORCH_LOWER_1_4 = _compare_version("torch", operator.lt, "1.4.0")
+_TORCH_LOWER_1_5 = _compare_version("torch", operator.lt, "1.5.0")
+_TORCH_LOWER_1_6 = _compare_version("torch", operator.lt, "1.6.0")
+_TORCH_GREATER_EQUAL_1_6 = _compare_version("torch", operator.ge, "1.6.0")
+_TORCH_GREATER_EQUAL_1_7 = _compare_version("torch", operator.ge, "1.7.0")

torchmetrics/wrappers/__init__.py

@@ -0,0 +1,14 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from torchmetrics.wrappers.bootstrapping import BootStrapper  # noqa: F401