Merge branch 'master' into iou-compute-in-cuda

Author: mergify[bot]

CHANGELOG.md

@@ -23,6 +23,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added `top_k` argument to `RetrievalMRR` in retrieval package ([#1961](https://github.com/Lightning-AI/torchmetrics/pull/1961))
 
 
+- Added warning to `MeanAveragePrecision` if too many detections are observed ([#1978](https://github.com/Lightning-AI/torchmetrics/pull/1978))
+
+
 ### Changed
 
 -

src/torchmetrics/detection/mean_ap.py

@@ -24,6 +24,7 @@
 
 from torchmetrics.detection.helpers import _fix_empty_tensors, _input_validator
 from torchmetrics.metric import Metric
+from torchmetrics.utilities import rank_zero_warn
 from torchmetrics.utilities.imports import (
     _MATPLOTLIB_AVAILABLE,
     _PYCOCOTOOLS_AVAILABLE,
@@ -239,6 +240,8 @@ class MeanAveragePrecision(Metric):
     groundtruth_crowds: List[Tensor]
     groundtruth_area: List[Tensor]
 
+    warn_on_many_detections: bool = True
+
     def __init__(
         self,
         box_format: Literal["xyxy", "xywh", "cxcywh"] = "xyxy",
@@ -329,7 +332,7 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]
         _input_validator(preds, target, iou_type=self.iou_type)
 
         for item in preds:
-            detections = self._get_safe_item_values(item)
+            detections = self._get_safe_item_values(item, warn=self.warn_on_many_detections)
 
             self.detections.append(detections)
             self.detection_labels.append(item["labels"])
@@ -542,11 +545,12 @@ def tm_to_coco(self, name: str = "tm_map_input") -> None:
         with open(f"{name}_target.json", "w") as f:
             f.write(target_json)
 
-    def _get_safe_item_values(self, item: Dict[str, Any]) -> Union[Tensor, Tuple]:
+    def _get_safe_item_values(self, item: Dict[str, Any], warn: bool = False) -> Union[Tensor, Tuple]:
         """Convert and return the boxes or masks from the item depending on the iou_type.
 
         Args:
             item: input dictionary containing the boxes or masks
+            warn: whether to warn if the number of boxes or masks exceeds the max_detection_thresholds
 
         Returns:
             boxes or masks depending on the iou_type
@@ -556,12 +560,16 @@ def _get_safe_item_values(self, item: Dict[str, Any]) -> Union[Tensor, Tuple]:
             boxes = _fix_empty_tensors(item["boxes"])
             if boxes.numel() > 0:
                 boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xywh")
+            if warn and len(boxes) > self.max_detection_thresholds[-1]:
+                _warning_on_too_many_detections(self.max_detection_thresholds[-1])
             return boxes
         if self.iou_type == "segm":
             masks = []
             for i in item["masks"].cpu().numpy():
                 rle = mask_utils.encode(np.asfortranarray(i))
                 masks.append((tuple(rle["size"]), rle["counts"]))
+            if warn and len(masks) > self.max_detection_thresholds[-1]:
+                _warning_on_too_many_detections(self.max_detection_thresholds[-1])
             return tuple(masks)
         raise Exception(f"IOU type {self.iou_type} is not supported")
 
@@ -747,3 +755,13 @@ def _gather_tuple_list(list_to_gather: List[Tuple], process_group: Optional[Any]
         dist.all_gather_object(list_gathered, list_to_gather, group=process_group)
 
         return [list_gathered[rank][idx] for idx in range(len(list_gathered[0])) for rank in range(world_size)]
+
+
+def _warning_on_too_many_detections(limit: int) -> None:
+    rank_zero_warn(
+        f"Encountered more than {limit} detections in a single image. This means that certain detections with the"
+        " lowest scores will be ignored, that may have an undesirable impact on performance. Please consider adjusting"
+        " the `max_detection_threshold` to suit your use case. To disable this warning, set attribute class"
+        " `warn_on_many_detections=False`, after initializing the metric.",
+        UserWarning,
+    )

src/torchmetrics/nominal/cramers.py

@@ -39,11 +39,20 @@ class CramersV(Metric):
         \chi^2 = \sum_{i,j} \ frac{\left(n_{ij} - \frac{n_{i.} n_{.j}}{n}\right)^2}{\frac{n_{i.} n_{.j}}{n}}
 
     where :math:`n_{ij}` denotes the number of times the values :math:`(A_i, B_j)` are observed with :math:`A_i, B_j`
-    represent frequencies of values in ``preds`` and ``target``, respectively.
+    represent frequencies of values in ``preds`` and ``target``, respectively. Cramer's V is a symmetric coefficient,
+    i.e. :math:`V(preds, target) = V(target, preds)`, so order of input arguments does not matter. The output values
+    lies in [0, 1] with 1 meaning the perfect association.
 
-    Cramer's V is a symmetric coefficient, i.e. :math:`V(preds, target) = V(target, preds)`.
+    As input to ``forward`` and ``update`` the metric accepts the following input:
 
-    The output values lies in [0, 1] with 1 meaning the perfect association.
+    - ``preds`` (:class:`~torch.Tensor`): Either 1D or 2D tensor of categorical (nominal) data from the first data
+      series with shape ``(batch_size,)`` or ``(batch_size, num_classes)``, respectively.
+    - ``target`` (:class:`~torch.Tensor`): Either 1D or 2D tensor of categorical (nominal) data from the second data
+      series with shape ``(batch_size,)`` or ``(batch_size, num_classes)``, respectively.
+
+    As output of ``forward`` and ``compute`` the metric returns the following output:
+
+    - ``cramers_v`` (:class:`~torch.Tensor`): Scalar tensor containing the Cramer's V statistic.
 
     Args:
         num_classes: Integer specifing the number of classes
@@ -52,16 +61,14 @@ class CramersV(Metric):
         nan_replace_value: Value to replace ``NaN``s when ``nan_strategy = 'replace'``
         kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info.
 
-    Returns:
-        Cramer's V statistic
-
     Raises:
         ValueError:
             If `nan_strategy` is not one of `'replace'` and `'drop'`
         ValueError:
             If `nan_strategy` is equal to `'replace'` and `nan_replace_value` is not an `int` or `float`
 
-    Example:
+    Example::
+
         >>> from torchmetrics.nominal import CramersV
         >>> _ = torch.manual_seed(42)
         >>> preds = torch.randint(0, 4, (100,))
@@ -98,20 +105,7 @@ def __init__(
         self.add_state("confmat", torch.zeros(num_classes, num_classes), dist_reduce_fx="sum")
 
     def update(self, preds: Tensor, target: Tensor) -> None:
-        """Update state with predictions and targets.
-
-        Args:
-            preds: 1D or 2D tensor of categorical (nominal) data
-
-                - 1D shape: (batch_size,)
-                - 2D shape: (batch_size, num_classes)
-
-            target: 1D or 2D tensor of categorical (nominal) data
-
-                - 1D shape: (batch_size,)
-                - 2D shape: (batch_size, num_classes)
-
-        """
+        """Update state with predictions and targets."""
         confmat = _cramers_v_update(preds, target, self.num_classes, self.nan_strategy, self.nan_replace_value)
         self.confmat += confmat
 

src/torchmetrics/nominal/pearson.py

@@ -43,30 +43,36 @@ class PearsonsContingencyCoefficient(Metric):
     .. math::
         \chi^2 = \sum_{i,j} \ frac{\left(n_{ij} - \frac{n_{i.} n_{.j}}{n}\right)^2}{\frac{n_{i.} n_{.j}}{n}}
 
-    where :math:`n_{ij}` denotes the number of times the values :math:`(A_i, B_j)` are observed
-    with :math:`A_i, B_j` represent frequencies of values in ``preds`` and ``target``, respectively.
+    where :math:`n_{ij}` denotes the number of times the values :math:`(A_i, B_j)` are observed with :math:`A_i, B_j`
+    represent frequencies of values in ``preds`` and ``target``, respectively. Pearson's Contingency Coefficient is a
+    symmetric coefficient, i.e. :math:`Pearson(preds, target) = Pearson(target, preds)`, so order of input arguments
+    does not matter. The output values lies in [0, 1] with 1 meaning the perfect association.
 
-    Pearson's Contingency Coefficient is a symmetric coefficient, i.e.
-    :math:`Pearson(preds, target) = Pearson(target, preds)`.
+    As input to ``forward`` and ``update`` the metric accepts the following input:
 
-    The output values lies in [0, 1] with 1 meaning the perfect association.
+    - ``preds`` (:class:`~torch.Tensor`): Either 1D or 2D tensor of categorical (nominal) data from the first data
+      series with shape ``(batch_size,)`` or ``(batch_size, num_classes)``, respectively.
+    - ``target`` (:class:`~torch.Tensor`): Either 1D or 2D tensor of categorical (nominal) data from the second data
+      series with shape ``(batch_size,)`` or ``(batch_size, num_classes)``, respectively.
+
+    As output of ``forward`` and ``compute`` the metric returns the following output:
+
+    - ``pearsons_cc`` (:class:`~torch.Tensor`): Scalar tensor containing the Pearsons Contingency Coefficient statistic.
 
     Args:
         num_classes: Integer specifing the number of classes
         nan_strategy: Indication of whether to replace or drop ``NaN`` values
         nan_replace_value: Value to replace ``NaN``s when ``nan_strategy = 'replace'``
         kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info.
 
-    Returns:
-        Pearson's Contingency Coefficient statistic
-
     Raises:
         ValueError:
             If `nan_strategy` is not one of `'replace'` and `'drop'`
         ValueError:
             If `nan_strategy` is equal to `'replace'` and `nan_replace_value` is not an `int` or `float`
 
-    Example:
+    Example::
+
         >>> from torchmetrics.nominal import PearsonsContingencyCoefficient
         >>> _ = torch.manual_seed(42)
         >>> preds = torch.randint(0, 4, (100,))
@@ -101,20 +107,7 @@ def __init__(
         self.add_state("confmat", torch.zeros(num_classes, num_classes), dist_reduce_fx="sum")
 
     def update(self, preds: Tensor, target: Tensor) -> None:
-        """Update state with predictions and targets.
-
-        Args:
-            preds: 1D or 2D tensor of categorical (nominal) data:
-
-                - 1D shape: (batch_size,)
-                - 2D shape: (batch_size, num_classes)
-
-            target: 1D or 2D tensor of categorical (nominal) data:
-
-                - 1D shape: (batch_size,)
-                - 2D shape: (batch_size, num_classes)
-
-        """
+        """Update state with predictions and targets."""
         confmat = _pearsons_contingency_coefficient_update(
             preds, target, self.num_classes, self.nan_strategy, self.nan_replace_value
         )

src/torchmetrics/nominal/theils_u.py

@@ -34,23 +34,31 @@ class TheilsU(Metric):
         U(X|Y) = \frac{H(X) - H(X|Y)}{H(X)}
 
     where :math:`H(X)` is entropy of variable :math:`X` while :math:`H(X|Y)` is the conditional entropy of :math:`X`
-    given :math:`Y`. It is also know as the Uncertainty Coefficient.
+    given :math:`Y`. It is also know as the Uncertainty Coefficient. Theils's U is an asymmetric coefficient, i.e.
+    :math:`TheilsU(preds, target) \neq TheilsU(target, preds)`, so the order of the inputs matters. The output values
+    lies in [0, 1], where a 0 means y has no information about x while value 1 means y has complete information about x.
 
-    Theils's U is an asymmetric coefficient, i.e. :math:`TheilsU(preds, target) \neq TheilsU(target, preds)`.
+    As input to ``forward`` and ``update`` the metric accepts the following input:
 
-    The output values lies in [0, 1]. 0 means y has no information about x while value 1 means y has complete
-    information about x.
+    - ``preds`` (:class:`~torch.Tensor`): Either 1D or 2D tensor of categorical (nominal) data from the first data
+      series (called X in the above definition) with shape ``(batch_size,)`` or ``(batch_size, num_classes)``,
+      respectively.
+    - ``target`` (:class:`~torch.Tensor`): Either 1D or 2D tensor of categorical (nominal) data from the second data
+      series (called Y in the above definition) with shape ``(batch_size,)`` or ``(batch_size, num_classes)``,
+      respectively.
+
+    As output of ``forward`` and ``compute`` the metric returns the following output:
+
+    - ``theils_u`` (:class:`~torch.Tensor`): Scalar tensor containing the Theil's U statistic.
 
     Args:
         num_classes: Integer specifing the number of classes
         nan_strategy: Indication of whether to replace or drop ``NaN`` values
         nan_replace_value: Value to replace ``NaN``s when ``nan_strategy = 'replace'``
         kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info.
 
-    Returns:
-        Theil's U Statistic: Tensor
+    Example::
 
-    Example:
         >>> from torchmetrics.nominal import TheilsU
         >>> _ = torch.manual_seed(42)
         >>> preds = torch.randint(10, (10,))
@@ -85,17 +93,7 @@ def __init__(
         self.add_state("confmat", torch.zeros(num_classes, num_classes), dist_reduce_fx="sum")
 
     def update(self, preds: Tensor, target: Tensor) -> None:
-        """Update state with predictions and targets.
-
-        Args:
-            preds: 1D or 2D tensor of categorical (nominal) data
-            - 1D shape: (batch_size,)
-            - 2D shape: (batch_size, num_classes)
-            target: 1D or 2D tensor of categorical (nominal) data
-            - 1D shape: (batch_size,)
-            - 2D shape: (batch_size, num_classes)
-
-        """
+        """Update state with predictions and targets."""
         confmat = _theils_u_update(preds, target, self.num_classes, self.nan_strategy, self.nan_replace_value)
         self.confmat += confmat
 

src/torchmetrics/nominal/tschuprows.py

@@ -38,12 +38,21 @@ class TschuprowsT(Metric):
     .. math::
         \chi^2 = \sum_{i,j} \ frac{\left(n_{ij} - \frac{n_{i.} n_{.j}}{n}\right)^2}{\frac{n_{i.} n_{.j}}{n}}
 
-    where :math:`n_{ij}` denotes the number of times the values :math:`(A_i, B_j)` are observed
-    with :math:`A_i, B_j` represent frequencies of values in ``preds`` and ``target``, respectively.
+    where :math:`n_{ij}` denotes the number of times the values :math:`(A_i, B_j)` are observed with :math:`A_i, B_j`
+    represent frequencies of values in ``preds`` and ``target``, respectively. Tschuprow's T is a symmetric coefficient,
+    i.e. :math:`T(preds, target) = T(target, preds)`, so order of input arguments does not matter. The output values
+    lies in [0, 1] with 1 meaning the perfect association.
 
-    Tschuprow's T is a symmetric coefficient, i.e. :math:`T(preds, target) = T(target, preds)`.
+    As input to ``forward`` and ``update`` the metric accepts the following input:
 
-    The output values lies in [0, 1] with 1 meaning the perfect association.
+    - ``preds`` (:class:`~torch.Tensor`): Either 1D or 2D tensor of categorical (nominal) data from the first data
+      series with shape ``(batch_size,)`` or ``(batch_size, num_classes)``, respectively.
+    - ``target`` (:class:`~torch.Tensor`): Either 1D or 2D tensor of categorical (nominal) data from the second data
+      series with shape ``(batch_size,)`` or ``(batch_size, num_classes)``, respectively.
+
+    As output of ``forward`` and ``compute`` the metric returns the following output:
+
+    - ``tschuprows_t`` (:class:`~torch.Tensor`): Scalar tensor containing the Tschuprow's T statistic.
 
     Args:
         num_classes: Integer specifing the number of classes
@@ -52,16 +61,14 @@ class TschuprowsT(Metric):
         nan_replace_value: Value to replace ``NaN``s when ``nan_strategy = 'replace'``
         kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info.
 
-    Returns:
-        Tschuprow's T statistic
-
     Raises:
         ValueError:
             If `nan_strategy` is not one of `'replace'` and `'drop'`
         ValueError:
             If `nan_strategy` is equal to `'replace'` and `nan_replace_value` is not an `int` or `float`
 
-    Example:
+    Example::
+
         >>> from torchmetrics.nominal import TschuprowsT
         >>> _ = torch.manual_seed(42)
         >>> preds = torch.randint(0, 4, (100,))
@@ -98,20 +105,7 @@ def __init__(
         self.add_state("confmat", torch.zeros(num_classes, num_classes), dist_reduce_fx="sum")
 
     def update(self, preds: Tensor, target: Tensor) -> None:
-        """Update state with predictions and targets.
-
-        Args:
-            preds: 1D or 2D tensor of categorical (nominal) data:
-
-                - 1D shape: (batch_size,)
-                - 2D shape: (batch_size, num_classes)
-
-            target: 1D or 2D tensor of categorical (nominal) data:
-
-                - 1D shape: (batch_size,)
-                - 2D shape: (batch_size, num_classes)
-
-        """
+        """Update state with predictions and targets."""
         confmat = _tschuprows_t_update(preds, target, self.num_classes, self.nan_strategy, self.nan_replace_value)
         self.confmat += confmat
 

tests/unittests/detection/test_map.py

@@ -628,19 +628,19 @@ def test_error_on_wrong_input():
         )
 
 
-def _generate_random_segm_input(device):
+def _generate_random_segm_input(device, batch_size=2, num_preds_size=10, num_gt_size=10, random_size=True):
     """Generate random inputs for mAP when iou_type=segm."""
     preds = []
     targets = []
-    for _ in range(2):
+    for _ in range(batch_size):
         result = {}
-        num_preds = torch.randint(0, 10, (1,)).item()
+        num_preds = torch.randint(0, num_preds_size, (1,)).item() if random_size else num_preds_size
         result["scores"] = torch.rand((num_preds,), device=device)
         result["labels"] = torch.randint(0, 10, (num_preds,), device=device)
         result["masks"] = torch.randint(0, 2, (num_preds, 10, 10), device=device).bool()
         preds.append(result)
         gt = {}
-        num_gt = torch.randint(0, 10, (1,)).item()
+        num_gt = torch.randint(0, num_gt_size, (1,)).item() if random_size else num_gt_size
         gt["labels"] = torch.randint(0, 10, (num_gt,), device=device)
         gt["masks"] = torch.randint(0, 2, (num_gt, 10, 10), device=device).bool()
         targets.append(gt)
@@ -690,3 +690,23 @@ def test_for_box_format(box_format, iou_val_expected, map_val_expected):
     result = metric.compute()
     assert result["map"].item() == map_val_expected
     assert round(float(metric.coco_eval.ious[(0, 0)]), 3) == iou_val_expected
+
+
+@pytest.mark.parametrize("iou_type", ["bbox", "segm"])
+def test_warning_on_many_detections(iou_type):
+    """Test that a warning is raised when there are many detections."""
+    if iou_type == "bbox":
+        preds = [
+            {
+                "boxes": torch.tensor([[0.5, 0.5, 1, 1]]).repeat(101, 1),
+                "scores": torch.tensor([1.0]).repeat(101),
+                "labels": torch.tensor([0]).repeat(101),
+            }
+        ]
+        targets = [{"boxes": torch.tensor([[0, 0, 1, 1]]), "labels": torch.tensor([0])}]
+    else:
+        preds, targets = _generate_random_segm_input("cpu", 1, 101, 10, False)
+
+    metric = MeanAveragePrecision(iou_type=iou_type)
+    with pytest.warns(UserWarning, match="Encountered more than 100 detections in a single image.*"):
+        metric.update(preds, targets)