From 3c4388e7cd9d2675b64da51e66e1676bdf299701 Mon Sep 17 00:00:00 2001 From: ChristophReich1996 Date: Wed, 14 Feb 2024 13:55:02 +0100 Subject: [PATCH 01/20] Add support for SQ & RQ as well as per-class metrics --- .../detection/panoptic_qualities.py | 77 ++++++++++++++++--- .../detection/_panoptic_quality_common.py | 20 +++-- .../detection/panoptic_qualities.py | 77 +++++++++++++++---- 3 files changed, 142 insertions(+), 32 deletions(-) diff --git a/src/torchmetrics/detection/panoptic_qualities.py b/src/torchmetrics/detection/panoptic_qualities.py index 5ad84ddd974..1a04bb2ebb8 100644 --- a/src/torchmetrics/detection/panoptic_qualities.py +++ b/src/torchmetrics/detection/panoptic_qualities.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Collection, Optional, Sequence, Union +from typing import Any, Collection, Dict, Optional, Sequence, Union import torch from torch import Tensor @@ -55,6 +55,10 @@ class PanopticQuality(Metric): allow_unknown_preds_category: Boolean flag to specify if unknown categories in the predictions are to be ignored in the metric computation or raise an exception when found. + return_sq_and_rq: + Boolean flag to specify if Segmentation Quality and Recognition Quality should be also returnd. + per_class: + Boolean flag to specify if the per-class values should be returned or the class average. Raises: @@ -80,6 +84,40 @@ class PanopticQuality(Metric): >>> panoptic_quality(preds, target) tensor(0.5463, dtype=torch.float64) + You can also return the segmentation and recongition quality alognside the PQ + >>> from torch import tensor + >>> from torchmetrics.detection import PanopticQuality + >>> preds = tensor([[[[6, 0], [0, 0], [6, 0], [6, 0]], + ... [[0, 0], [0, 0], [6, 0], [0, 1]], + ... [[0, 0], [0, 0], [6, 0], [0, 1]], + ... [[0, 0], [7, 0], [6, 0], [1, 0]], + ... [[0, 0], [7, 0], [7, 0], [7, 0]]]]) + >>> target = tensor([[[[6, 0], [0, 1], [6, 0], [0, 1]], + ... [[0, 1], [0, 1], [6, 0], [0, 1]], + ... [[0, 1], [0, 1], [6, 0], [1, 0]], + ... [[0, 1], [7, 0], [1, 0], [1, 0]], + ... [[0, 1], [7, 0], [7, 0], [7, 0]]]]) + >>> panoptic_quality = PanopticQuality(things = {0, 1}, stuffs = {6, 7}, return_sq_and_rq=True) + >>> panoptic_quality(preds, target) + {'pq': tensor(0.5463, dtype=torch.float64), 'sq': tensor(0.5463, dtype=torch.float64), 'rq': tensor(0.5463, dtype=torch.float64)} + + You can also specify to return the per-class metrics + >>> from torch import tensor + >>> from torchmetrics.detection import PanopticQuality + >>> preds = tensor([[[[6, 0], [0, 0], [6, 0], [6, 0]], + ... [[0, 0], [0, 0], [6, 0], [0, 1]], + ... [[0, 0], [0, 0], [6, 0], [0, 1]], + ... [[0, 0], [7, 0], [6, 0], [1, 0]], + ... [[0, 0], [7, 0], [7, 0], [7, 0]]]]) + >>> target = tensor([[[[6, 0], [0, 1], [6, 0], [0, 1]], + ... [[0, 1], [0, 1], [6, 0], [0, 1]], + ... [[0, 1], [0, 1], [6, 0], [1, 0]], + ... [[0, 1], [7, 0], [1, 0], [1, 0]], + ... [[0, 1], [7, 0], [7, 0], [7, 0]]]]) + >>> panoptic_quality = PanopticQuality(things = {0, 1}, stuffs = {6, 7}, per_class=True) + >>> panoptic_quality(preds, target) + tensor([0.5185, 0.0000, 0.6667, 1.0000], dtype=torch.float64) + """ is_differentiable: bool = False higher_is_better: bool = True @@ -97,6 +135,8 @@ def __init__( things: Collection[int], stuffs: Collection[int], allow_unknown_preds_category: bool = False, + return_sq_and_rq: bool = False, + per_class: bool = False, **kwargs: Any, ) -> None: super().__init__(**kwargs) @@ -107,6 +147,8 @@ def __init__( self.void_color = _get_void_color(things, stuffs) self.cat_id_to_continuous_id = _get_category_id_to_continuous_id(things, stuffs) self.allow_unknown_preds_category = allow_unknown_preds_category + self.return_sq_and_rq = return_sq_and_rq + self.per_class = per_class # per category intermediate metrics num_categories = len(things) + len(stuffs) @@ -151,12 +193,22 @@ def update(self, preds: Tensor, target: Tensor) -> None: self.false_positives += false_positives self.false_negatives += false_negatives - def compute(self) -> Tensor: + def compute(self) -> Union[Tensor, Dict[str, Tensor]]: """Compute panoptic quality based on inputs passed in to ``update`` previously.""" - return _panoptic_quality_compute(self.iou_sum, self.true_positives, self.false_positives, self.false_negatives) + pq, sq, rq, pq_avg, sq_avg, rq_avg = _panoptic_quality_compute(self.iou_sum, self.true_positives, + self.false_positives, + self.false_negatives) + if self.per_class: + if self.return_sq_and_rq: + return {"pq": pq, "sq": sq, "rq": rq} + else: + return pq + if self.return_sq_and_rq: + return {"pq": pq_avg, "sq": sq_avg, "rq": rq_avg} + return pq_avg def plot( - self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None + self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None ) -> _PLOT_OUT_TYPE: """Plot a single or multiple values from the metric. @@ -271,11 +323,11 @@ class ModifiedPanopticQuality(Metric): false_negatives: Tensor def __init__( - self, - things: Collection[int], - stuffs: Collection[int], - allow_unknown_preds_category: bool = False, - **kwargs: Any, + self, + things: Collection[int], + stuffs: Collection[int], + allow_unknown_preds_category: bool = False, + **kwargs: Any, ) -> None: super().__init__(**kwargs) @@ -335,10 +387,13 @@ def update(self, preds: Tensor, target: Tensor) -> None: def compute(self) -> Tensor: """Compute panoptic quality based on inputs passed in to ``update`` previously.""" - return _panoptic_quality_compute(self.iou_sum, self.true_positives, self.false_positives, self.false_negatives) + _, _, _, pq_avg, _, _ = _panoptic_quality_compute(self.iou_sum, self.true_positives, + self.false_positives, + self.false_negatives) + return pq_avg def plot( - self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None + self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None ) -> _PLOT_OUT_TYPE: """Plot a single or multiple values from the metric. diff --git a/src/torchmetrics/functional/detection/_panoptic_quality_common.py b/src/torchmetrics/functional/detection/_panoptic_quality_common.py index 9361039a7c4..8cbc06da807 100644 --- a/src/torchmetrics/functional/detection/_panoptic_quality_common.py +++ b/src/torchmetrics/functional/detection/_panoptic_quality_common.py @@ -449,7 +449,7 @@ def _panoptic_quality_compute( true_positives: Tensor, false_positives: Tensor, false_negatives: Tensor, -) -> Tensor: +) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]: """Compute the final panoptic quality from interim values. Args: @@ -459,11 +459,17 @@ def _panoptic_quality_compute( false_negatives: the FN value from the update step Returns: - Panoptic quality as a tensor containing a single scalar. + A tuple containing the per-class panoptic, segmentation and recognition quality followed by the averages """ - # per category calculation - denominator = (true_positives + 0.5 * false_positives + 0.5 * false_negatives).double() - panoptic_quality = torch.where(denominator > 0.0, iou_sum / denominator, 0.0) - # Reduce across categories. TODO: is it useful to have the option of returning per class metrics? - return torch.mean(panoptic_quality[denominator > 0]) + # compute segmentation and recognition quality (per-class) + sq: Tensor = torch.where(true_positives > 0.0, iou_sum / true_positives, 0.0) + denominator: Tensor = (true_positives + 0.5 * false_positives + 0.5 * false_negatives) + rq: Tensor = torch.where(denominator > 0.0, true_positives / denominator, 0.0) + # compute per-class panoptic quality + pq: Tensor = sq * rq + # compute averages + pq_avg: Tensor = torch.mean(pq[denominator > 0]) + sq_avg: Tensor = torch.mean(pq[denominator > 0]) + rq_avg: Tensor = torch.mean(pq[denominator > 0]) + return pq, sq, rq, pq_avg, sq_avg, rq_avg diff --git a/src/torchmetrics/functional/detection/panoptic_qualities.py b/src/torchmetrics/functional/detection/panoptic_qualities.py index be34439f883..640c8016e70 100644 --- a/src/torchmetrics/functional/detection/panoptic_qualities.py +++ b/src/torchmetrics/functional/detection/panoptic_qualities.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Collection +from typing import Collection, Dict, Union from torch import Tensor @@ -27,12 +27,14 @@ def panoptic_quality( - preds: Tensor, - target: Tensor, - things: Collection[int], - stuffs: Collection[int], - allow_unknown_preds_category: bool = False, -) -> Tensor: + preds: Tensor, + target: Tensor, + things: Collection[int], + stuffs: Collection[int], + allow_unknown_preds_category: bool = False, + return_sq_and_rq: bool = False, + per_class: bool = False, +) -> Union[Tensor, Dict[str, Tensor]]: r"""Compute `Panoptic Quality`_ for panoptic segmentations. .. math:: @@ -61,6 +63,10 @@ def panoptic_quality( allow_unknown_preds_category: Boolean flag to specify if unknown categories in the predictions are to be ignored in the metric computation or raise an exception when found. + return_sq_and_rq: + Boolean flag to specify if Segmentation Quality and Recognition Quality should be also returnd. + per_class: + Boolean flag to specify if the per-class values should be returned or the class average. Raises: ValueError: @@ -91,6 +97,36 @@ def panoptic_quality( >>> panoptic_quality(preds, target, things = {0, 1}, stuffs = {6, 7}) tensor(0.5463, dtype=torch.float64) + You can also return the segmentation and recongition quality alognside the PQ + >>> from torch import tensor + >>> preds = tensor([[[[6, 0], [0, 0], [6, 0], [6, 0]], + ... [[0, 0], [0, 0], [6, 0], [0, 1]], + ... [[0, 0], [0, 0], [6, 0], [0, 1]], + ... [[0, 0], [7, 0], [6, 0], [1, 0]], + ... [[0, 0], [7, 0], [7, 0], [7, 0]]]]) + >>> target = tensor([[[[6, 0], [0, 1], [6, 0], [0, 1]], + ... [[0, 1], [0, 1], [6, 0], [0, 1]], + ... [[0, 1], [0, 1], [6, 0], [1, 0]], + ... [[0, 1], [7, 0], [1, 0], [1, 0]], + ... [[0, 1], [7, 0], [7, 0], [7, 0]]]]) + >>> panoptic_quality(preds, target, things = {0, 1}, stuffs = {6, 7}, return_sq_and_rq=True) + {'pq': tensor(0.5463, dtype=torch.float64), 'sq': tensor(0.5463, dtype=torch.float64), 'rq': tensor(0.5463, dtype=torch.float64)} + + You can also specify to return the per-class metrics + >>> from torch import tensor + >>> preds = tensor([[[[6, 0], [0, 0], [6, 0], [6, 0]], + ... [[0, 0], [0, 0], [6, 0], [0, 1]], + ... [[0, 0], [0, 0], [6, 0], [0, 1]], + ... [[0, 0], [7, 0], [6, 0], [1, 0]], + ... [[0, 0], [7, 0], [7, 0], [7, 0]]]]) + >>> target = tensor([[[[6, 0], [0, 1], [6, 0], [0, 1]], + ... [[0, 1], [0, 1], [6, 0], [0, 1]], + ... [[0, 1], [0, 1], [6, 0], [1, 0]], + ... [[0, 1], [7, 0], [1, 0], [1, 0]], + ... [[0, 1], [7, 0], [7, 0], [7, 0]]]]) + >>> panoptic_quality(preds, target, things = {0, 1}, stuffs = {6, 7}, per_class=True) + tensor([0.5185, 0.0000, 0.6667, 1.0000], dtype=torch.float64) + """ things, stuffs = _parse_categories(things, stuffs) _validate_inputs(preds, target) @@ -101,15 +137,25 @@ def panoptic_quality( iou_sum, true_positives, false_positives, false_negatives = _panoptic_quality_update( flatten_preds, flatten_target, cat_id_to_continuous_id, void_color ) - return _panoptic_quality_compute(iou_sum, true_positives, false_positives, false_negatives) + pq, sq, rq, pq_avg, sq_avg, rq_avg = _panoptic_quality_compute(iou_sum, true_positives, + false_positives, + false_negatives, ) + if per_class: + if return_sq_and_rq: + return {"pq": pq, "sq": sq, "rq": rq} + else: + return pq + if return_sq_and_rq: + return {"pq": pq_avg, "sq": sq_avg, "rq": rq_avg} + return pq_avg def modified_panoptic_quality( - preds: Tensor, - target: Tensor, - things: Collection[int], - stuffs: Collection[int], - allow_unknown_preds_category: bool = False, + preds: Tensor, + target: Tensor, + things: Collection[int], + stuffs: Collection[int], + allow_unknown_preds_category: bool = False, ) -> Tensor: r"""Compute `Modified Panoptic Quality`_ for panoptic segmentations. @@ -177,4 +223,7 @@ def modified_panoptic_quality( void_color, modified_metric_stuffs=stuffs, ) - return _panoptic_quality_compute(iou_sum, true_positives, false_positives, false_negatives) + _, _, _, pq_avg, _, _ = _panoptic_quality_compute(iou_sum, true_positives, + false_positives, + false_negatives) + return pq_avg From bd62a9fe0ae4b16c29809facd27f776aa958f273 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 14 Feb 2024 12:59:52 +0000 Subject: [PATCH 02/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../detection/panoptic_qualities.py | 30 +++++++------- .../detection/_panoptic_quality_common.py | 2 +- .../detection/panoptic_qualities.py | 41 ++++++++++--------- 3 files changed, 37 insertions(+), 36 deletions(-) diff --git a/src/torchmetrics/detection/panoptic_qualities.py b/src/torchmetrics/detection/panoptic_qualities.py index 1a04bb2ebb8..0016d8e3ffc 100644 --- a/src/torchmetrics/detection/panoptic_qualities.py +++ b/src/torchmetrics/detection/panoptic_qualities.py @@ -56,7 +56,7 @@ class PanopticQuality(Metric): Boolean flag to specify if unknown categories in the predictions are to be ignored in the metric computation or raise an exception when found. return_sq_and_rq: - Boolean flag to specify if Segmentation Quality and Recognition Quality should be also returnd. + Boolean flag to specify if Segmentation Quality and Recognition Quality should be also returned. per_class: Boolean flag to specify if the per-class values should be returned or the class average. @@ -84,7 +84,7 @@ class PanopticQuality(Metric): >>> panoptic_quality(preds, target) tensor(0.5463, dtype=torch.float64) - You can also return the segmentation and recongition quality alognside the PQ + You can also return the segmentation and recognition quality alognside the PQ >>> from torch import tensor >>> from torchmetrics.detection import PanopticQuality >>> preds = tensor([[[[6, 0], [0, 0], [6, 0], [6, 0]], @@ -195,9 +195,9 @@ def update(self, preds: Tensor, target: Tensor) -> None: def compute(self) -> Union[Tensor, Dict[str, Tensor]]: """Compute panoptic quality based on inputs passed in to ``update`` previously.""" - pq, sq, rq, pq_avg, sq_avg, rq_avg = _panoptic_quality_compute(self.iou_sum, self.true_positives, - self.false_positives, - self.false_negatives) + pq, sq, rq, pq_avg, sq_avg, rq_avg = _panoptic_quality_compute( + self.iou_sum, self.true_positives, self.false_positives, self.false_negatives + ) if self.per_class: if self.return_sq_and_rq: return {"pq": pq, "sq": sq, "rq": rq} @@ -208,7 +208,7 @@ def compute(self) -> Union[Tensor, Dict[str, Tensor]]: return pq_avg def plot( - self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None + self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None ) -> _PLOT_OUT_TYPE: """Plot a single or multiple values from the metric. @@ -323,11 +323,11 @@ class ModifiedPanopticQuality(Metric): false_negatives: Tensor def __init__( - self, - things: Collection[int], - stuffs: Collection[int], - allow_unknown_preds_category: bool = False, - **kwargs: Any, + self, + things: Collection[int], + stuffs: Collection[int], + allow_unknown_preds_category: bool = False, + **kwargs: Any, ) -> None: super().__init__(**kwargs) @@ -387,13 +387,13 @@ def update(self, preds: Tensor, target: Tensor) -> None: def compute(self) -> Tensor: """Compute panoptic quality based on inputs passed in to ``update`` previously.""" - _, _, _, pq_avg, _, _ = _panoptic_quality_compute(self.iou_sum, self.true_positives, - self.false_positives, - self.false_negatives) + _, _, _, pq_avg, _, _ = _panoptic_quality_compute( + self.iou_sum, self.true_positives, self.false_positives, self.false_negatives + ) return pq_avg def plot( - self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None + self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None ) -> _PLOT_OUT_TYPE: """Plot a single or multiple values from the metric. diff --git a/src/torchmetrics/functional/detection/_panoptic_quality_common.py b/src/torchmetrics/functional/detection/_panoptic_quality_common.py index 8cbc06da807..fa293974822 100644 --- a/src/torchmetrics/functional/detection/_panoptic_quality_common.py +++ b/src/torchmetrics/functional/detection/_panoptic_quality_common.py @@ -464,7 +464,7 @@ def _panoptic_quality_compute( """ # compute segmentation and recognition quality (per-class) sq: Tensor = torch.where(true_positives > 0.0, iou_sum / true_positives, 0.0) - denominator: Tensor = (true_positives + 0.5 * false_positives + 0.5 * false_negatives) + denominator: Tensor = true_positives + 0.5 * false_positives + 0.5 * false_negatives rq: Tensor = torch.where(denominator > 0.0, true_positives / denominator, 0.0) # compute per-class panoptic quality pq: Tensor = sq * rq diff --git a/src/torchmetrics/functional/detection/panoptic_qualities.py b/src/torchmetrics/functional/detection/panoptic_qualities.py index 640c8016e70..ed38108d61d 100644 --- a/src/torchmetrics/functional/detection/panoptic_qualities.py +++ b/src/torchmetrics/functional/detection/panoptic_qualities.py @@ -27,13 +27,13 @@ def panoptic_quality( - preds: Tensor, - target: Tensor, - things: Collection[int], - stuffs: Collection[int], - allow_unknown_preds_category: bool = False, - return_sq_and_rq: bool = False, - per_class: bool = False, + preds: Tensor, + target: Tensor, + things: Collection[int], + stuffs: Collection[int], + allow_unknown_preds_category: bool = False, + return_sq_and_rq: bool = False, + per_class: bool = False, ) -> Union[Tensor, Dict[str, Tensor]]: r"""Compute `Panoptic Quality`_ for panoptic segmentations. @@ -64,7 +64,7 @@ def panoptic_quality( Boolean flag to specify if unknown categories in the predictions are to be ignored in the metric computation or raise an exception when found. return_sq_and_rq: - Boolean flag to specify if Segmentation Quality and Recognition Quality should be also returnd. + Boolean flag to specify if Segmentation Quality and Recognition Quality should be also returned. per_class: Boolean flag to specify if the per-class values should be returned or the class average. @@ -97,7 +97,7 @@ def panoptic_quality( >>> panoptic_quality(preds, target, things = {0, 1}, stuffs = {6, 7}) tensor(0.5463, dtype=torch.float64) - You can also return the segmentation and recongition quality alognside the PQ + You can also return the segmentation and recognition quality alognside the PQ >>> from torch import tensor >>> preds = tensor([[[[6, 0], [0, 0], [6, 0], [6, 0]], ... [[0, 0], [0, 0], [6, 0], [0, 1]], @@ -137,9 +137,12 @@ def panoptic_quality( iou_sum, true_positives, false_positives, false_negatives = _panoptic_quality_update( flatten_preds, flatten_target, cat_id_to_continuous_id, void_color ) - pq, sq, rq, pq_avg, sq_avg, rq_avg = _panoptic_quality_compute(iou_sum, true_positives, - false_positives, - false_negatives, ) + pq, sq, rq, pq_avg, sq_avg, rq_avg = _panoptic_quality_compute( + iou_sum, + true_positives, + false_positives, + false_negatives, + ) if per_class: if return_sq_and_rq: return {"pq": pq, "sq": sq, "rq": rq} @@ -151,11 +154,11 @@ def panoptic_quality( def modified_panoptic_quality( - preds: Tensor, - target: Tensor, - things: Collection[int], - stuffs: Collection[int], - allow_unknown_preds_category: bool = False, + preds: Tensor, + target: Tensor, + things: Collection[int], + stuffs: Collection[int], + allow_unknown_preds_category: bool = False, ) -> Tensor: r"""Compute `Modified Panoptic Quality`_ for panoptic segmentations. @@ -223,7 +226,5 @@ def modified_panoptic_quality( void_color, modified_metric_stuffs=stuffs, ) - _, _, _, pq_avg, _, _ = _panoptic_quality_compute(iou_sum, true_positives, - false_positives, - false_negatives) + _, _, _, pq_avg, _, _ = _panoptic_quality_compute(iou_sum, true_positives, false_positives, false_negatives) return pq_avg From 967dab8ea18e1ccabf4caf34cb2cac84c16651bc Mon Sep 17 00:00:00 2001 From: ChristophReich1996 Date: Wed, 14 Feb 2024 14:11:09 +0100 Subject: [PATCH 03/20] Fix RQ and SQ --- .../functional/detection/_panoptic_quality_common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/torchmetrics/functional/detection/_panoptic_quality_common.py b/src/torchmetrics/functional/detection/_panoptic_quality_common.py index 8cbc06da807..e77f451b91c 100644 --- a/src/torchmetrics/functional/detection/_panoptic_quality_common.py +++ b/src/torchmetrics/functional/detection/_panoptic_quality_common.py @@ -470,6 +470,6 @@ def _panoptic_quality_compute( pq: Tensor = sq * rq # compute averages pq_avg: Tensor = torch.mean(pq[denominator > 0]) - sq_avg: Tensor = torch.mean(pq[denominator > 0]) - rq_avg: Tensor = torch.mean(pq[denominator > 0]) + sq_avg: Tensor = torch.mean(sq[denominator > 0]) + rq_avg: Tensor = torch.mean(rq[denominator > 0]) return pq, sq, rq, pq_avg, sq_avg, rq_avg From be1a4a0fed50df168e1cddb57e4decb6bf84f5cc Mon Sep 17 00:00:00 2001 From: ChristophReich1996 Date: Thu, 15 Feb 2024 11:44:03 +0100 Subject: [PATCH 04/20] Change return type and refactor flag name --- .../detection/panoptic_qualities.py | 46 +++++++------- .../detection/panoptic_qualities.py | 60 ++++++++++++------- 2 files changed, 63 insertions(+), 43 deletions(-) diff --git a/src/torchmetrics/detection/panoptic_qualities.py b/src/torchmetrics/detection/panoptic_qualities.py index 0016d8e3ffc..c56d9567be5 100644 --- a/src/torchmetrics/detection/panoptic_qualities.py +++ b/src/torchmetrics/detection/panoptic_qualities.py @@ -57,7 +57,7 @@ class PanopticQuality(Metric): computation or raise an exception when found. return_sq_and_rq: Boolean flag to specify if Segmentation Quality and Recognition Quality should be also returned. - per_class: + return_per_class: Boolean flag to specify if the per-class values should be returned or the class average. @@ -99,7 +99,7 @@ class PanopticQuality(Metric): ... [[0, 1], [7, 0], [7, 0], [7, 0]]]]) >>> panoptic_quality = PanopticQuality(things = {0, 1}, stuffs = {6, 7}, return_sq_and_rq=True) >>> panoptic_quality(preds, target) - {'pq': tensor(0.5463, dtype=torch.float64), 'sq': tensor(0.5463, dtype=torch.float64), 'rq': tensor(0.5463, dtype=torch.float64)} + tensor([0.5463, 0.6111, 0.6667], dtype=torch.float64) You can also specify to return the per-class metrics >>> from torch import tensor @@ -114,9 +114,9 @@ class PanopticQuality(Metric): ... [[0, 1], [0, 1], [6, 0], [1, 0]], ... [[0, 1], [7, 0], [1, 0], [1, 0]], ... [[0, 1], [7, 0], [7, 0], [7, 0]]]]) - >>> panoptic_quality = PanopticQuality(things = {0, 1}, stuffs = {6, 7}, per_class=True) + >>> panoptic_quality = PanopticQuality(things = {0, 1}, stuffs = {6, 7}, return_per_class=True) >>> panoptic_quality(preds, target) - tensor([0.5185, 0.0000, 0.6667, 1.0000], dtype=torch.float64) + tensor([[0.5185, 0.0000, 0.6667, 1.0000]], dtype=torch.float64) """ is_differentiable: bool = False @@ -131,13 +131,13 @@ class PanopticQuality(Metric): false_negatives: Tensor def __init__( - self, - things: Collection[int], - stuffs: Collection[int], - allow_unknown_preds_category: bool = False, - return_sq_and_rq: bool = False, - per_class: bool = False, - **kwargs: Any, + self, + things: Collection[int], + stuffs: Collection[int], + allow_unknown_preds_category: bool = False, + return_sq_and_rq: bool = False, + return_per_class: bool = False, + **kwargs: Any, ) -> None: super().__init__(**kwargs) @@ -148,7 +148,7 @@ def __init__( self.cat_id_to_continuous_id = _get_category_id_to_continuous_id(things, stuffs) self.allow_unknown_preds_category = allow_unknown_preds_category self.return_sq_and_rq = return_sq_and_rq - self.per_class = per_class + self.return_per_class = return_per_class # per category intermediate metrics num_categories = len(things) + len(stuffs) @@ -198,17 +198,17 @@ def compute(self) -> Union[Tensor, Dict[str, Tensor]]: pq, sq, rq, pq_avg, sq_avg, rq_avg = _panoptic_quality_compute( self.iou_sum, self.true_positives, self.false_positives, self.false_negatives ) - if self.per_class: + if self.return_per_class: if self.return_sq_and_rq: - return {"pq": pq, "sq": sq, "rq": rq} + return torch.stack((pq, sq, rq), dim=-1) else: - return pq + return pq.view(1, -1) if self.return_sq_and_rq: - return {"pq": pq_avg, "sq": sq_avg, "rq": rq_avg} + return torch.stack((pq_avg, sq_avg, rq_avg), dim=0) return pq_avg def plot( - self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None + self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None ) -> _PLOT_OUT_TYPE: """Plot a single or multiple values from the metric. @@ -323,11 +323,11 @@ class ModifiedPanopticQuality(Metric): false_negatives: Tensor def __init__( - self, - things: Collection[int], - stuffs: Collection[int], - allow_unknown_preds_category: bool = False, - **kwargs: Any, + self, + things: Collection[int], + stuffs: Collection[int], + allow_unknown_preds_category: bool = False, + **kwargs: Any, ) -> None: super().__init__(**kwargs) @@ -393,7 +393,7 @@ def compute(self) -> Tensor: return pq_avg def plot( - self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None + self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None ) -> _PLOT_OUT_TYPE: """Plot a single or multiple values from the metric. diff --git a/src/torchmetrics/functional/detection/panoptic_qualities.py b/src/torchmetrics/functional/detection/panoptic_qualities.py index ed38108d61d..384f9a2d2f0 100644 --- a/src/torchmetrics/functional/detection/panoptic_qualities.py +++ b/src/torchmetrics/functional/detection/panoptic_qualities.py @@ -13,6 +13,7 @@ # limitations under the License. from typing import Collection, Dict, Union +import torch from torch import Tensor from torchmetrics.functional.detection._panoptic_quality_common import ( @@ -27,13 +28,13 @@ def panoptic_quality( - preds: Tensor, - target: Tensor, - things: Collection[int], - stuffs: Collection[int], - allow_unknown_preds_category: bool = False, - return_sq_and_rq: bool = False, - per_class: bool = False, + preds: Tensor, + target: Tensor, + things: Collection[int], + stuffs: Collection[int], + allow_unknown_preds_category: bool = False, + return_sq_and_rq: bool = False, + return_per_class: bool = False, ) -> Union[Tensor, Dict[str, Tensor]]: r"""Compute `Panoptic Quality`_ for panoptic segmentations. @@ -65,7 +66,7 @@ def panoptic_quality( computation or raise an exception when found. return_sq_and_rq: Boolean flag to specify if Segmentation Quality and Recognition Quality should be also returned. - per_class: + return_per_class: Boolean flag to specify if the per-class values should be returned or the class average. Raises: @@ -110,7 +111,7 @@ def panoptic_quality( ... [[0, 1], [7, 0], [1, 0], [1, 0]], ... [[0, 1], [7, 0], [7, 0], [7, 0]]]]) >>> panoptic_quality(preds, target, things = {0, 1}, stuffs = {6, 7}, return_sq_and_rq=True) - {'pq': tensor(0.5463, dtype=torch.float64), 'sq': tensor(0.5463, dtype=torch.float64), 'rq': tensor(0.5463, dtype=torch.float64)} + tensor([0.5463, 0.6111, 0.6667], dtype=torch.float64) You can also specify to return the per-class metrics >>> from torch import tensor @@ -124,8 +125,27 @@ def panoptic_quality( ... [[0, 1], [0, 1], [6, 0], [1, 0]], ... [[0, 1], [7, 0], [1, 0], [1, 0]], ... [[0, 1], [7, 0], [7, 0], [7, 0]]]]) - >>> panoptic_quality(preds, target, things = {0, 1}, stuffs = {6, 7}, per_class=True) - tensor([0.5185, 0.0000, 0.6667, 1.0000], dtype=torch.float64) + >>> panoptic_quality(preds, target, things = {0, 1}, stuffs = {6, 7}, return_per_class=True) + tensor([[0.5185, 0.0000, 0.6667, 1.0000]], dtype=torch.float64) + + You can also specify to return the per-class metrics and the segmentation and recognition quality + >>> from torch import tensor + >>> preds = tensor([[[[6, 0], [0, 0], [6, 0], [6, 0]], + ... [[0, 0], [0, 0], [6, 0], [0, 1]], + ... [[0, 0], [0, 0], [6, 0], [0, 1]], + ... [[0, 0], [7, 0], [6, 0], [1, 0]], + ... [[0, 0], [7, 0], [7, 0], [7, 0]]]]) + >>> target = tensor([[[[6, 0], [0, 1], [6, 0], [0, 1]], + ... [[0, 1], [0, 1], [6, 0], [0, 1]], + ... [[0, 1], [0, 1], [6, 0], [1, 0]], + ... [[0, 1], [7, 0], [1, 0], [1, 0]], + ... [[0, 1], [7, 0], [7, 0], [7, 0]]]]) + >>> panoptic_quality(preds, target, things = {0, 1}, stuffs = {6, 7}, + ... return_per_class=True, return_sq_and_rq=True) + tensor([[0.5185, 0.7778, 0.6667], + [0.0000, 0.0000, 0.0000], + [0.6667, 0.6667, 1.0000], + [1.0000, 1.0000, 1.0000]], dtype=torch.float64) """ things, stuffs = _parse_categories(things, stuffs) @@ -143,22 +163,22 @@ def panoptic_quality( false_positives, false_negatives, ) - if per_class: + if return_per_class: if return_sq_and_rq: - return {"pq": pq, "sq": sq, "rq": rq} + return torch.stack((pq, sq, rq), dim=-1) else: - return pq + return pq.view(1, -1) if return_sq_and_rq: - return {"pq": pq_avg, "sq": sq_avg, "rq": rq_avg} + return torch.stack((pq_avg, sq_avg, rq_avg), dim=0) return pq_avg def modified_panoptic_quality( - preds: Tensor, - target: Tensor, - things: Collection[int], - stuffs: Collection[int], - allow_unknown_preds_category: bool = False, + preds: Tensor, + target: Tensor, + things: Collection[int], + stuffs: Collection[int], + allow_unknown_preds_category: bool = False, ) -> Tensor: r"""Compute `Modified Panoptic Quality`_ for panoptic segmentations. From a7deffd609eb7c6e34154722331454cfc123c5e0 Mon Sep 17 00:00:00 2001 From: ChristophReich1996 Date: Thu, 15 Feb 2024 11:44:32 +0100 Subject: [PATCH 05/20] Fix typing --- src/torchmetrics/detection/panoptic_qualities.py | 2 +- src/torchmetrics/functional/detection/panoptic_qualities.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/torchmetrics/detection/panoptic_qualities.py b/src/torchmetrics/detection/panoptic_qualities.py index c56d9567be5..e4f2e327e54 100644 --- a/src/torchmetrics/detection/panoptic_qualities.py +++ b/src/torchmetrics/detection/panoptic_qualities.py @@ -193,7 +193,7 @@ def update(self, preds: Tensor, target: Tensor) -> None: self.false_positives += false_positives self.false_negatives += false_negatives - def compute(self) -> Union[Tensor, Dict[str, Tensor]]: + def compute(self) -> Tensor: """Compute panoptic quality based on inputs passed in to ``update`` previously.""" pq, sq, rq, pq_avg, sq_avg, rq_avg = _panoptic_quality_compute( self.iou_sum, self.true_positives, self.false_positives, self.false_negatives diff --git a/src/torchmetrics/functional/detection/panoptic_qualities.py b/src/torchmetrics/functional/detection/panoptic_qualities.py index 384f9a2d2f0..7d37e7f40fa 100644 --- a/src/torchmetrics/functional/detection/panoptic_qualities.py +++ b/src/torchmetrics/functional/detection/panoptic_qualities.py @@ -35,7 +35,7 @@ def panoptic_quality( allow_unknown_preds_category: bool = False, return_sq_and_rq: bool = False, return_per_class: bool = False, -) -> Union[Tensor, Dict[str, Tensor]]: +) -> Tensor: r"""Compute `Panoptic Quality`_ for panoptic segmentations. .. math:: From d897c9a3fdecea599582ee0131a88358b4a2b3df Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 Feb 2024 10:45:29 +0000 Subject: [PATCH 06/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../detection/panoptic_qualities.py | 28 +++++++++---------- .../detection/panoptic_qualities.py | 24 ++++++++-------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/torchmetrics/detection/panoptic_qualities.py b/src/torchmetrics/detection/panoptic_qualities.py index e4f2e327e54..3d61179cad5 100644 --- a/src/torchmetrics/detection/panoptic_qualities.py +++ b/src/torchmetrics/detection/panoptic_qualities.py @@ -131,13 +131,13 @@ class PanopticQuality(Metric): false_negatives: Tensor def __init__( - self, - things: Collection[int], - stuffs: Collection[int], - allow_unknown_preds_category: bool = False, - return_sq_and_rq: bool = False, - return_per_class: bool = False, - **kwargs: Any, + self, + things: Collection[int], + stuffs: Collection[int], + allow_unknown_preds_category: bool = False, + return_sq_and_rq: bool = False, + return_per_class: bool = False, + **kwargs: Any, ) -> None: super().__init__(**kwargs) @@ -208,7 +208,7 @@ def compute(self) -> Tensor: return pq_avg def plot( - self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None + self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None ) -> _PLOT_OUT_TYPE: """Plot a single or multiple values from the metric. @@ -323,11 +323,11 @@ class ModifiedPanopticQuality(Metric): false_negatives: Tensor def __init__( - self, - things: Collection[int], - stuffs: Collection[int], - allow_unknown_preds_category: bool = False, - **kwargs: Any, + self, + things: Collection[int], + stuffs: Collection[int], + allow_unknown_preds_category: bool = False, + **kwargs: Any, ) -> None: super().__init__(**kwargs) @@ -393,7 +393,7 @@ def compute(self) -> Tensor: return pq_avg def plot( - self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None + self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None ) -> _PLOT_OUT_TYPE: """Plot a single or multiple values from the metric. diff --git a/src/torchmetrics/functional/detection/panoptic_qualities.py b/src/torchmetrics/functional/detection/panoptic_qualities.py index 7d37e7f40fa..65a2f37f8eb 100644 --- a/src/torchmetrics/functional/detection/panoptic_qualities.py +++ b/src/torchmetrics/functional/detection/panoptic_qualities.py @@ -28,13 +28,13 @@ def panoptic_quality( - preds: Tensor, - target: Tensor, - things: Collection[int], - stuffs: Collection[int], - allow_unknown_preds_category: bool = False, - return_sq_and_rq: bool = False, - return_per_class: bool = False, + preds: Tensor, + target: Tensor, + things: Collection[int], + stuffs: Collection[int], + allow_unknown_preds_category: bool = False, + return_sq_and_rq: bool = False, + return_per_class: bool = False, ) -> Tensor: r"""Compute `Panoptic Quality`_ for panoptic segmentations. @@ -174,11 +174,11 @@ def panoptic_quality( def modified_panoptic_quality( - preds: Tensor, - target: Tensor, - things: Collection[int], - stuffs: Collection[int], - allow_unknown_preds_category: bool = False, + preds: Tensor, + target: Tensor, + things: Collection[int], + stuffs: Collection[int], + allow_unknown_preds_category: bool = False, ) -> Tensor: r"""Compute `Modified Panoptic Quality`_ for panoptic segmentations. From 35f15d5b9d22e8eb7ca95fdc7365c720f3ae1e9b Mon Sep 17 00:00:00 2001 From: Nicki Skafte Detlefsen Date: Thu, 15 Feb 2024 12:50:50 +0100 Subject: [PATCH 07/20] changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f2aa69ba29..28224a70b50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `QualityWithNoReference` metric ([#2288](https://github.com/Lightning-AI/torchmetrics/pull/2288)) +- Added support for calculating segmentation quality and recognition quality in `PanopticQuality` metric ([#2381](https://github.com/Lightning-AI/torchmetrics/pull/2381)) + + ### Changed - From cf8e0f5bbd9b8e565e34885d86bd71738be7c9cd Mon Sep 17 00:00:00 2001 From: Nicki Skafte Detlefsen Date: Thu, 15 Feb 2024 12:58:52 +0100 Subject: [PATCH 08/20] input/output docstring --- .../detection/panoptic_qualities.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/torchmetrics/detection/panoptic_qualities.py b/src/torchmetrics/detection/panoptic_qualities.py index 3d61179cad5..95dfcb9baa2 100644 --- a/src/torchmetrics/detection/panoptic_qualities.py +++ b/src/torchmetrics/detection/panoptic_qualities.py @@ -47,6 +47,23 @@ class PanopticQuality(Metric): Points in the target tensor that do not map to a known category ID are automatically ignored in the metric computation. + As input to ``forward`` and ``update`` the metric accepts the following input: + + - ``preds`` (:class:`~torch.Tensor`): An int tensor of shape ``(B, *spatial_dims, 2)``, where there needs to + be atleast one spatial dimension. + - ``target`` (:class:`~torch.Tensor`): An int tensor of shape ``(B, *spatial_dims, 2)``, where there needs to + be atleast one spatial dimension. + + As output to ``forward`` and ``compute`` the metric returns the following output: + + - ``quality`` (:class:`~torch.Tensor`): If ``return_sq_and_rq=False`` and ``return_per_class=False`` then a single + scalar tensor is returned with average panoptic quality over all classes. If ``return_sq_and_rq=True`` and + ``return_per_class=False`` a tensor of length 3 is returned with panoptic, segmentation and recognition quality + (in that order). If If ``return_sq_and_rq=False`` and ``return_per_class=True`` a tensor of length equal to the + number of classes are returned, with panoptic quality for each class. Finally, if both arguments are ``True`` + a tensor of shape ``(3, C)`` is returned with individual panoptic, segmentation and recognition quality for each + class. + Args: things: Set of ``category_id`` for countable things. From e860aed20ed1a653118529ec33bbb878dda9e4ea Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 Feb 2024 11:59:30 +0000 Subject: [PATCH 09/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/torchmetrics/detection/panoptic_qualities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/torchmetrics/detection/panoptic_qualities.py b/src/torchmetrics/detection/panoptic_qualities.py index 95dfcb9baa2..799aedca9ab 100644 --- a/src/torchmetrics/detection/panoptic_qualities.py +++ b/src/torchmetrics/detection/panoptic_qualities.py @@ -50,9 +50,9 @@ class PanopticQuality(Metric): As input to ``forward`` and ``update`` the metric accepts the following input: - ``preds`` (:class:`~torch.Tensor`): An int tensor of shape ``(B, *spatial_dims, 2)``, where there needs to - be atleast one spatial dimension. + be at least one spatial dimension. - ``target`` (:class:`~torch.Tensor`): An int tensor of shape ``(B, *spatial_dims, 2)``, where there needs to - be atleast one spatial dimension. + be at least one spatial dimension. As output to ``forward`` and ``compute`` the metric returns the following output: From 9b70dfddfb1d79b92d3a276ee2a0001fd1fc1cae Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Wed, 6 Mar 2024 19:20:55 +0100 Subject: [PATCH 10/20] guard against older versions --- .../detection/panoptic_qualities.py | 23 ++++++++++--------- .../detection/panoptic_qualities.py | 9 +++++--- .../detection/test_panoptic_quality.py | 2 ++ 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/torchmetrics/detection/panoptic_qualities.py b/src/torchmetrics/detection/panoptic_qualities.py index 67afcf54856..7106f655f76 100644 --- a/src/torchmetrics/detection/panoptic_qualities.py +++ b/src/torchmetrics/detection/panoptic_qualities.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Collection, Dict, Optional, Sequence, Union +from typing import Any, Collection, Optional, Sequence, Union import torch from torch import Tensor @@ -26,7 +26,7 @@ _validate_inputs, ) from torchmetrics.metric import Metric -from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE +from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE, _TORCH_GREATER_EQUAL_1_12 from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE if not _MATPLOTLIB_AVAILABLE: @@ -56,13 +56,13 @@ class PanopticQuality(Metric): As output to ``forward`` and ``compute`` the metric returns the following output: - - ``quality`` (:class:`~torch.Tensor`): If ``return_sq_and_rq=False`` and ``return_per_class=False`` then a single - scalar tensor is returned with average panoptic quality over all classes. If ``return_sq_and_rq=True`` and - ``return_per_class=False`` a tensor of length 3 is returned with panoptic, segmentation and recognition quality - (in that order). If If ``return_sq_and_rq=False`` and ``return_per_class=True`` a tensor of length equal to the - number of classes are returned, with panoptic quality for each class. Finally, if both arguments are ``True`` - a tensor of shape ``(3, C)`` is returned with individual panoptic, segmentation and recognition quality for each - class. + - ``quality`` (:class:`~torch.Tensor`): If ``return_sq_and_rq=False`` and ``return_per_class=False`` then a + single scalar tensor is returned with average panoptic quality over all classes. If ``return_sq_and_rq=True`` + and ``return_per_class=False`` a tensor of length 3 is returned with panoptic, segmentation and recognition + quality (in that order). If If ``return_sq_and_rq=False`` and ``return_per_class=True`` a tensor of length + equal to the number of classes are returned, with panoptic quality for each class. Finally, if both arguments + are ``True`` a tensor of shape ``(3, C)`` is returned with individual panoptic, segmentation and recognition + quality for each class. Args: things: @@ -158,6 +158,8 @@ def __init__( **kwargs: Any, ) -> None: super().__init__(**kwargs) + if not _TORCH_GREATER_EQUAL_1_12: + raise RuntimeError("Panoptic Quality metric requires PyTorch 1.12 or later") things, stuffs = _parse_categories(things, stuffs) self.things = things @@ -219,8 +221,7 @@ def compute(self) -> Tensor: if self.return_per_class: if self.return_sq_and_rq: return torch.stack((pq, sq, rq), dim=-1) - else: - return pq.view(1, -1) + return pq.view(1, -1) if self.return_sq_and_rq: return torch.stack((pq_avg, sq_avg, rq_avg), dim=0) return pq_avg diff --git a/src/torchmetrics/functional/detection/panoptic_qualities.py b/src/torchmetrics/functional/detection/panoptic_qualities.py index 65a2f37f8eb..fb8f443f490 100644 --- a/src/torchmetrics/functional/detection/panoptic_qualities.py +++ b/src/torchmetrics/functional/detection/panoptic_qualities.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Collection, Dict, Union +from typing import Collection import torch from torch import Tensor @@ -25,6 +25,7 @@ _prepocess_inputs, _validate_inputs, ) +from torchmetrics.utilities.imports import _TORCH_GREATER_EQUAL_1_12 def panoptic_quality( @@ -148,6 +149,9 @@ def panoptic_quality( [1.0000, 1.0000, 1.0000]], dtype=torch.float64) """ + if not _TORCH_GREATER_EQUAL_1_12: + raise RuntimeError("Panoptic Quality metric requires PyTorch 1.12 or later") + things, stuffs = _parse_categories(things, stuffs) _validate_inputs(preds, target) void_color = _get_void_color(things, stuffs) @@ -166,8 +170,7 @@ def panoptic_quality( if return_per_class: if return_sq_and_rq: return torch.stack((pq, sq, rq), dim=-1) - else: - return pq.view(1, -1) + return pq.view(1, -1) if return_sq_and_rq: return torch.stack((pq_avg, sq_avg, rq_avg), dim=0) return pq_avg diff --git a/tests/unittests/detection/test_panoptic_quality.py b/tests/unittests/detection/test_panoptic_quality.py index 9a2b801e0f4..a548ab2608e 100644 --- a/tests/unittests/detection/test_panoptic_quality.py +++ b/tests/unittests/detection/test_panoptic_quality.py @@ -18,6 +18,7 @@ import torch from torchmetrics.detection.panoptic_qualities import PanopticQuality from torchmetrics.functional.detection.panoptic_qualities import panoptic_quality +from torchmetrics.utilities.imports import _TORCH_GREATER_EQUAL_1_12 from unittests import _Input from unittests.helpers import seed_all @@ -83,6 +84,7 @@ def _reference_fn_1_2(preds, target) -> np.ndarray: return np.array([(2 / 3 + 1 + 2 / 3) / 3]) +@pytest.mark.skipif(not _TORCH_GREATER_EQUAL_1_12, reason="PanopticQuality metric only supports PyTorch >= 1.12") class TestPanopticQuality(MetricTester): """Test class for `PanopticQuality` metric.""" From f3d33c89eae9440dc306ccf8ae7f32f72a977be0 Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Wed, 6 Mar 2024 19:42:42 +0100 Subject: [PATCH 11/20] skip doctests on older versions --- src/torchmetrics/detection/panoptic_qualities.py | 3 +++ src/torchmetrics/functional/detection/panoptic_qualities.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/torchmetrics/detection/panoptic_qualities.py b/src/torchmetrics/detection/panoptic_qualities.py index 7106f655f76..26fe12befa4 100644 --- a/src/torchmetrics/detection/panoptic_qualities.py +++ b/src/torchmetrics/detection/panoptic_qualities.py @@ -32,6 +32,9 @@ if not _MATPLOTLIB_AVAILABLE: __doctest_skip__ = ["PanopticQuality.plot", "ModifiedPanopticQuality.plot"] +if not _TORCH_GREATER_EQUAL_1_12: + __doctest_skip__ = ["PanopticQuality", "ModifiedPanopticQuality"] + class PanopticQuality(Metric): r"""Compute the `Panoptic Quality`_ for panoptic segmentations. diff --git a/src/torchmetrics/functional/detection/panoptic_qualities.py b/src/torchmetrics/functional/detection/panoptic_qualities.py index fb8f443f490..a7db353abb0 100644 --- a/src/torchmetrics/functional/detection/panoptic_qualities.py +++ b/src/torchmetrics/functional/detection/panoptic_qualities.py @@ -27,6 +27,9 @@ ) from torchmetrics.utilities.imports import _TORCH_GREATER_EQUAL_1_12 +if not _TORCH_GREATER_EQUAL_1_12: + __doctest_skip__ = ["panoptic_qualit"] + def panoptic_quality( preds: Tensor, From 07524f29fb349bf3769c650fa9f3ad8aa72155fc Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 14 Mar 2024 13:17:09 +0100 Subject: [PATCH 12/20] ci/gpu: do not fail if HF cache is not present --- .azure/gpu-unittests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.azure/gpu-unittests.yml b/.azure/gpu-unittests.yml index 6bd336848f0..3b368269331 100644 --- a/.azure/gpu-unittests.yml +++ b/.azure/gpu-unittests.yml @@ -210,4 +210,6 @@ jobs: - bash: | printf "cache location: $(HF_HOME)\n" ls -lh $(HF_HOME) # show what was restored... + # do not fail if the cache is not present + continue-on-error: true displayName: "Show HF artifacts" From ef0cc27ca8596b59b2575801f1d0a76bb982b28c Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 14 Mar 2024 13:20:29 +0100 Subject: [PATCH 13/20] ci/gpu: do not update ref on PR --- .azure/gpu-unittests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.azure/gpu-unittests.yml b/.azure/gpu-unittests.yml index 3b368269331..8f1bc90759e 100644 --- a/.azure/gpu-unittests.yml +++ b/.azure/gpu-unittests.yml @@ -181,6 +181,8 @@ jobs: # copy potentially updated cache to the machine filesystem to be reused with next jobs cp -r --update tests/_cache-references /var/tmp/cached-references # set as extra step to not pollute general cache when jobs fails or crashes + # so do this update only with successful jobs on master + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) displayName: "Update cached refs" - bash: | From 9fb488da6c830264effdebe0167465a767ae9e61 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 13:20:56 +0100 Subject: [PATCH 14/20] build(deps): update fire requirement from <=0.5.0 to <=0.6.0 in /requirements (#2445) build(deps): update fire requirement in /requirements Updates the requirements on [fire](https://github.com/google/python-fire) to permit the latest version. - [Release notes](https://github.com/google/python-fire/releases) - [Commits](https://github.com/google/python-fire/compare/v0.1.0...v0.6.0) --- updated-dependencies: - dependency-name: fire dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/_tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/_tests.txt b/requirements/_tests.txt index 0dccca202fe..121827a92ba 100644 --- a/requirements/_tests.txt +++ b/requirements/_tests.txt @@ -12,7 +12,7 @@ phmdoctest ==1.4.0 psutil <5.10.0 pyGithub ==2.2.0 -fire <=0.5.0 +fire <=0.6.0 cloudpickle >1.3, <=3.0.0 scikit-learn >=1.1.1, <1.4.0 From 02401101fd64f439c1cff6d7ba1b1a71e5951188 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 13:24:17 +0100 Subject: [PATCH 15/20] build(deps): bump pytest-timeout from 2.2.0 to 2.3.1 in /requirements (#2444) Bumps [pytest-timeout](https://github.com/pytest-dev/pytest-timeout) from 2.2.0 to 2.3.1. - [Commits](https://github.com/pytest-dev/pytest-timeout/compare/2.2.0...2.3.1) --- updated-dependencies: - dependency-name: pytest-timeout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/_tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/_tests.txt b/requirements/_tests.txt index 121827a92ba..455a31d0e44 100644 --- a/requirements/_tests.txt +++ b/requirements/_tests.txt @@ -6,7 +6,7 @@ pytest ==8.1.1 pytest-cov ==4.1.0 pytest-doctestplus ==1.2.1 pytest-rerunfailures ==13.0 -pytest-timeout ==2.2.0 +pytest-timeout ==2.3.1 pytest-xdist ==3.5.0 phmdoctest ==1.4.0 From 090e3ed5c50478e43b4f12c07edd0387129a5b4e Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 14 Mar 2024 13:27:16 +0100 Subject: [PATCH 16/20] ci/mergify: rename label `ready` --- .github/mergify.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/mergify.yml b/.github/mergify.yml index 62f1e436925..d1ce4949dc8 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -57,7 +57,7 @@ pull_request_rules: - "#check-failure<5" actions: label: - add: ["0:] Ready-To-Go"] + add: ["ready"] - name: Not ready yet conditions: @@ -69,7 +69,7 @@ pull_request_rules: - "#check-failure>=5" actions: label: - remove: ["0:] Ready-To-Go"] + remove: ["ready"] - name: add core reviewer conditions: From 0e04dbe29c30decbb2cb1be080ede15b031d2da1 Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Sun, 24 Mar 2024 15:32:40 +0100 Subject: [PATCH 17/20] fix skipping on older versions --- src/torchmetrics/detection/_deprecated.py | 9 +++++++++ src/torchmetrics/detection/panoptic_qualities.py | 6 +++--- src/torchmetrics/functional/detection/_deprecated.py | 4 ++++ .../functional/detection/panoptic_qualities.py | 2 +- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/torchmetrics/detection/_deprecated.py b/src/torchmetrics/detection/_deprecated.py index c162c751554..898f341bd62 100644 --- a/src/torchmetrics/detection/_deprecated.py +++ b/src/torchmetrics/detection/_deprecated.py @@ -1,8 +1,17 @@ from typing import Any, Collection from torchmetrics.detection import ModifiedPanopticQuality, PanopticQuality +from torchmetrics.utilities.imports import _TORCH_GREATER_EQUAL_1_12 from torchmetrics.utilities.prints import _deprecated_root_import_class +if not _TORCH_GREATER_EQUAL_1_12: + __doctest_skip__ = [ + "_PanopticQuality", + "_PanopticQuality.*", + "_ModifiedPanopticQuality", + "_ModifiedPanopticQuality.*", + ] + class _ModifiedPanopticQuality(ModifiedPanopticQuality): """Wrapper for deprecated import. diff --git a/src/torchmetrics/detection/panoptic_qualities.py b/src/torchmetrics/detection/panoptic_qualities.py index 26fe12befa4..a19456c1f54 100644 --- a/src/torchmetrics/detection/panoptic_qualities.py +++ b/src/torchmetrics/detection/panoptic_qualities.py @@ -29,12 +29,12 @@ from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE, _TORCH_GREATER_EQUAL_1_12 from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE +if not _TORCH_GREATER_EQUAL_1_12: + __doctest_skip__ = ["PanopticQuality", "PanopticQuality.*", "ModifiedPanopticQuality", "ModifiedPanopticQuality.*"] + if not _MATPLOTLIB_AVAILABLE: __doctest_skip__ = ["PanopticQuality.plot", "ModifiedPanopticQuality.plot"] -if not _TORCH_GREATER_EQUAL_1_12: - __doctest_skip__ = ["PanopticQuality", "ModifiedPanopticQuality"] - class PanopticQuality(Metric): r"""Compute the `Panoptic Quality`_ for panoptic segmentations. diff --git a/src/torchmetrics/functional/detection/_deprecated.py b/src/torchmetrics/functional/detection/_deprecated.py index ce0e1ba6acf..b2500d34f0d 100644 --- a/src/torchmetrics/functional/detection/_deprecated.py +++ b/src/torchmetrics/functional/detection/_deprecated.py @@ -3,8 +3,12 @@ from torch import Tensor from torchmetrics.functional.detection.panoptic_qualities import modified_panoptic_quality, panoptic_quality +from torchmetrics.utilities.imports import _TORCH_GREATER_EQUAL_1_12 from torchmetrics.utilities.prints import _deprecated_root_import_func +if not _TORCH_GREATER_EQUAL_1_12: + __doctest_skip__ = ["_panoptic_quality", "_modified_panoptic_quality"] + def _modified_panoptic_quality( preds: Tensor, diff --git a/src/torchmetrics/functional/detection/panoptic_qualities.py b/src/torchmetrics/functional/detection/panoptic_qualities.py index a7db353abb0..2de9fa09bfa 100644 --- a/src/torchmetrics/functional/detection/panoptic_qualities.py +++ b/src/torchmetrics/functional/detection/panoptic_qualities.py @@ -28,7 +28,7 @@ from torchmetrics.utilities.imports import _TORCH_GREATER_EQUAL_1_12 if not _TORCH_GREATER_EQUAL_1_12: - __doctest_skip__ = ["panoptic_qualit"] + __doctest_skip__ = ["panoptic_quality", "modified_panoptic_quality"] def panoptic_quality( From 699673d2b8412d131f5d0ec72c94691e136a0477 Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Sun, 24 Mar 2024 15:57:26 +0100 Subject: [PATCH 18/20] change order of skipping --- src/torchmetrics/detection/panoptic_qualities.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/torchmetrics/detection/panoptic_qualities.py b/src/torchmetrics/detection/panoptic_qualities.py index a19456c1f54..5f8aefabbc1 100644 --- a/src/torchmetrics/detection/panoptic_qualities.py +++ b/src/torchmetrics/detection/panoptic_qualities.py @@ -29,13 +29,14 @@ from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE, _TORCH_GREATER_EQUAL_1_12 from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE -if not _TORCH_GREATER_EQUAL_1_12: - __doctest_skip__ = ["PanopticQuality", "PanopticQuality.*", "ModifiedPanopticQuality", "ModifiedPanopticQuality.*"] - if not _MATPLOTLIB_AVAILABLE: __doctest_skip__ = ["PanopticQuality.plot", "ModifiedPanopticQuality.plot"] +if not _TORCH_GREATER_EQUAL_1_12: + __doctest_skip__ = ["PanopticQuality", "PanopticQuality.*", "ModifiedPanopticQuality", "ModifiedPanopticQuality.*"] + + class PanopticQuality(Metric): r"""Compute the `Panoptic Quality`_ for panoptic segmentations. From 78cbbf89eac3efb12408350ac0379c3f9adbfcdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Furkan=20=C3=87elik?= Date: Sun, 24 Mar 2024 16:12:18 +0100 Subject: [PATCH 19/20] Fix return type documentation of inception score (#2467) * Fix return type documentation of inception score * fix too long line --------- Co-authored-by: Nicki Skafte Detlefsen --- src/torchmetrics/image/inception.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/torchmetrics/image/inception.py b/src/torchmetrics/image/inception.py index 80444c911ee..4913a5fe50b 100644 --- a/src/torchmetrics/image/inception.py +++ b/src/torchmetrics/image/inception.py @@ -38,7 +38,7 @@ class InceptionScore(Metric): IS = exp(\mathbb{E}_x KL(p(y | x ) || p(y))) where :math:`KL(p(y | x) || p(y))` is the KL divergence between the conditional distribution :math:`p(y|x)` - and the margianl distribution :math:`p(y)`. Both the conditional and marginal distribution is calculated + and the marginal distribution :math:`p(y)`. Both the conditional and marginal distribution is calculated from features extracted from the images. The score is calculated on random splits of the images such that both a mean and standard deviation of the score are returned. The metric was originally proposed in `inception ref1`_. @@ -59,7 +59,9 @@ class InceptionScore(Metric): As output of `forward` and `compute` the metric returns the following output - - ``fid`` (:class:`~torch.Tensor`): float scalar tensor with mean FID value over samples + - ``inception_mean`` (:class:`~torch.Tensor`): float scalar tensor with mean inception score over subsets + - ``inception_std`` (:class:`~torch.Tensor`): float scalar tensor with standard deviation of inception score + over subsets Args: feature: From 1d650bf8abcd6baa2a0cc21f82c16457353f583a Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Sun, 24 Mar 2024 16:12:43 +0100 Subject: [PATCH 20/20] fixes --- tests/unittests/detection/test_modified_panoptic_quality.py | 6 ++++++ tests/unittests/detection/test_panoptic_quality.py | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/tests/unittests/detection/test_modified_panoptic_quality.py b/tests/unittests/detection/test_modified_panoptic_quality.py index 4c864d0e9af..1d5a067a609 100644 --- a/tests/unittests/detection/test_modified_panoptic_quality.py +++ b/tests/unittests/detection/test_modified_panoptic_quality.py @@ -18,6 +18,7 @@ import torch from torchmetrics.detection import ModifiedPanopticQuality from torchmetrics.functional.detection import modified_panoptic_quality +from torchmetrics.utilities.imports import _TORCH_GREATER_EQUAL_1_12 from unittests import _Input from unittests._helpers import seed_all @@ -76,6 +77,7 @@ def _reference_fn_1_2(preds, target) -> np.ndarray: return np.array([23 / 30]) +@pytest.mark.skipif(not _TORCH_GREATER_EQUAL_1_12, reason="PanopticQuality metric only supports PyTorch >= 1.12") class TestModifiedPanopticQuality(MetricTester): """Test class for `ModifiedPanopticQuality` metric.""" @@ -111,6 +113,7 @@ def test_panoptic_quality_functional(self): ) +@pytest.mark.skipif(not _TORCH_GREATER_EQUAL_1_12, reason="PanopticQuality metric only supports PyTorch >= 1.12") def test_empty_metric(): """Test empty metric.""" with pytest.raises(ValueError, match="At least one of `things` and `stuffs` must be non-empty"): @@ -120,6 +123,7 @@ def test_empty_metric(): assert torch.isnan(metric.compute()) +@pytest.mark.skipif(not _TORCH_GREATER_EQUAL_1_12, reason="PanopticQuality metric only supports PyTorch >= 1.12") def test_error_on_wrong_input(): """Test class input validation.""" with pytest.raises(TypeError, match="Expected argument `stuffs` to contain `int` categories.*"): @@ -162,6 +166,7 @@ def test_error_on_wrong_input(): metric.update(preds, preds) +@pytest.mark.skipif(not _TORCH_GREATER_EQUAL_1_12, reason="PanopticQuality metric only supports PyTorch >= 1.12") def test_extreme_values(): """Test that the metric returns expected values in trivial cases.""" # Exact match between preds and target => metric is 1 @@ -170,6 +175,7 @@ def test_extreme_values(): assert modified_panoptic_quality(_INPUTS_0.target[0], _INPUTS_0.target[0] + 1, **_ARGS_0) == 0.0 +@pytest.mark.skipif(not _TORCH_GREATER_EQUAL_1_12, reason="PanopticQuality metric only supports PyTorch >= 1.12") @pytest.mark.parametrize( ("inputs", "args", "cat_dim"), [ diff --git a/tests/unittests/detection/test_panoptic_quality.py b/tests/unittests/detection/test_panoptic_quality.py index efa934cd5ab..4d087073266 100644 --- a/tests/unittests/detection/test_panoptic_quality.py +++ b/tests/unittests/detection/test_panoptic_quality.py @@ -120,6 +120,7 @@ def test_panoptic_quality_functional(self): ) +@pytest.mark.skipif(not _TORCH_GREATER_EQUAL_1_12, reason="PanopticQuality metric only supports PyTorch >= 1.12") def test_empty_metric(): """Test empty metric.""" with pytest.raises(ValueError, match="At least one of `things` and `stuffs` must be non-empty"): @@ -129,6 +130,7 @@ def test_empty_metric(): assert torch.isnan(metric.compute()) +@pytest.mark.skipif(not _TORCH_GREATER_EQUAL_1_12, reason="PanopticQuality metric only supports PyTorch >= 1.12") def test_error_on_wrong_input(): """Test class input validation.""" with pytest.raises(TypeError, match="Expected argument `stuffs` to contain `int` categories.*"): @@ -171,6 +173,7 @@ def test_error_on_wrong_input(): metric.update(preds, preds) +@pytest.mark.skipif(not _TORCH_GREATER_EQUAL_1_12, reason="PanopticQuality metric only supports PyTorch >= 1.12") def test_extreme_values(): """Test that the metric returns expected values in trivial cases.""" # Exact match between preds and target => metric is 1 @@ -179,6 +182,7 @@ def test_extreme_values(): assert panoptic_quality(_INPUTS_0.target[0], _INPUTS_0.target[0] + 1, **_ARGS_0) == 0.0 +@pytest.mark.skipif(not _TORCH_GREATER_EQUAL_1_12, reason="PanopticQuality metric only supports PyTorch >= 1.12") @pytest.mark.parametrize( ("inputs", "args", "cat_dim"), [