From 642289a68a31e564908cb40bc4ada8d22d8c5169 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 23 Mar 2021 22:42:32 +0100 Subject: [PATCH 1/9] classif --- .../metrics/functional/classification.py | 300 ++---------------- pytorch_lightning/metrics/metric.py | 24 +- pytorch_lightning/metrics/utils.py | 31 +- tests/deprecated_api/test_remove_1-4.py | 8 +- 4 files changed, 38 insertions(+), 325 deletions(-) diff --git a/pytorch_lightning/metrics/functional/classification.py b/pytorch_lightning/metrics/functional/classification.py index f145b7c1a5c67..a46d6fb7d3bbe 100644 --- a/pytorch_lightning/metrics/functional/classification.py +++ b/pytorch_lightning/metrics/functional/classification.py @@ -31,26 +31,8 @@ def stat_scores( argmax_dim: int = 1, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """ - Calculates the number of true positive, false positive, true negative - and false negative for a specific class - - Args: - pred: prediction tensor - target: target tensor - class_index: class to calculate over - argmax_dim: if pred is a tensor of probabilities, this indicates the - axis the argmax transformation will be applied over - - Return: - True Positive, False Positive, True Negative, False Negative, Support - - Example: - - >>> x = torch.tensor([1, 2, 3]) - >>> y = torch.tensor([0, 2, 3]) - >>> tp, fp, tn, fn, sup = stat_scores(x, y, class_index=1) - >>> tp, fp, tn, fn, sup - (tensor(0), tensor(1), tensor(2), tensor(0), tensor(0)) + .. deprecated:: + Use :func:`torchmetrics.functional.stat_scores`. Will be removed in v1.4.0. """ if pred.ndim == target.ndim + 1: pred = to_categorical(pred, argmax_dim=argmax_dim) @@ -73,16 +55,9 @@ def stat_scores_multiple_classes( reduction: str = 'none', ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """ - Calculates the number of true positive, false positive, true negative - and false negative for each class - - .. warning :: Deprecated in favor of :func:`~pytorch_lightning.metrics.functional.stat_scores` - - Raises: - ValueError: - If ``reduction`` is not one of ``"none"``, ``"sum"`` or ``"elementwise_mean"``. + .. deprecated:: + Use :func:`torchmetrics.functional.stat_scores`. Will be removed in v1.4.0. """ - rank_zero_warn( "This `stat_scores_multiple_classes` was deprecated in v1.2.0 in favor of" " `from pytorch_lightning.metrics.functional import stat_scores`." @@ -166,37 +141,8 @@ def precision_recall( return_state: bool = False ) -> Tuple[torch.Tensor, torch.Tensor]: """ - Computes precision and recall for different thresholds - - .. warning :: Deprecated in favor of - :func:`~pytorch_lightning.metrics.functional.precision_recall`. - Will be removed in v1.4.0. - - Args: - pred: estimated probabilities - target: ground-truth labels - num_classes: number of classes - class_reduction: method to reduce metric score over labels - - - ``'micro'``: calculate metrics globally (default) - - ``'macro'``: calculate metrics for each label, and find their unweighted mean. - - ``'weighted'``: calculate metrics for each label, and find their weighted mean. - - ``'none'``: returns calculated metric per class - - return_support: returns the support for each class, need for fbeta/f1 calculations - return_state: returns a internal state that can be ddp reduced - before doing the final calculation - - Return: - Tensor with precision and recall - - Example: - - >>> x = torch.tensor([0, 1, 2, 3]) - >>> y = torch.tensor([0, 2, 2, 2]) - >>> precision_recall(x, y, class_reduction='macro') - (tensor(0.5000), tensor(0.3333)) - + .. deprecated:: + Use :func:`torchmetrics.functional.precision_recall`. Will be removed in v1.4.0. """ rank_zero_warn( "This `precision_recall` was deprecated in v1.2.0 in favor of" @@ -223,32 +169,8 @@ def precision( class_reduction: str = 'micro', ) -> torch.Tensor: """ - Computes precision score. - - .. warning :: Deprecated in favor of - :func:`~pytorch_lightning.metrics.functional.recall`. Will be removed in v1.4.0. - - Args: - pred: estimated probabilities - target: ground-truth labels - num_classes: number of classes - class_reduction: method to reduce metric score over labels - - - ``'micro'``: calculate metrics globally (default) - - ``'macro'``: calculate metrics for each label, and find their unweighted mean. - - ``'weighted'``: calculate metrics for each label, and find their weighted mean. - - ``'none'``: returns calculated metric per class - - Return: - Tensor with precision. - - Example: - - >>> x = torch.tensor([0, 1, 2, 3]) - >>> y = torch.tensor([0, 1, 2, 2]) - >>> precision(x, y) - tensor(0.7500) - + .. deprecated:: + Use :func:`torchmetrics.functional.precision_recall`. Will be removed in v1.4.0. """ rank_zero_warn( "This `precision` was deprecated in v1.2.0 in favor of" @@ -267,31 +189,8 @@ def recall( class_reduction: str = 'micro', ) -> torch.Tensor: """ - Computes recall score. - - .. warning :: Deprecated in favor of - :func:`~pytorch_lightning.metrics.functional.recall`. Will be removed in v1.4.0. - - Args: - pred: estimated probabilities - target: ground-truth labels - num_classes: number of classes - class_reduction: method to reduce metric score over labels - - - ``'micro'``: calculate metrics globally (default) - - ``'macro'``: calculate metrics for each label, and find their unweighted mean. - - ``'weighted'``: calculate metrics for each label, and find their weighted mean. - - ``'none'``: returns calculated metric per class - - Return: - Tensor with recall. - - Example: - - >>> x = torch.tensor([0, 1, 2, 3]) - >>> y = torch.tensor([0, 1, 2, 2]) - >>> recall(x, y) - tensor(0.7500) + .. deprecated:: + Use :func:`torchmetrics.functional.precision_recall`. Will be removed in v1.4.0. """ rank_zero_warn( "This `recall` was deprecated in v1.2.0 in favor of" @@ -308,25 +207,8 @@ def auc( y: torch.Tensor, ) -> torch.Tensor: """ - Computes Area Under the Curve (AUC) using the trapezoidal rule - - .. warning :: Deprecated in favor of - :func:`~pytorch_lightning.metrics.functional.auc.auc`. Will be removed - in v1.4.0. - - Args: - x: x-coordinates - y: y-coordinates - - Return: - Tensor containing AUC score (float) - - Example: - - >>> x = torch.tensor([0, 1, 2, 3]) - >>> y = torch.tensor([0, 1, 2, 2]) - >>> auc(x, y) - tensor(4.) + .. deprecated:: + Use :func:`torchmetrics.functional.auc`. Will be removed in v1.4.0. """ rank_zero_warn( "This `auc` was deprecated in v1.2.0 in favor of" @@ -337,8 +219,8 @@ def auc( # todo: remove in 1.4 -def auc_decorator() -> Callable: - rank_zero_warn("This `auc_decorator` was deprecated in v1.2.0." " It will be removed in v1.4.0", DeprecationWarning) +def _auc_decorator() -> Callable: + rank_zero_warn("This `_auc_decorator` was deprecated in v1.2.0." " It will be removed in v1.4.0", DeprecationWarning) def wrapper(func_to_decorate: Callable) -> Callable: @@ -354,9 +236,9 @@ def new_func(*args, **kwargs) -> torch.Tensor: # todo: remove in 1.4 -def multiclass_auc_decorator() -> Callable: +def _multiclass_auc_decorator() -> Callable: rank_zero_warn( - "This `multiclass_auc_decorator` was deprecated in v1.2.0." + "This `_multiclass_auc_decorator` was deprecated in v1.2.0." " It will be removed in v1.4.0", DeprecationWarning ) @@ -385,29 +267,8 @@ def auroc( max_fpr: float = None, ) -> torch.Tensor: """ - Compute Area Under the Receiver Operating Characteristic Curve (ROC AUC) from prediction scores - - .. warning :: Deprecated in favor of - :func:`~pytorch_lightning.metrics.functional.auroc.auroc`. Will be removed - in v1.4.0. - - Args: - pred: estimated probabilities - target: ground-truth labels - sample_weight: sample weights - pos_label: the label for the positive class - max_fpr: If not ``None``, calculates standardized partial AUC over the - range [0, max_fpr]. Should be a float between 0 and 1. - - Return: - Tensor containing ROCAUC score - - Example: - - >>> x = torch.tensor([0, 1, 2, 3]) - >>> y = torch.tensor([0, 1, 1, 0]) - >>> auroc(x, y) - tensor(0.5000) + .. deprecated:: + Use :func:`torchmetrics.functional.auroc`. Will be removed in v1.4.0. """ rank_zero_warn( "This `auroc` was deprecated in v1.2.0 in favor of" @@ -427,41 +288,8 @@ def multiclass_auroc( num_classes: Optional[int] = None, ) -> torch.Tensor: """ - Compute Area Under the Receiver Operating Characteristic Curve (ROC AUC) from multiclass - prediction scores - - .. warning :: Deprecated in favor of - :func:`~pytorch_lightning.metrics.functional.auroc.auroc`. Will be removed - in v1.4.0. - - Args: - pred: estimated probabilities, with shape [N, C] - target: ground-truth labels, with shape [N,] - sample_weight: sample weights - num_classes: number of classes (default: None, computes automatically from data) - - Return: - Tensor containing ROCAUC score - - Raises: - ValueError: - If ``pred`` don't sum up to ``1`` over classes for ``Multiclass AUROC``. - ValueError: - If number of classes found in ``target`` does not equal the number of - columns in ``pred``. - ValueError: - If number of classes deduced from ``pred`` does not equal the number of - classes passed in ``num_classes``. - - Example: - - >>> pred = torch.tensor([[0.85, 0.05, 0.05, 0.05], - ... [0.05, 0.85, 0.05, 0.05], - ... [0.05, 0.05, 0.85, 0.05], - ... [0.05, 0.05, 0.05, 0.85]]) - >>> target = torch.tensor([0, 1, 3, 2]) - >>> multiclass_auroc(pred, target, num_classes=4) - tensor(0.6667) + .. deprecated:: + Use :func:`torchmetrics.functional.auroc`. Will be removed in v1.4.0. """ rank_zero_warn( "This `multiclass_auroc` was deprecated in v1.2.0 in favor of" @@ -469,26 +297,6 @@ def multiclass_auroc( " It will be removed in v1.4.0", DeprecationWarning ) - if not torch.allclose(pred.sum(dim=1), torch.tensor(1.0)): - raise ValueError( - "Multiclass AUROC metric expects the target scores to be" - " probabilities, i.e. they should sum up to 1.0 over classes" - ) - - if torch.unique(target).size(0) != pred.size(1): - raise ValueError( - f"Number of classes found in in 'target' ({torch.unique(target).size(0)})" - f" does not equal the number of columns in 'pred' ({pred.size(1)})." - " Multiclass AUROC is not defined when all of the classes do not" - " occur in the target labels." - ) - - if num_classes is not None and num_classes != pred.size(1): - raise ValueError( - f"Number of classes deduced from 'pred' ({pred.size(1)}) does not equal" - f" the number of classes passed in 'num_classes' ({num_classes})." - ) - return __auroc(preds=pred, target=target, sample_weights=sample_weight, num_classes=num_classes) @@ -501,34 +309,8 @@ def dice_score( reduction: str = 'elementwise_mean', ) -> torch.Tensor: """ - Compute dice score from prediction scores - - Args: - pred: estimated probabilities - target: ground-truth labels - bg: whether to also compute dice for the background - nan_score: score to return, if a NaN occurs during computation - no_fg_score: score to return, if no foreground pixel was found in target - reduction: a method to reduce metric score over labels. - - - ``'elementwise_mean'``: takes the mean (default) - - ``'sum'``: takes the sum - - ``'none'``: no reduction will be applied - - Return: - Tensor containing dice score - - Example: - - >>> from pytorch_lightning.metrics.functional import dice_score - >>> pred = torch.tensor([[0.85, 0.05, 0.05, 0.05], - ... [0.05, 0.85, 0.05, 0.05], - ... [0.05, 0.05, 0.85, 0.05], - ... [0.05, 0.05, 0.05, 0.85]]) - >>> target = torch.tensor([0, 1, 3, 2]) - >>> dice_score(pred, target) - tensor(0.3333) - + .. deprecated:: + Use :func:`torchmetrics.functional.dice_score`. Will be removed in v1.4.0. """ num_classes = pred.shape[1] bg = (1 - int(bool(bg))) @@ -558,42 +340,8 @@ def iou( reduction: str = 'elementwise_mean', ) -> torch.Tensor: """ - Intersection over union, or Jaccard index calculation. - - .. warning :: Deprecated in favor of - :func:`~pytorch_lightning.metrics.functional.iou.iou`. Will be removed in - v1.4.0. - - Args: - pred: Tensor containing integer predictions, with shape [N, d1, d2, ...] - target: Tensor containing integer targets, with shape [N, d1, d2, ...] - ignore_index: optional int specifying a target class to ignore. If given, this class index does not contribute - to the returned score, regardless of reduction method. Has no effect if given an int that is not in the - range [0, num_classes-1], where num_classes is either given or derived from pred and target. By default, no - index is ignored, and all classes are used. - absent_score: score to use for an individual class, if no instances of the class index were present in - `pred` AND no instances of the class index were present in `target`. For example, if we have 3 classes, - [0, 0] for `pred`, and [0, 2] for `target`, then class 1 would be assigned the `absent_score`. Default is - 0.0. - num_classes: Optionally specify the number of classes - reduction: a method to reduce metric score over labels. - - - ``'elementwise_mean'``: takes the mean (default) - - ``'sum'``: takes the sum - - ``'none'``: no reduction will be applied - - Return: - IoU score : Tensor containing single value if reduction is - 'elementwise_mean', or number of classes if reduction is 'none' - - Example: - - >>> target = torch.randint(0, 2, (10, 25, 25)) - >>> pred = torch.tensor(target) - >>> pred[2:5, 7:13, 9:15] = 1 - pred[2:5, 7:13, 9:15] - >>> iou(pred, target) - tensor(0.9660) - + .. deprecated:: + Use :func:`torchmetrics.functional.iou`. Will be removed in v1.4.0. """ rank_zero_warn( "This `iou` was deprecated in v1.2.0 in favor of" diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py index 918c92049846e..fba3e77c077ed 100644 --- a/pytorch_lightning/metrics/metric.py +++ b/pytorch_lightning/metrics/metric.py @@ -26,6 +26,7 @@ class Metric(_Metric): Use :class:`torchmetrics.Metric`. Will be removed in v1.5.0. """ + @deprecated(target=_Metric, ver_deprecate="1.3.0", ver_remove="1.5.0") def __init__( self, compute_on_step: bool = True, @@ -33,24 +34,17 @@ def __init__( process_group: Optional[Any] = None, dist_sync_fn: Callable = None, ): - rank_zero_warn( - "This `Metric` was deprecated since v1.3.0 in favor of `torchmetrics.Metric`." - " It will be removed in v1.5.0", DeprecationWarning - ) - super(Metric, self).__init__( - compute_on_step=compute_on_step, - dist_sync_on_step=dist_sync_on_step, - process_group=process_group, - dist_sync_fn=dist_sync_fn, - ) + """ + .. deprecated:: + Use :class:`torchmetrics.Metric`. Will be removed in v1.5.0. + """ class MetricCollection(_MetricCollection): - """ - .. deprecated:: - Use :class:`torchmetrics.MetricCollection`. Will be removed in v1.5.0. - """ @deprecated(target=_MetricCollection, ver_deprecate="1.3.0", ver_remove="1.5.0") def __init__(self, metrics: Union[List[Metric], Tuple[Metric], Dict[str, Metric]]): - pass + """ + .. deprecated:: + Use :class:`torchmetrics.MetricCollection`. Will be removed in v1.5.0. + """ diff --git a/pytorch_lightning/metrics/utils.py b/pytorch_lightning/metrics/utils.py index b758e317c6c8d..cf7e82fc36ad8 100644 --- a/pytorch_lightning/metrics/utils.py +++ b/pytorch_lightning/metrics/utils.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Optional +from typing import Optional import torch from torchmetrics.utilities.data import dim_zero_cat as _dim_zero_cat @@ -42,35 +42,6 @@ def dim_zero_mean(x): pass -def get_group_indexes(idx: torch.Tensor) -> List[torch.Tensor]: - """ - Given an integer `torch.Tensor` `idx`, return a `torch.Tensor` of indexes for - each different value in `idx`. - - Args: - idx: a `torch.Tensor` of integers - - Return: - A list of integer `torch.Tensor`s - - Example: - - >>> indexes = torch.tensor([0, 0, 0, 1, 1, 1, 1]) - >>> groups = get_group_indexes(indexes) - >>> groups - [tensor([0, 1, 2]), tensor([3, 4, 5, 6])] - """ - - indexes = dict() - for i, _id in enumerate(idx): - _id = _id.item() - if _id in indexes: - indexes[_id] += [i] - else: - indexes[_id] = [i] - return [torch.tensor(x, dtype=torch.int64) for x in indexes.values()] - - @deprecated(target=_to_onehot, ver_deprecate="1.3.0", ver_remove="1.5.0") def to_onehot(label_tensor: torch.Tensor, num_classes: Optional[int] = None) -> torch.Tensor: """ diff --git a/tests/deprecated_api/test_remove_1-4.py b/tests/deprecated_api/test_remove_1-4.py index 74bec18e8ddb3..f5ca312888cd4 100644 --- a/tests/deprecated_api/test_remove_1-4.py +++ b/tests/deprecated_api/test_remove_1-4.py @@ -142,13 +142,13 @@ def test_v1_4_0_deprecated_metrics(): with pytest.deprecated_call(match='will be removed in v1.4'): multiclass_auroc(torch.rand(20, 5).softmax(dim=-1), torch.randint(0, 5, (20, )), num_classes=5) - from pytorch_lightning.metrics.functional.classification import auc_decorator + from pytorch_lightning.metrics.functional.classification import _auc_decorator with pytest.deprecated_call(match='will be removed in v1.4'): - auc_decorator() + _auc_decorator() - from pytorch_lightning.metrics.functional.classification import multiclass_auc_decorator + from pytorch_lightning.metrics.functional.classification import _multiclass_auc_decorator with pytest.deprecated_call(match='will be removed in v1.4'): - multiclass_auc_decorator() + _multiclass_auc_decorator() class CustomDDPPlugin(DDPSpawnPlugin): From 54794eb554c680980c3a2286dc1dd24d65680938 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 23 Mar 2021 22:44:28 +0100 Subject: [PATCH 2/9] grad_img --- .../metrics/functional/image_gradients.py | 64 ++----------------- 1 file changed, 5 insertions(+), 59 deletions(-) diff --git a/pytorch_lightning/metrics/functional/image_gradients.py b/pytorch_lightning/metrics/functional/image_gradients.py index 721b814bb6f4c..b65c21613a5a5 100644 --- a/pytorch_lightning/metrics/functional/image_gradients.py +++ b/pytorch_lightning/metrics/functional/image_gradients.py @@ -14,68 +14,14 @@ from typing import Tuple import torch +from torchmetrics.functional import image_gradients as _image_gradients - -def _image_gradients_validate(img: torch.Tensor) -> torch.Tensor: - """ Validates whether img is a 4D torch Tensor """ - - if not isinstance(img, torch.Tensor): - raise TypeError(f"The `img` expects a value of type but got {type(img)}") - if img.ndim != 4: - raise RuntimeError(f"The `img` expects a 4D tensor but got {img.ndim}D tensor") - - -def _compute_image_gradients(img: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: - """ Computes image gradients (dy/dx) for a given image """ - - batch_size, channels, height, width = img.shape - - dy = img[..., 1:, :] - img[..., :-1, :] - dx = img[..., :, 1:] - img[..., :, :-1] - - shapey = [batch_size, channels, 1, width] - dy = torch.cat([dy, torch.zeros(shapey, device=img.device, dtype=img.dtype)], dim=2) - dy = dy.view(img.shape) - - shapex = [batch_size, channels, height, 1] - dx = torch.cat([dx, torch.zeros(shapex, device=img.device, dtype=img.dtype)], dim=3) - dx = dx.view(img.shape) - - return dy, dx +from pytorch_lightning.utilities.deprecation import deprecated +@deprecated(target=_image_gradients, ver_deprecate="1.3.0", ver_remove="1.5.0") def image_gradients(img: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: """ - Computes the `gradients `_ of a given image using finite difference - - Args: - img: An ``(N, C, H, W)`` input tensor where C is the number of image channels - - Return: - Tuple of (dy, dx) with each gradient of shape ``[N, C, H, W]`` - - Raises: - TypeError: - If ``img`` is not of the type . - RuntimeError: - If ``img`` is not a 4D tensor. - - Example: - >>> from pytorch_lightning.metrics.functional import image_gradients - >>> image = torch.arange(0, 1*1*5*5, dtype=torch.float32) - >>> image = torch.reshape(image, (1, 1, 5, 5)) - >>> dy, dx = image_gradients(image) - >>> dy[0, 0, :, :] - tensor([[5., 5., 5., 5., 5.], - [5., 5., 5., 5., 5.], - [5., 5., 5., 5., 5.], - [5., 5., 5., 5., 5.], - [0., 0., 0., 0., 0.]]) - - .. note:: The implementation follows the 1-step finite difference method as followed - by the TF implementation. The values are organized such that the gradient of - [I(x+1, y)-[I(x, y)]] are at the (x, y) location + .. deprecated:: + Use :func:`torchmetrics.functional.image_gradients`. Will be removed in v1.5.0. """ - _image_gradients_validate(img) - - return _compute_image_gradients(img) From d6b8c8a6ea8b71275fc73a0a03e3fc2d4d83a698 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 23 Mar 2021 22:48:19 +0100 Subject: [PATCH 3/9] nlp --- pytorch_lightning/metrics/functional/nlp.py | 86 ++------------------- tests/metrics/test_remove_1-5_metrics.py | 40 ++++++---- 2 files changed, 30 insertions(+), 96 deletions(-) diff --git a/pytorch_lightning/metrics/functional/nlp.py b/pytorch_lightning/metrics/functional/nlp.py index b1466c66112bc..e0b5bd4af9525 100644 --- a/pytorch_lightning/metrics/functional/nlp.py +++ b/pytorch_lightning/metrics/functional/nlp.py @@ -20,30 +20,12 @@ from typing import List, Sequence import torch +from torchmetrics.functional import bleu_score as _bleu_score - -def _count_ngram(ngram_input_list: List[str], n_gram: int) -> Counter: - """ - Counting how many times each word appears in a given text with ngram - - Args: - ngram_input_list: A list of translated text or reference texts - n_gram: gram value ranged 1 to 4 - - Return: - ngram_counter: a collections.Counter object of ngram - """ - - ngram_counter = Counter() - - for i in range(1, n_gram + 1): - for j in range(len(ngram_input_list) - i + 1): - ngram_key = tuple(ngram_input_list[j:(i + j)]) - ngram_counter[ngram_key] += 1 - - return ngram_counter +from pytorch_lightning.utilities.deprecation import deprecated +@deprecated(target=_bleu_score, ver_deprecate="1.3.0", ver_remove="1.5.0") def bleu_score( translate_corpus: Sequence[str], reference_corpus: Sequence[str], @@ -51,64 +33,6 @@ def bleu_score( smooth: bool = False ) -> torch.Tensor: """ - Calculate BLEU score of machine translated text with one or more references - - Args: - translate_corpus: An iterable of machine translated corpus - reference_corpus: An iterable of iterables of reference corpus - n_gram: Gram value ranged from 1 to 4 (Default 4) - smooth: Whether or not to apply smoothing – Lin et al. 2004 - - Return: - Tensor with BLEU Score - - Example: - >>> from pytorch_lightning.metrics.functional import bleu_score - >>> translate_corpus = ['the cat is on the mat'.split()] - >>> reference_corpus = [['there is a cat on the mat'.split(), 'a cat is on the mat'.split()]] - >>> bleu_score(translate_corpus, reference_corpus) - tensor(0.7598) + .. deprecated:: + Use :func:`torchmetrics.functional.bleu_score`. Will be removed in v1.5.0. """ - - assert len(translate_corpus) == len(reference_corpus) - numerator = torch.zeros(n_gram) - denominator = torch.zeros(n_gram) - c = 0.0 - r = 0.0 - - for (translation, references) in zip(translate_corpus, reference_corpus): - c += len(translation) - ref_len_list = [len(ref) for ref in references] - ref_len_diff = [abs(len(translation) - x) for x in ref_len_list] - r += ref_len_list[ref_len_diff.index(min(ref_len_diff))] - translation_counter = _count_ngram(translation, n_gram) - reference_counter = Counter() - - for ref in references: - reference_counter |= _count_ngram(ref, n_gram) - - ngram_counter_clip = translation_counter & reference_counter - - for counter_clip in ngram_counter_clip: - numerator[len(counter_clip) - 1] += ngram_counter_clip[counter_clip] - - for counter in translation_counter: - denominator[len(counter) - 1] += translation_counter[counter] - - trans_len = torch.tensor(c) - ref_len = torch.tensor(r) - - if min(numerator) == 0.0: - return torch.tensor(0.0) - - if smooth: - precision_scores = torch.add(numerator, torch.ones(n_gram)) / torch.add(denominator, torch.ones(n_gram)) - else: - precision_scores = numerator / denominator - - log_precision_scores = torch.tensor([1.0 / n_gram] * n_gram) * torch.log(precision_scores) - geometric_mean = torch.exp(torch.sum(log_precision_scores)) - brevity_penalty = torch.tensor(1.0) if c > r else torch.exp(1 - (ref_len / trans_len)) - bleu = brevity_penalty * geometric_mean - - return bleu diff --git a/tests/metrics/test_remove_1-5_metrics.py b/tests/metrics/test_remove_1-5_metrics.py index 43dd330bcfcbe..82f49d8e1aa59 100644 --- a/tests/metrics/test_remove_1-5_metrics.py +++ b/tests/metrics/test_remove_1-5_metrics.py @@ -61,7 +61,7 @@ recall, roc, ssim, - stat_scores, + stat_scores, bleu_score, ) from pytorch_lightning.metrics.functional.accuracy import accuracy from pytorch_lightning.metrics.functional.mean_relative_error import mean_relative_error @@ -135,9 +135,9 @@ def test_v1_5_metric_auc_auroc(): roc.warned = False with pytest.deprecated_call(match='It will be removed in v1.5.0'): fpr, tpr, thrs = roc(preds, target, pos_label=1) - assert torch.equal(fpr, torch.tensor([0., 0., 0., 0., 1.])) - assert torch.allclose(tpr, torch.tensor([0.0000, 0.3333, 0.6667, 1.0000, 1.0000]), atol=1e-4) - assert torch.equal(thrs, torch.tensor([4, 3, 2, 1, 0])) + assert torch.equal(fpr, torch.tensor([0., 0., 0., 0., 1.])) + assert torch.allclose(tpr, torch.tensor([0.0000, 0.3333, 0.6667, 1.0000, 1.0000]), atol=1e-4) + assert torch.equal(thrs, torch.tensor([4, 3, 2, 1, 0])) preds = torch.tensor([0.13, 0.26, 0.08, 0.19, 0.34]) target = torch.tensor([0, 0, 1, 1, 1]) @@ -180,15 +180,15 @@ def test_v1_5_metric_precision_recall(): precision_recall.warned = False with pytest.deprecated_call(match='It will be removed in v1.5.0'): prec, rc = precision_recall(pred, target) - assert prec == torch.tensor(0.5) - assert rc == torch.tensor(0.5) + assert prec == torch.tensor(0.5) + assert rc == torch.tensor(0.5) precision_recall_curve.warned = False with pytest.deprecated_call(match='It will be removed in v1.5.0'): prec, rc, thrs = precision_recall_curve(pred, target) - assert torch.equal(prec, torch.tensor([1., 1., 1., 1.])) - assert torch.allclose(rc, torch.tensor([1., 0.6667, 0.3333, 0.]), atol=1e-4) - assert torch.equal(thrs, torch.tensor([1, 2, 3])) + assert torch.equal(prec, torch.tensor([1., 1., 1., 1.])) + assert torch.allclose(rc, torch.tensor([1., 0.6667, 0.3333, 0.]), atol=1e-4) + assert torch.equal(thrs, torch.tensor([1, 2, 3])) def test_v1_5_metric_classif_mix(): @@ -251,7 +251,8 @@ def test_v1_5_metric_detect(): preds[2:5, 7:13, 9:15] = 1 - preds[2:5, 7:13, 9:15] iou.warned = False with pytest.deprecated_call(match='It will be removed in v1.5.0'): - assert torch.allclose(iou(preds, target), torch.tensor(0.9660), atol=1e-4) + res = iou(preds, target) + assert torch.allclose(res, torch.tensor(0.9660), atol=1e-4) def test_v1_5_metric_regress(): @@ -276,7 +277,7 @@ def test_v1_5_metric_regress(): explained_variance.warned = False with pytest.deprecated_call(match='It will be removed in v1.5.0'): res = explained_variance(preds, target) - assert torch.allclose(res, torch.tensor(0.9572), atol=1e-4) + assert torch.allclose(res, torch.tensor(0.9572), atol=1e-4) x = torch.tensor([0., 1, 2, 3]) y = torch.tensor([0., 1, 2, 2]) @@ -295,7 +296,7 @@ def test_v1_5_metric_regress(): mean_squared_log_error.warned = False with pytest.deprecated_call(match='It will be removed in v1.5.0'): res = mean_squared_log_error(x, y) - assert torch.allclose(res, torch.tensor(0.0207), atol=1e-4) + assert torch.allclose(res, torch.tensor(0.0207), atol=1e-4) PSNR.__init__.warned = False with pytest.deprecated_call(match='It will be removed in v1.5.0'): @@ -314,18 +315,27 @@ def test_v1_5_metric_regress(): psnr.warned = False with pytest.deprecated_call(match='It will be removed in v1.5.0'): res = psnr(preds, target) - assert torch.allclose(res, torch.tensor(2.5527), atol=1e-4) + assert torch.allclose(res, torch.tensor(2.5527), atol=1e-4) target = torch.tensor([3, -0.5, 2, 7]) preds = torch.tensor([2.5, 0.0, 2, 8]) r2score.warned = False with pytest.deprecated_call(match='It will be removed in v1.5.0'): res = r2score(preds, target) - assert torch.allclose(res, torch.tensor(0.9486), atol=1e-4) + assert torch.allclose(res, torch.tensor(0.9486), atol=1e-4) preds = torch.rand([16, 1, 16, 16]) target = preds * 0.75 ssim.warned = False with pytest.deprecated_call(match='It will be removed in v1.5.0'): res = ssim(preds, target) - assert torch.allclose(res, torch.tensor(0.9219), atol=1e-4) + assert torch.allclose(res, torch.tensor(0.9219), atol=1e-4) + + +def test_v1_5_metric_others(): + translate_corpus = ['the cat is on the mat'.split()] + reference_corpus = [['there is a cat on the mat'.split(), 'a cat is on the mat'.split()]] + bleu_score.warned = False + with pytest.deprecated_call(match='It will be removed in v1.5.0'): + res = bleu_score(translate_corpus, reference_corpus) + assert torch.allclose(res, torch.tensor(0.7598), atol=1e-4) From 02e61f9ba3629d4b8b9ea2e9b18074d58fcac405 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 23 Mar 2021 22:51:33 +0100 Subject: [PATCH 4/9] ssl --- pytorch_lightning/metrics/functional/nlp.py | 3 +- .../metrics/functional/self_supervised.py | 41 +++---------------- tests/metrics/test_remove_1-5_metrics.py | 11 ++++- 3 files changed, 17 insertions(+), 38 deletions(-) diff --git a/pytorch_lightning/metrics/functional/nlp.py b/pytorch_lightning/metrics/functional/nlp.py index e0b5bd4af9525..29feb959f2f18 100644 --- a/pytorch_lightning/metrics/functional/nlp.py +++ b/pytorch_lightning/metrics/functional/nlp.py @@ -16,8 +16,7 @@ # Authors: torchtext authors and @sluks # Date: 2020-07-18 # Link: https://pytorch.org/text/_modules/torchtext/data/metrics.html#bleu_score -from collections import Counter -from typing import List, Sequence +from typing import Sequence import torch from torchmetrics.functional import bleu_score as _bleu_score diff --git a/pytorch_lightning/metrics/functional/self_supervised.py b/pytorch_lightning/metrics/functional/self_supervised.py index ed00677bb32d3..c3dc1cbfad659 100644 --- a/pytorch_lightning/metrics/functional/self_supervised.py +++ b/pytorch_lightning/metrics/functional/self_supervised.py @@ -12,8 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. import torch +from torchmetrics.functional import embedding_similarity as _embedding_similarity +from pytorch_lightning.utilities.deprecation import deprecated + +@deprecated(target=_embedding_similarity, ver_deprecate="1.3.0", ver_remove="1.5.0") def embedding_similarity( batch: torch.Tensor, similarity: str = 'cosine', @@ -21,39 +25,6 @@ def embedding_similarity( zero_diagonal: bool = True ) -> torch.Tensor: """ - Computes representation similarity - - Example: - >>> from pytorch_lightning.metrics.functional import embedding_similarity - >>> embeddings = torch.tensor([[1., 2., 3., 4.], [1., 2., 3., 4.], [4., 5., 6., 7.]]) - >>> embedding_similarity(embeddings) - tensor([[0.0000, 1.0000, 0.9759], - [1.0000, 0.0000, 0.9759], - [0.9759, 0.9759, 0.0000]]) - - Args: - batch: (batch, dim) - similarity: 'dot' or 'cosine' - reduction: 'none', 'sum', 'mean' (all along dim -1) - zero_diagonal: if True, the diagonals are set to zero - - Return: - A square matrix (batch, batch) with the similarity scores between all elements - If sum or mean are used, then returns (b, 1) with the reduced value for each row + .. deprecated:: + Use :func:`torchmetrics.functional.embedding_similarity`. Will be removed in v1.5.0. """ - if similarity == 'cosine': - norm = torch.norm(batch, p=2, dim=1) - batch = batch / norm.unsqueeze(1) - - sqr_mtx = batch.mm(batch.transpose(1, 0)) - - if zero_diagonal: - sqr_mtx = sqr_mtx.fill_diagonal_(0) - - if reduction == 'mean': - sqr_mtx = sqr_mtx.mean(dim=-1) - - if reduction == 'sum': - sqr_mtx = sqr_mtx.sum(dim=-1) - - return sqr_mtx diff --git a/tests/metrics/test_remove_1-5_metrics.py b/tests/metrics/test_remove_1-5_metrics.py index 82f49d8e1aa59..61cbbbae6c9e3 100644 --- a/tests/metrics/test_remove_1-5_metrics.py +++ b/tests/metrics/test_remove_1-5_metrics.py @@ -61,7 +61,7 @@ recall, roc, ssim, - stat_scores, bleu_score, + stat_scores, bleu_score, embedding_similarity, ) from pytorch_lightning.metrics.functional.accuracy import accuracy from pytorch_lightning.metrics.functional.mean_relative_error import mean_relative_error @@ -339,3 +339,12 @@ def test_v1_5_metric_others(): with pytest.deprecated_call(match='It will be removed in v1.5.0'): res = bleu_score(translate_corpus, reference_corpus) assert torch.allclose(res, torch.tensor(0.7598), atol=1e-4) + + embeddings = torch.tensor([[1., 2., 3., 4.], [1., 2., 3., 4.], [4., 5., 6., 7.]]) + embedding_similarity.warned = False + with pytest.deprecated_call(match='It will be removed in v1.5.0'): + res = embedding_similarity(embeddings) + assert torch.allclose(res, torch.tensor([[0.0000, 1.0000, 0.9759], + [1.0000, 0.0000, 0.9759], + [0.9759, 0.9759, 0.0000]]), atol=1e-4) + From fcea2ca27ad6ba36520bcd0a6d53cc53e798be91 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 23 Mar 2021 22:53:39 +0100 Subject: [PATCH 5/9] format --- .../metrics/functional/classification.py | 5 ++++- pytorch_lightning/metrics/metric.py | 1 - tests/deprecated_api/test_remove_1-5.py | 2 +- tests/metrics/test_remove_1-5_metrics.py | 11 ++++++----- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pytorch_lightning/metrics/functional/classification.py b/pytorch_lightning/metrics/functional/classification.py index a46d6fb7d3bbe..9c42b2941d579 100644 --- a/pytorch_lightning/metrics/functional/classification.py +++ b/pytorch_lightning/metrics/functional/classification.py @@ -220,7 +220,10 @@ def auc( # todo: remove in 1.4 def _auc_decorator() -> Callable: - rank_zero_warn("This `_auc_decorator` was deprecated in v1.2.0." " It will be removed in v1.4.0", DeprecationWarning) + rank_zero_warn( + "This `_auc_decorator` was deprecated in v1.2.0." + " It will be removed in v1.4.0", DeprecationWarning + ) def wrapper(func_to_decorate: Callable) -> Callable: diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py index fba3e77c077ed..b76c91dcdf2f1 100644 --- a/pytorch_lightning/metrics/metric.py +++ b/pytorch_lightning/metrics/metric.py @@ -17,7 +17,6 @@ from torchmetrics.collections import MetricCollection as _MetricCollection from pytorch_lightning.utilities.deprecation import deprecated -from pytorch_lightning.utilities.distributed import rank_zero_warn class Metric(_Metric): diff --git a/tests/deprecated_api/test_remove_1-5.py b/tests/deprecated_api/test_remove_1-5.py index 725db1180d9e8..fc3fe3112e71e 100644 --- a/tests/deprecated_api/test_remove_1-5.py +++ b/tests/deprecated_api/test_remove_1-5.py @@ -20,7 +20,7 @@ from pytorch_lightning import Callback, Trainer from pytorch_lightning.callbacks import ModelCheckpoint from pytorch_lightning.loggers import WandbLogger -from pytorch_lightning.profiler import BaseProfiler, SimpleProfiler, AdvancedProfiler, PyTorchProfiler +from pytorch_lightning.profiler import AdvancedProfiler, BaseProfiler, PyTorchProfiler, SimpleProfiler from pytorch_lightning.trainer.callback_hook import warning_cache as callback_warning_cache from tests.deprecated_api import no_deprecated_call from tests.helpers import BoringModel diff --git a/tests/metrics/test_remove_1-5_metrics.py b/tests/metrics/test_remove_1-5_metrics.py index 61cbbbae6c9e3..a5b3f10db24d8 100644 --- a/tests/metrics/test_remove_1-5_metrics.py +++ b/tests/metrics/test_remove_1-5_metrics.py @@ -44,7 +44,9 @@ auc, auroc, average_precision, + bleu_score, confusion_matrix, + embedding_similarity, explained_variance, f1, fbeta, @@ -61,7 +63,7 @@ recall, roc, ssim, - stat_scores, bleu_score, embedding_similarity, + stat_scores, ) from pytorch_lightning.metrics.functional.accuracy import accuracy from pytorch_lightning.metrics.functional.mean_relative_error import mean_relative_error @@ -344,7 +346,6 @@ def test_v1_5_metric_others(): embedding_similarity.warned = False with pytest.deprecated_call(match='It will be removed in v1.5.0'): res = embedding_similarity(embeddings) - assert torch.allclose(res, torch.tensor([[0.0000, 1.0000, 0.9759], - [1.0000, 0.0000, 0.9759], - [0.9759, 0.9759, 0.0000]]), atol=1e-4) - + assert torch.allclose( + res, torch.tensor([[0.0000, 1.0000, 0.9759], [1.0000, 0.0000, 0.9759], [0.9759, 0.9759, 0.0000]]), atol=1e-4 + ) From 27a39a636bf5055d60b4b1d0e9ea48e7c53b87bb Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 23 Mar 2021 22:58:05 +0100 Subject: [PATCH 6/9] chlog --- CHANGELOG.md | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b3359ace54f9..0b4ab2f0a47fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,23 +96,17 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Deprecated metrics in favor of `torchmetrics` ([#6505](https://github.com/PyTorchLightning/pytorch-lightning/pull/6505), - [#6530](https://github.com/PyTorchLightning/pytorch-lightning/pull/6530), - + [#6540](https://github.com/PyTorchLightning/pytorch-lightning/pull/6540), [#6547](https://github.com/PyTorchLightning/pytorch-lightning/pull/6547), - [#6515](https://github.com/PyTorchLightning/pytorch-lightning/pull/6515), - [#6572](https://github.com/PyTorchLightning/pytorch-lightning/pull/6572), - [#6573](https://github.com/PyTorchLightning/pytorch-lightning/pull/6573), - [#6584](https://github.com/PyTorchLightning/pytorch-lightning/pull/6584), - [#6636](https://github.com/PyTorchLightning/pytorch-lightning/pull/6636), - [#6637](https://github.com/PyTorchLightning/pytorch-lightning/pull/6637), - + [#6649](https://github.com/PyTorchLightning/pytorch-lightning/pull/6649), + [#6659](https://github.com/PyTorchLightning/pytorch-lightning/pull/6659), ) From 87edc6651781bd3e66b040c10c7f8f6d10878cdc Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 23 Mar 2021 22:58:36 +0100 Subject: [PATCH 7/9] tests --- tests/metrics/functional/__init__.py | 0 .../metrics/functional/test_classification.py | 15 --- .../functional/test_image_gradients.py | 109 ------------------ tests/metrics/functional/test_nlp.py | 68 ----------- tests/metrics/functional/test_reduction.py | 27 ----- .../functional/test_self_supervised.py | 32 ----- tests/metrics/regression/__init__.py | 0 7 files changed, 251 deletions(-) delete mode 100644 tests/metrics/functional/__init__.py delete mode 100644 tests/metrics/functional/test_classification.py delete mode 100644 tests/metrics/functional/test_image_gradients.py delete mode 100644 tests/metrics/functional/test_nlp.py delete mode 100644 tests/metrics/functional/test_reduction.py delete mode 100644 tests/metrics/functional/test_self_supervised.py delete mode 100644 tests/metrics/regression/__init__.py diff --git a/tests/metrics/functional/__init__.py b/tests/metrics/functional/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/tests/metrics/functional/test_classification.py b/tests/metrics/functional/test_classification.py deleted file mode 100644 index f9f3033427b1b..0000000000000 --- a/tests/metrics/functional/test_classification.py +++ /dev/null @@ -1,15 +0,0 @@ -import pytest -import torch - -from pytorch_lightning.metrics.functional.classification import dice_score - - -@pytest.mark.parametrize(['pred', 'target', 'expected'], [ - pytest.param([[0, 0], [1, 1]], [[0, 0], [1, 1]], 1.), - pytest.param([[1, 1], [0, 0]], [[0, 0], [1, 1]], 0.), - pytest.param([[1, 1], [1, 1]], [[1, 1], [0, 0]], 2 / 3), - pytest.param([[1, 1], [0, 0]], [[1, 1], [0, 0]], 1.), -]) -def test_dice_score(pred, target, expected): - score = dice_score(torch.tensor(pred), torch.tensor(target)) - assert score == expected diff --git a/tests/metrics/functional/test_image_gradients.py b/tests/metrics/functional/test_image_gradients.py deleted file mode 100644 index 2e406793b4370..0000000000000 --- a/tests/metrics/functional/test_image_gradients.py +++ /dev/null @@ -1,109 +0,0 @@ -import pytest -import torch - -from pytorch_lightning.metrics.functional.image_gradients import image_gradients - - -def test_invalid_input_img_type(): - """Test Whether the module successfully handles invalid input data type""" - invalid_dummy_input = [[1, 2, 3], [4, 5, 6], [7, 8, 9]] - - with pytest.raises(TypeError): - image_gradients(invalid_dummy_input) - - -def test_invalid_input_ndims(): - """ - Test whether the module successfully handles invalid number of dimensions - of input tensor - """ - - BATCH_SIZE = 1 - HEIGHT = 5 - WIDTH = 5 - CHANNELS = 1 - - image = torch.arange(0, BATCH_SIZE * HEIGHT * WIDTH * CHANNELS, dtype=torch.float32) - image = torch.reshape(image, (HEIGHT, WIDTH)) - - with pytest.raises(RuntimeError): - image_gradients(image) - - -def test_multi_batch_image_gradients(): - """Test whether the module correctly calculates gradients for known input - with non-unity batch size.Example input-output pair taken from TF's implementation of i - mage-gradients - """ - - BATCH_SIZE = 5 - HEIGHT = 5 - WIDTH = 5 - CHANNELS = 1 - - single_channel_img = torch.arange(0, 1 * HEIGHT * WIDTH * CHANNELS, dtype=torch.float32) - single_channel_img = torch.reshape(single_channel_img, (CHANNELS, HEIGHT, WIDTH)) - image = torch.stack([single_channel_img for _ in range(BATCH_SIZE)], dim=0) - - true_dy = [ - [5., 5., 5., 5., 5.], - [5., 5., 5., 5., 5.], - [5., 5., 5., 5., 5.], - [5., 5., 5., 5., 5.], - [0., 0., 0., 0., 0.], - ] - - true_dx = [ - [1., 1., 1., 1., 0.], - [1., 1., 1., 1., 0.], - [1., 1., 1., 1., 0.], - [1., 1., 1., 1., 0.], - [1., 1., 1., 1., 0.], - ] - true_dy = torch.Tensor(true_dy) - true_dx = torch.Tensor(true_dx) - - dy, dx = image_gradients(image) - - for batch_id in range(BATCH_SIZE): - assert torch.allclose(dy[batch_id, 0, :, :], true_dy) - assert dy.shape == (BATCH_SIZE, 1, HEIGHT, WIDTH) - assert dx.shape == (BATCH_SIZE, 1, HEIGHT, WIDTH) - - -def test_image_gradients(): - """Test whether the module correctly calculates gradients for known input. - Example input-output pair taken from TF's implementation of image-gradients - """ - - BATCH_SIZE = 1 - HEIGHT = 5 - WIDTH = 5 - CHANNELS = 1 - - image = torch.arange(0, BATCH_SIZE * HEIGHT * WIDTH * CHANNELS, dtype=torch.float32) - image = torch.reshape(image, (BATCH_SIZE, CHANNELS, HEIGHT, WIDTH)) - - true_dy = [ - [5., 5., 5., 5., 5.], - [5., 5., 5., 5., 5.], - [5., 5., 5., 5., 5.], - [5., 5., 5., 5., 5.], - [0., 0., 0., 0., 0.], - ] - - true_dx = [ - [1., 1., 1., 1., 0.], - [1., 1., 1., 1., 0.], - [1., 1., 1., 1., 0.], - [1., 1., 1., 1., 0.], - [1., 1., 1., 1., 0.], - ] - - true_dy = torch.Tensor(true_dy) - true_dx = torch.Tensor(true_dx) - - dy, dx = image_gradients(image) - - assert torch.allclose(dy, true_dy), "dy fails test" - assert torch.allclose(dx, true_dx), "dx fails tests" diff --git a/tests/metrics/functional/test_nlp.py b/tests/metrics/functional/test_nlp.py deleted file mode 100644 index b8faadc16085f..0000000000000 --- a/tests/metrics/functional/test_nlp.py +++ /dev/null @@ -1,68 +0,0 @@ -import pytest -import torch -from nltk.translate.bleu_score import corpus_bleu, sentence_bleu, SmoothingFunction - -from pytorch_lightning.metrics.functional.nlp import bleu_score - -# example taken from -# https://www.nltk.org/api/nltk.translate.html?highlight=bleu%20score#nltk.translate.bleu_score.sentence_bleu -HYPOTHESIS1 = tuple( - "It is a guide to action which ensures that the military always obeys the commands of the party".split() -) -REFERENCE1 = tuple("It is a guide to action that ensures that the military will forever heed Party commands".split()) -REFERENCE2 = tuple( - "It is a guiding principle which makes the military forces always being under the command of the Party".split() -) -REFERENCE3 = tuple("It is the practical guide for the army always to heed the directions of the party".split()) - -# example taken from -# https://www.nltk.org/api/nltk.translate.html?highlight=bleu%20score#nltk.translate.bleu_score.corpus_bleu -HYP1 = "It is a guide to action which ensures that the military always obeys the commands of the party".split() -HYP2 = "he read the book because he was interested in world history".split() - -REF1A = "It is a guide to action that ensures that the military will forever heed Party commands".split() -REF1B = "It is a guiding principle which makes the military force always being under the command of the Party".split() -REF1C = "It is the practical guide for the army always to heed the directions of the party".split() -REF2A = "he was interested in world history because he read the book".split() - -LIST_OF_REFERENCES = [[REF1A, REF1B, REF1C], [REF2A]] -HYPOTHESES = [HYP1, HYP2] - -# https://www.nltk.org/api/nltk.translate.html?highlight=bleu%20score#nltk.translate.bleu_score.SmoothingFunction -smooth_func = SmoothingFunction().method2 - - -@pytest.mark.parametrize( - ["weights", "n_gram", "smooth_func", "smooth"], - [ - pytest.param([1], 1, None, False), - pytest.param([0.5, 0.5], 2, smooth_func, True), - pytest.param([0.333333, 0.333333, 0.333333], 3, None, False), - pytest.param([0.25, 0.25, 0.25, 0.25], 4, smooth_func, True), - ], -) -def test_bleu_score(weights, n_gram, smooth_func, smooth): - nltk_output = sentence_bleu( - [REFERENCE1, REFERENCE2, REFERENCE3], - HYPOTHESIS1, - weights=weights, - smoothing_function=smooth_func, - ) - pl_output = bleu_score([HYPOTHESIS1], [[REFERENCE1, REFERENCE2, REFERENCE3]], n_gram=n_gram, smooth=smooth) - assert torch.allclose(pl_output, torch.tensor(nltk_output)) - - nltk_output = corpus_bleu(LIST_OF_REFERENCES, HYPOTHESES, weights=weights, smoothing_function=smooth_func) - pl_output = bleu_score(HYPOTHESES, LIST_OF_REFERENCES, n_gram=n_gram, smooth=smooth) - assert torch.allclose(pl_output, torch.tensor(nltk_output)) - - -def test_bleu_empty(): - hyp = [[]] - ref = [[[]]] - assert bleu_score(hyp, ref) == torch.tensor(0.0) - - -def test_no_4_gram(): - hyps = [["My", "full", "pytorch-lightning"]] - refs = [[["My", "full", "pytorch-lightning", "test"], ["Completely", "Different"]]] - assert bleu_score(hyps, refs) == torch.tensor(0.0) diff --git a/tests/metrics/functional/test_reduction.py b/tests/metrics/functional/test_reduction.py deleted file mode 100644 index 9949c8086a44a..0000000000000 --- a/tests/metrics/functional/test_reduction.py +++ /dev/null @@ -1,27 +0,0 @@ -import pytest -import torch -from torchmetrics.utilities import class_reduce, reduce - - -def test_reduce(): - start_tensor = torch.rand(50, 40, 30) - - assert torch.allclose(reduce(start_tensor, 'elementwise_mean'), torch.mean(start_tensor)) - assert torch.allclose(reduce(start_tensor, 'sum'), torch.sum(start_tensor)) - assert torch.allclose(reduce(start_tensor, 'none'), start_tensor) - - with pytest.raises(ValueError): - reduce(start_tensor, 'error_reduction') - - -def test_class_reduce(): - num = torch.randint(1, 10, (100, )).float() - denom = torch.randint(10, 20, (100, )).float() - weights = torch.randint(1, 100, (100, )).float() - - assert torch.allclose(class_reduce(num, denom, weights, 'micro'), torch.sum(num) / torch.sum(denom)) - assert torch.allclose(class_reduce(num, denom, weights, 'macro'), torch.mean(num / denom)) - assert torch.allclose( - class_reduce(num, denom, weights, 'weighted'), torch.sum(num / denom * (weights / torch.sum(weights))) - ) - assert torch.allclose(class_reduce(num, denom, weights, 'none'), num / denom) diff --git a/tests/metrics/functional/test_self_supervised.py b/tests/metrics/functional/test_self_supervised.py deleted file mode 100644 index fbabc5e93cffc..0000000000000 --- a/tests/metrics/functional/test_self_supervised.py +++ /dev/null @@ -1,32 +0,0 @@ -import pytest -import torch -from sklearn.metrics import pairwise - -from pytorch_lightning.metrics.functional.self_supervised import embedding_similarity - - -@pytest.mark.parametrize('similarity', ['cosine', 'dot']) -@pytest.mark.parametrize('reduction', ['none', 'mean', 'sum']) -def test_against_sklearn(similarity, reduction): - """Compare PL metrics to sklearn version.""" - device = 'cuda' if torch.cuda.is_available() else 'cpu' - - batch = torch.randn(5, 10, device=device) # 100 samples in 10 dimensions - - pl_dist = embedding_similarity(batch, similarity=similarity, reduction=reduction, zero_diagonal=False) - - def sklearn_embedding_distance(batch, similarity, reduction): - - metric_func = {'cosine': pairwise.cosine_similarity, 'dot': pairwise.linear_kernel}[similarity] - - dist = metric_func(batch, batch) - if reduction == 'mean': - return dist.mean(axis=-1) - if reduction == 'sum': - return dist.sum(axis=-1) - return dist - - sk_dist = sklearn_embedding_distance(batch.cpu().detach().numpy(), similarity=similarity, reduction=reduction) - sk_dist = torch.tensor(sk_dist, dtype=torch.float, device=device) - - assert torch.allclose(sk_dist, pl_dist) diff --git a/tests/metrics/regression/__init__.py b/tests/metrics/regression/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 From 57e105e04bc5ab8c1e2adeb2c07c71239f434c49 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 23 Mar 2021 23:07:43 +0100 Subject: [PATCH 8/9] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Carlos Mocholí --- pytorch_lightning/metrics/functional/classification.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/metrics/functional/classification.py b/pytorch_lightning/metrics/functional/classification.py index 9c42b2941d579..90d5c0a66550a 100644 --- a/pytorch_lightning/metrics/functional/classification.py +++ b/pytorch_lightning/metrics/functional/classification.py @@ -170,7 +170,7 @@ def precision( ) -> torch.Tensor: """ .. deprecated:: - Use :func:`torchmetrics.functional.precision_recall`. Will be removed in v1.4.0. + Use :func:`torchmetrics.functional.precision`. Will be removed in v1.4.0. """ rank_zero_warn( "This `precision` was deprecated in v1.2.0 in favor of" @@ -190,7 +190,7 @@ def recall( ) -> torch.Tensor: """ .. deprecated:: - Use :func:`torchmetrics.functional.precision_recall`. Will be removed in v1.4.0. + Use :func:`torchmetrics.functional.recall`. Will be removed in v1.4.0. """ rank_zero_warn( "This `recall` was deprecated in v1.2.0 in favor of" From 53582defaca535c42b38fe6c6faff5793c6d9bf5 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Wed, 24 Mar 2021 01:14:46 +0100 Subject: [PATCH 9/9] ... --- tests/metrics/test_metric_lightning.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/metrics/test_metric_lightning.py b/tests/metrics/test_metric_lightning.py index 2e040a881d49f..00ff2bd6bc889 100644 --- a/tests/metrics/test_metric_lightning.py +++ b/tests/metrics/test_metric_lightning.py @@ -1,11 +1,12 @@ import torch -from torchmetrics import Metric, MetricCollection +from torchmetrics import Metric as TMetric from pytorch_lightning import Trainer +from pytorch_lightning.metrics import Metric as PLMetric, MetricCollection from tests.helpers.boring_model import BoringModel -class SumMetric(Metric): +class SumMetric(TMetric): def __init__(self): super().__init__() @@ -18,7 +19,7 @@ def compute(self): return self.x -class DiffMetric(Metric): +class DiffMetric(PLMetric): def __init__(self): super().__init__()