From 983c030326eebc4439d15c192bd6cec51ab271a7 Mon Sep 17 00:00:00 2001 From: Rohit Gupta Date: Sun, 9 Aug 2020 15:33:24 +0530 Subject: [PATCH] fix reduction docstring and clean tests (#2885) * fix reduction docstring * Update docstring and some cleanup * miss * suggestion from code review Co-authored-by: Ananya Harsh Jha Co-authored-by: Ananya Harsh Jha --- pytorch_lightning/metrics/classification.py | 55 ++++++++----------- pytorch_lightning/metrics/converters.py | 2 +- .../metrics/functional/classification.py | 21 +++---- pytorch_lightning/metrics/functional/nlp.py | 23 ++++++-- .../metrics/functional/regression.py | 27 ++++----- pytorch_lightning/metrics/regression.py | 34 ++++++------ pytorch_lightning/metrics/sklearns.py | 2 +- tests/metrics/test_converters.py | 2 - 8 files changed, 85 insertions(+), 81 deletions(-) diff --git a/pytorch_lightning/metrics/classification.py b/pytorch_lightning/metrics/classification.py index aad3c60183400..4a8f1d00c0bfa 100644 --- a/pytorch_lightning/metrics/classification.py +++ b/pytorch_lightning/metrics/classification.py @@ -4,21 +4,21 @@ from pytorch_lightning.metrics.functional.classification import ( accuracy, - confusion_matrix, - precision_recall_curve, - precision, - recall, - average_precision, auroc, - fbeta_score, - f1_score, - roc, - multiclass_roc, - multiclass_precision_recall_curve, + average_precision, + confusion_matrix, dice_score, + f1_score, + fbeta_score, iou, + multiclass_precision_recall_curve, + multiclass_roc, + precision, + precision_recall_curve, + recall, + roc ) -from pytorch_lightning.metrics.metric import TensorMetric, TensorCollectionMetric +from pytorch_lightning.metrics.metric import TensorCollectionMetric, TensorMetric class Accuracy(TensorMetric): @@ -45,7 +45,7 @@ def __init__( """ Args: num_classes: number of classes - reduction: a method for reducing accuracies over labels (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean - none: pass array @@ -208,7 +208,7 @@ def __init__( """ Args: num_classes: number of classes - reduction: a method for reducing accuracies over labels (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean - none: pass array @@ -262,7 +262,7 @@ def __init__( """ Args: num_classes: number of classes - reduction: a method for reducing accuracies over labels (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean - none: pass array @@ -428,7 +428,7 @@ def __init__( Args: beta: determines the weight of recall in the combined score. num_classes: number of classes - reduction: a method for reducing accuracies over labels (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean - none: pass array @@ -484,7 +484,7 @@ def __init__( """ Args: num_classes: number of classes - reduction: a method for reducing accuracies over labels (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean - none: pass array @@ -605,11 +605,6 @@ def __init__( """ Args: num_classes: number of classes - reduction: a method for reducing accuracies over labels (default: takes the mean) - Available reduction methods: - - elementwise_mean: takes the mean - - none: pass array - - sum: add elements reduce_group: the process group to reduce metric results from DDP reduce_op: the operation to perform for ddp reduction """ @@ -669,11 +664,6 @@ def __init__( """ Args: num_classes: number of classes - reduction: a method for reducing accuracies over labels (default: takes the mean) - Available reduction methods: - - elementwise_mean: takes the mean - - none: pass array - - sum: add elements reduce_group: the process group to reduce metric results from DDP reduce_op: the operation to perform for ddp reduction @@ -737,7 +727,7 @@ def __init__( include_background: whether to also compute dice for the background nan_score: score to return, if a NaN occurs during computation (denom zero) no_fg_score: score to return, if no foreground pixel was found in target - reduction: a method for reducing accuracies over labels (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean - none: pass array @@ -790,16 +780,19 @@ class IoU(TensorMetric): tensor(0.7045) """ - def __init__(self, - remove_bg: bool = False, - reduction: str = 'elementwise_mean'): + + def __init__( + self, + remove_bg: bool = False, + reduction: str = 'elementwise_mean' + ): """ Args: remove_bg: Flag to state whether a background class has been included within input parameters. If true, will remove background class. If false, return IoU over all classes. Assumes that background is '0' class in input tensor - reduction: a method for reducing IoU over labels (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean diff --git a/pytorch_lightning/metrics/converters.py b/pytorch_lightning/metrics/converters.py index c58a3f55f0dde..b0dbd9798b888 100644 --- a/pytorch_lightning/metrics/converters.py +++ b/pytorch_lightning/metrics/converters.py @@ -5,7 +5,7 @@ """ import numbers -from typing import Union, Any, Callable, Optional +from typing import Any, Callable, Optional, Union import numpy as np import torch diff --git a/pytorch_lightning/metrics/functional/classification.py b/pytorch_lightning/metrics/functional/classification.py index 0a77dd6b67682..c80bc5599a4de 100644 --- a/pytorch_lightning/metrics/functional/classification.py +++ b/pytorch_lightning/metrics/functional/classification.py @@ -5,7 +5,7 @@ from torch.nn import functional as F from pytorch_lightning.metrics.functional.reduction import reduce -from pytorch_lightning.utilities import rank_zero_warn, FLOAT16_EPSILON +from pytorch_lightning.utilities import FLOAT16_EPSILON, rank_zero_warn def to_onehot( @@ -149,7 +149,7 @@ def stat_scores_multiple_classes( num_classes: number of classes if known argmax_dim: if pred is a tensor of probabilities, this indicates the axis the argmax transformation will be applied over - reduction: method for reducing result values (default: none) + reduction: a method to reduce metric score over labels (default: none) Available reduction methods: - elementwise_mean: takes the mean @@ -174,6 +174,7 @@ def stat_scores_multiple_classes( tensor([1., 0., 0., 0.]) >>> sups tensor([1., 0., 1., 1.]) + """ if pred.ndim == target.ndim + 1: pred = to_categorical(pred, argmax_dim=argmax_dim) @@ -247,7 +248,7 @@ def accuracy( pred: predicted labels target: ground truth labels num_classes: number of classes - reduction: a method for reducing accuracies over labels (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean @@ -327,7 +328,7 @@ def precision_recall( pred: estimated probabilities target: ground-truth labels num_classes: number of classes - reduction: method for reducing precision-recall values (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean @@ -376,7 +377,7 @@ def precision( pred: estimated probabilities target: ground-truth labels num_classes: number of classes - reduction: method for reducing precision values (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean @@ -411,7 +412,7 @@ def recall( pred: estimated probabilities target: ground-truth labels num_classes: number of classes - reduction: method for reducing recall values (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean @@ -452,7 +453,7 @@ def fbeta_score( beta = 0: only precision beta -> inf: only recall num_classes: number of classes - reduction: method for reducing F-score (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean @@ -497,7 +498,7 @@ def f1_score( pred: estimated probabilities target: ground-truth labels num_classes: number of classes - reduction: method for reducing F1-score (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean @@ -920,7 +921,7 @@ def dice_score( bg: whether to also compute dice for the background nan_score: score to return, if a NaN occurs during computation no_fg_score: score to return, if no foreground pixel was found in target - reduction: a method for reducing accuracies over labels (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean @@ -977,7 +978,7 @@ def iou( within input parameters. If true, will remove background class. If false, return IoU over all classes Assumes that background is '0' class in input tensor - reduction: a method for reducing IoU over labels (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean diff --git a/pytorch_lightning/metrics/functional/nlp.py b/pytorch_lightning/metrics/functional/nlp.py index 22645bb5494b6..85c33642704cd 100644 --- a/pytorch_lightning/metrics/functional/nlp.py +++ b/pytorch_lightning/metrics/functional/nlp.py @@ -4,13 +4,14 @@ # Date: 2020-07-18 # Link: https://pytorch.org/text/_modules/torchtext/data/metrics.html#bleu_score from collections import Counter -from typing import Sequence, List +from typing import List, Sequence import torch def _count_ngram(ngram_input_list: List[str], n_gram: int) -> Counter: - """Counting how many times each word appears in a given text with ngram + """ + Counting how many times each word appears in a given text with ngram Args: ngram_input_list: A list of translated text or reference texts @@ -24,16 +25,20 @@ def _count_ngram(ngram_input_list: List[str], n_gram: int) -> Counter: for i in range(1, n_gram + 1): for j in range(len(ngram_input_list) - i + 1): - ngram_key = tuple(ngram_input_list[j : i + j]) + ngram_key = tuple(ngram_input_list[j:(i + j)]) ngram_counter[ngram_key] += 1 return ngram_counter def bleu_score( - translate_corpus: Sequence[str], reference_corpus: Sequence[str], n_gram: int = 4, smooth: bool = False + translate_corpus: Sequence[str], + reference_corpus: Sequence[str], + n_gram: int = 4, + smooth: bool = False ) -> torch.Tensor: - """Calculate BLEU score of machine translated text with one or more references. + """ + Calculate BLEU score of machine translated text with one or more references Args: translate_corpus: An iterable of machine translated corpus @@ -42,7 +47,7 @@ def bleu_score( smooth: Whether or not to apply smoothing – Lin et al. 2004 Return: - A Tensor with BLEU Score + Tensor with BLEU Score Example: @@ -50,6 +55,7 @@ def bleu_score( >>> reference_corpus = [['there is a cat on the mat'.split(), 'a cat is on the mat'.split()]] >>> bleu_score(translate_corpus, reference_corpus) tensor(0.7598) + """ assert len(translate_corpus) == len(reference_corpus) @@ -58,6 +64,7 @@ def bleu_score( precision_scores = torch.zeros(n_gram) c = 0.0 r = 0.0 + for (translation, references) in zip(translate_corpus, reference_corpus): c += len(translation) ref_len_list = [len(ref) for ref in references] @@ -65,10 +72,12 @@ def bleu_score( r += ref_len_list[ref_len_diff.index(min(ref_len_diff))] translation_counter = _count_ngram(translation, n_gram) reference_counter = Counter() + for ref in references: reference_counter |= _count_ngram(ref, n_gram) ngram_counter_clip = translation_counter & reference_counter + for counter_clip in ngram_counter_clip: numerator[len(counter_clip) - 1] += ngram_counter_clip[counter_clip] @@ -77,6 +86,7 @@ def bleu_score( trans_len = torch.tensor(c) ref_len = torch.tensor(r) + if min(numerator) == 0.0: return torch.tensor(0.0) @@ -84,6 +94,7 @@ def bleu_score( precision_scores = torch.add(numerator, torch.ones(n_gram)) / torch.add(denominator, torch.ones(n_gram)) else: precision_scores = numerator / denominator + log_precision_scores = torch.tensor([1.0 / n_gram] * n_gram) * torch.log(precision_scores) geometric_mean = torch.exp(torch.sum(log_precision_scores)) brevity_penalty = torch.tensor(1.0) if c > r else torch.exp(1 - (ref_len / trans_len)) diff --git a/pytorch_lightning/metrics/functional/regression.py b/pytorch_lightning/metrics/functional/regression.py index 84f50d37270ca..deab2bb4cb702 100644 --- a/pytorch_lightning/metrics/functional/regression.py +++ b/pytorch_lightning/metrics/functional/regression.py @@ -17,7 +17,7 @@ def mse( Args: pred: estimated labels target: ground truth labels - reduction: method for reducing mse (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean @@ -51,7 +51,7 @@ def rmse( Args: pred: estimated labels target: ground truth labels - reduction: method for reducing rmse (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean @@ -83,7 +83,7 @@ def mae( Args: pred: estimated labels target: ground truth labels - reduction: method for reducing mae (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean @@ -117,7 +117,7 @@ def rmsle( Args: pred: estimated labels target: ground truth labels - reduction: method for reducing rmsle (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean @@ -154,7 +154,7 @@ def psnr( target: groun truth signal data_range: the range of the data. If None, it is determined from the data (max - min) base: a base of a logarithm to use (default: 10) - reduction: method for reducing psnr (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean @@ -215,13 +215,13 @@ def ssim( Computes Structual Similarity Index Measure Args: - pred: Estimated image - target: Ground truth image - kernel_size: Size of the gaussian kernel. Default: (11, 11) - sigma: Standard deviation of the gaussian kernel. Default: (1.5, 1.5) - reduction: A method for reducing ssim over all elements in the ``pred`` tensor. Default: ``elementwise_mean`` - + pred: estimated image + target: ground truth image + kernel_size: size of the gaussian kernel (default: (11, 11)) + sigma: Standard deviation of the gaussian kernel (default: (1.5, 1.5)) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: + - elementwise_mean: takes the mean - none: pass away - sum: add elements @@ -230,8 +230,8 @@ def ssim( k1: Parameter of SSIM. Default: 0.01 k2: Parameter of SSIM. Default: 0.03 - Returns: - A Tensor with SSIM + Return: + Tensor with SSIM score Example: @@ -239,6 +239,7 @@ def ssim( >>> target = pred * 0.75 >>> ssim(pred, target) tensor(0.9219) + """ if pred.dtype != target.dtype: diff --git a/pytorch_lightning/metrics/regression.py b/pytorch_lightning/metrics/regression.py index 1edb388fe293d..de4308f73aa9b 100644 --- a/pytorch_lightning/metrics/regression.py +++ b/pytorch_lightning/metrics/regression.py @@ -33,7 +33,7 @@ def __init__( ): """ Args: - reduction: a method for reducing mse over labels (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean - none: pass array @@ -76,7 +76,7 @@ def __init__( ): """ Args: - reduction: a method for reducing mse over labels (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean - none: pass array @@ -119,7 +119,7 @@ def __init__( ): """ Args: - reduction: a method for reducing mse over labels (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean - none: pass array @@ -162,7 +162,7 @@ def __init__( ): """ Args: - reduction: a method for reducing mse over labels (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean - none: pass array @@ -209,7 +209,7 @@ def __init__( Args: data_range: the range of the data. If None, it is determined from the data (max - min) base: a base of a logarithm to use (default: 10) - reduction: method for reducing psnr (default: takes the mean) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean - none: pass array @@ -245,23 +245,23 @@ class SSIM(Metric): >>> metric = SSIM() >>> metric(pred, target) tensor(0.9219) + """ def __init__( - self, - kernel_size: Sequence[int] = (11, 11), - sigma: Sequence[float] = (1.5, 1.5), - reduction: str = "elementwise_mean", - data_range: float = None, - k1: float = 0.01, - k2: float = 0.03 + self, + kernel_size: Sequence[int] = (11, 11), + sigma: Sequence[float] = (1.5, 1.5), + reduction: str = "elementwise_mean", + data_range: float = None, + k1: float = 0.01, + k2: float = 0.03 ): """ Args: - kernel_size: Size of the gaussian kernel. Default: (11, 11) - sigma: Standard deviation of the gaussian kernel. Default: (1.5, 1.5) - reduction: A method for reducing ssim. Default: ``elementwise_mean`` - + kernel_size: Size of the gaussian kernel (default: (11, 11)) + sigma: Standard deviation of the gaussian kernel (default: (1.5, 1.5)) + reduction: a method to reduce metric score over labels (default: takes the mean) Available reduction methods: - elementwise_mean: takes the mean - none: pass away @@ -288,6 +288,6 @@ def forward(self, pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor: target: Ground truth image Return: - torch.Tensor: SSIM Score + A Tensor with SSIM score. """ return ssim(pred, target, self.kernel_size, self.sigma, self.reduction, self.data_range, self.k1, self.k2) diff --git a/pytorch_lightning/metrics/sklearns.py b/pytorch_lightning/metrics/sklearns.py index df8f826f69eb9..2947dcbb585db 100644 --- a/pytorch_lightning/metrics/sklearns.py +++ b/pytorch_lightning/metrics/sklearns.py @@ -1,4 +1,4 @@ -from typing import Any, Optional, Union, Sequence, List +from typing import Any, List, Optional, Sequence, Union import numpy as np import torch diff --git a/tests/metrics/test_converters.py b/tests/metrics/test_converters.py index 1d1412dd8171a..e37dfa9c7c5a5 100644 --- a/tests/metrics/test_converters.py +++ b/tests/metrics/test_converters.py @@ -126,8 +126,6 @@ def _ddp_test_fn(rank, worldsize, add_offset: bool, reduction_mean=False): reduced_tensor = _sync_ddp_if_available(tensor, reduce_op='avg') manual_reduction = sum([i for i in range(dist.get_world_size())]) / dist.get_world_size() - print(reduced_tensor) - print(manual_reduction) assert reduced_tensor.item() == manual_reduction else: reduced_tensor = _sync_ddp_if_available(tensor)