From 1e501f0f40c873369f6d47214e2471c53311443a Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Fri, 11 Dec 2020 22:56:19 +0100 Subject: [PATCH 1/2] add back compatibility for deprecated metrics 2/n (#5068) * add back compatibility for deprecated metrics * fix * imports * imports --- .../metrics/classification/__init__.py | 2 +- .../metrics/classification/f_beta.py | 29 ++++++++++++ .../metrics/functional/__init__.py | 2 + .../metrics/functional/classification.py | 46 +++++++++++++++++++ tests/test_deprecated.py | 14 ++++++ 5 files changed, 92 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/metrics/classification/__init__.py b/pytorch_lightning/metrics/classification/__init__.py index 13cb705f30b17..b4cbb6b073efe 100644 --- a/pytorch_lightning/metrics/classification/__init__.py +++ b/pytorch_lightning/metrics/classification/__init__.py @@ -14,7 +14,7 @@ from pytorch_lightning.metrics.classification.accuracy import Accuracy from pytorch_lightning.metrics.classification.average_precision import AveragePrecision from pytorch_lightning.metrics.classification.confusion_matrix import ConfusionMatrix -from pytorch_lightning.metrics.classification.f_beta import FBeta, F1 +from pytorch_lightning.metrics.classification.f_beta import FBeta, Fbeta, F1 from pytorch_lightning.metrics.classification.precision_recall import Precision, Recall from pytorch_lightning.metrics.classification.precision_recall_curve import PrecisionRecallCurve from pytorch_lightning.metrics.classification.roc import ROC diff --git a/pytorch_lightning/metrics/classification/f_beta.py b/pytorch_lightning/metrics/classification/f_beta.py index 56cc00f9a5dce..d6147b00463b3 100755 --- a/pytorch_lightning/metrics/classification/f_beta.py +++ b/pytorch_lightning/metrics/classification/f_beta.py @@ -20,6 +20,7 @@ _fbeta_compute ) from pytorch_lightning.metrics.metric import Metric +from pytorch_lightning.utilities import rank_zero_warn class FBeta(Metric): @@ -131,6 +132,34 @@ def compute(self) -> torch.Tensor: self.actual_positives, self.beta, self.average) +# todo: remove in v1.2 +class Fbeta(FBeta): + r""" + Computes `F-score `_ + + .. warning :: Deprecated in favor of :func:`~pytorch_lightning.metrics.classification.f_beta.FBeta` + """ + def __init__( + self, + num_classes: int, + beta: float = 1.0, + threshold: float = 0.5, + average: str = "micro", + multilabel: bool = False, + compute_on_step: bool = True, + dist_sync_on_step: bool = False, + process_group: Optional[Any] = None, + ): + rank_zero_warn( + "This `Fbeta` was deprecated in v1.0.x in favor of" + " `from pytorch_lightning.metrics.classification.f_beta import FBeta`." + " It will be removed in v1.2.0", DeprecationWarning + ) + super().__init__( + num_classes, beta, threshold, average, multilabel, compute_on_step, dist_sync_on_step, process_group + ) + + class F1(FBeta): """ Computes F1 metric. F1 metrics correspond to a harmonic mean of the diff --git a/pytorch_lightning/metrics/functional/__init__.py b/pytorch_lightning/metrics/functional/__init__.py index fe9c93525aa69..e38ab5f415c32 100644 --- a/pytorch_lightning/metrics/functional/__init__.py +++ b/pytorch_lightning/metrics/functional/__init__.py @@ -17,6 +17,8 @@ auc, auroc, dice_score, + f1_score, + fbeta_score, get_num_classes, iou, multiclass_auroc, diff --git a/pytorch_lightning/metrics/functional/classification.py b/pytorch_lightning/metrics/functional/classification.py index cc4318ac63969..e1ba601b51553 100644 --- a/pytorch_lightning/metrics/functional/classification.py +++ b/pytorch_lightning/metrics/functional/classification.py @@ -17,6 +17,7 @@ import torch from pytorch_lightning.metrics.functional.average_precision import average_precision as __ap +from pytorch_lightning.metrics.functional.f_beta import fbeta as __fb, f1 as __f1 from pytorch_lightning.metrics.functional.precision_recall_curve import _binary_clf_curve, precision_recall_curve as __prc from pytorch_lightning.metrics.functional.roc import roc as __roc from pytorch_lightning.metrics.utils import ( @@ -871,3 +872,48 @@ def average_precision( " It will be removed in v1.3.0", DeprecationWarning ) return __ap(preds=pred, target=target, sample_weights=sample_weight, pos_label=pos_label) + + +# todo: remove in 1.2 +def fbeta_score( + pred: torch.Tensor, + target: torch.Tensor, + beta: float, + num_classes: Optional[int] = None, + class_reduction: str = 'micro', +) -> torch.Tensor: + """ + Computes the F-beta score which is a weighted harmonic mean of precision and recall. + + .. warning :: Deprecated in favor of :func:`~pytorch_lightning.metrics.functional.f_beta.fbeta` + """ + rank_zero_warn( + "This `average_precision` was deprecated in v1.0.x in favor of" + " `from pytorch_lightning.metrics.functional.f_beta import fbeta`." + " It will be removed in v1.2.0", DeprecationWarning + ) + if num_classes is None: + num_classes = get_num_classes(pred, target) + return __fb(preds=pred, target=target, beta=beta, num_classes=num_classes, average=class_reduction) + + +# todo: remove in 1.2 +def f1_score( + pred: torch.Tensor, + target: torch.Tensor, + num_classes: Optional[int] = None, + class_reduction: str = 'micro', +) -> torch.Tensor: + """ + Computes the F1-score (a.k.a F-measure), which is the harmonic mean of the precision and recall. + + .. warning :: Deprecated in favor of :func:`~pytorch_lightning.metrics.functional.f_beta.f1` + """ + rank_zero_warn( + "This `average_precision` was deprecated in v1.0.x in favor of" + " `from pytorch_lightning.metrics.functional.f_beta import f1`." + " It will be removed in v1.2.0", DeprecationWarning + ) + if num_classes is None: + num_classes = get_num_classes(pred, target) + return __f1(preds=pred, target=target, num_classes=num_classes, average=class_reduction) diff --git a/tests/test_deprecated.py b/tests/test_deprecated.py index 58384fed0cd4f..59c6728009b6f 100644 --- a/tests/test_deprecated.py +++ b/tests/test_deprecated.py @@ -119,6 +119,20 @@ def test_tbd_remove_in_v1_2_0(): checkpoint_cb = ModelCheckpoint(filepath='.', dirpath='.') +def test_tbd_remove_in_v1_2_0_metrics(): + from pytorch_lightning.metrics.classification import Fbeta + from pytorch_lightning.metrics.functional.classification import f1_score, fbeta_score + + with pytest.deprecated_call(match='will be removed in v1.2'): + Fbeta(2) + + with pytest.deprecated_call(match='will be removed in v1.2'): + fbeta_score(torch.tensor([0, 1, 2, 3]), torch.tensor([0, 1, 2, 1]), 0.2) + + with pytest.deprecated_call(match='will be removed in v1.2'): + f1_score(torch.tensor([0, 1, 0, 1]), torch.tensor([0, 1, 0, 0])) + + # TODO: remove bool from Trainer.profiler param in v1.3.0, update profiler_connector.py @pytest.mark.parametrize(['profiler', 'expected'], [ (True, SimpleProfiler), From 63fb7f9510567142ff8ffd57eae6c8bff06c99cb Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Sat, 12 Dec 2020 00:17:19 +0100 Subject: [PATCH 2/2] CI: upload report only on failer (#5086) * CI: upload report only on failer * Apply suggestions from code review Co-authored-by: chaton * Apply suggestions from code review Co-authored-by: chaton Co-authored-by: Roger Shieh Co-authored-by: Rohit Gupta --- .github/workflows/ci_test-base.yml | 3 +-- .github/workflows/ci_test-conda.yml | 3 +-- .github/workflows/ci_test-full.yml | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_test-base.yml b/.github/workflows/ci_test-base.yml index afd86ca98c213..c0b97439737ff 100644 --- a/.github/workflows/ci_test-base.yml +++ b/.github/workflows/ci_test-base.yml @@ -76,8 +76,7 @@ jobs: with: name: pytest-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }} path: junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml - # Use always() to always run this step to publish test results when there are test failures - if: always() + if: failure() - name: Statistics if: success() diff --git a/.github/workflows/ci_test-conda.yml b/.github/workflows/ci_test-conda.yml index 7fc4de8ddbfd3..d64fedbfbe590 100644 --- a/.github/workflows/ci_test-conda.yml +++ b/.github/workflows/ci_test-conda.yml @@ -50,5 +50,4 @@ jobs: with: name: pytest-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }} path: junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml - # Use always() to always run this step to publish test results when there are test failures - if: always() + if: failure() diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 1a2115a40fcfd..b87a1d8557843 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -129,8 +129,7 @@ jobs: with: name: pytest-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }} path: junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml - # Use always() to always run this step to publish test results when there are test failures - if: always() + if: failure() - name: Statistics if: success()