Merge branch 'master' into refactor/imports

Author: Borda

.azure/gpu-pipeline.yml

@@ -21,9 +21,10 @@ jobs:
     strategy:
       matrix:
         'PyTorch | old':
-          docker-image: 'pytorch/pytorch:1.9.0-cuda11.1-cudnn8-runtime'
+          # Torch does not have build wheels with old Torch versions for newer CUDA
+          docker-image: 'nvidia/cuda:11.1.1-cudnn8-devel-ubuntu20.04'
           agent-pool: 'lit-rtx-3090'
-          torch-ver: '1.9.0'
+          torch-ver: '1.8.1'
         'PyTorch | 1.X':
           docker-image: 'pytorch/pytorch:1.13.1-cuda11.6-cudnn8-runtime'
           agent-pool: 'lit-rtx-3090'
@@ -60,10 +61,33 @@ jobs:
 
     steps:
 
+    - script: |
+        container_id=$(head -1 /proc/self/cgroup|cut -d/ -f3)
+        echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
+        /tmp/docker exec -t -u 0 $container_id \
+          sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
+        echo "##vso[task.setvariable variable=CONTAINER_ID]$container_id"
+      displayName: 'Install Sudo in container (thanks Microsoft!)'
+
+    - script: |
+        sudo apt-get update -q --fix-missing
+        sudo apt-get install -q -y --no-install-recommends \
+          build-essential wget python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-distutils
+        sudo update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1
+        wget https://bootstrap.pypa.io/get-pip.py --progress=bar:force:noscroll --no-check-certificate
+        python get-pip.py
+      env:
+        PYTHON_VERSION: "3.8"
+      condition: startsWith(variables['docker-image'], 'nvidia/cuda:')
+      displayName: 'install python & pip'
+
     - bash: |
-        echo "##vso[task.setvariable variable=CONTAINER_ID]$(head -1 /proc/self/cgroup|cut -d/ -f3)"
         echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)"
-      displayName: 'Set environment variables'
+        CUDA_version=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\.[0-9]\+\).*$/\1/p')
+        CUDA_version_mm="${CUDA_version//'.'/''}"
+        echo "##vso[task.setvariable variable=CUDA_VERSION_MM]$CUDA_version_mm"
+        echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${CUDA_version_mm}/torch_stable.html"
+      displayName: 'set Env. vars'
 
     - bash: |
         whoami && id
@@ -72,23 +96,18 @@ jobs:
         nvidia-smi
         echo $CUDA_VISIBLE_DEVICES
         echo $CONTAINER_ID
+        echo $TORCH_URL
         python --version
         pip --version
         pip cache dir
         pip list
-        python -c "import torch ; print(torch.cuda.get_arch_list())"
       displayName: 'Image info & NVIDIA'
 
-    - script: |
-        /tmp/docker exec -t -u 0 $CONTAINER_ID \
-          sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
-      displayName: 'Install Sudo in container (thanks Microsoft!)'
-
     - bash: |
         pip install -q packaging
-        python ./requirements/adjust-versions.py requirements.txt
+        python ./requirements/adjust-versions.py requirements.txt $(torch-ver)
         for fpath in `ls requirements/*.txt`; do
-            python ./requirements/adjust-versions.py $fpath
+            python ./requirements/adjust-versions.py $fpath $(torch-ver)
         done
       displayName: 'Adjust versions'
 
@@ -97,15 +116,15 @@ jobs:
         sudo apt-get update -qq --fix-missing
         sudo apt-get install -y build-essential gcc g++ cmake ffmpeg git libsndfile1 unzip --no-install-recommends
         # pip install pip -U
-        pip install -e . -r ./requirements/devel.txt
+        pip install -e . -U -r ./requirements/devel.txt --find-links ${TORCH_URL}
         pip install mkl-service==2.4.0  # needed for the gpu multiprocessing
         pip list
       displayName: 'Install environment'
 
     - bash: |
         set -e
         python -c "from torch import __version__ as ver ; assert str(ver).split('+')[0] == '$(torch-ver)', f'PyTorch: {ver}'"
-        python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu >= 2, f'GPU: {mgpu}'"
+        python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu >= 2, f'found GPUs: {mgpu}'"
       displayName: 'Sanity check'
 
     - bash: |
@@ -117,7 +136,7 @@ jobs:
         ls -lh $(TRANSFORMERS_CACHE)
       displayName: 'Show caches'
 
-    - bash: python -m pytest torchmetrics --timeout=180 --durations=50
+    - bash: python -m pytest torchmetrics --timeout=240 --durations=50
       env:
         SKIP_SLOW_DOCTEST: 1
       workingDirectory: src
@@ -131,7 +150,7 @@ jobs:
       workingDirectory: tests
       displayName: 'Pull testing data from S3'
 
-    - bash: python -m pytest unittests -v --cov=torchmetrics --timeout=180 --durations=500
+    - bash: python -m pytest unittests -v --cov=torchmetrics --timeout=240 --durations=500
       env:
         CUDA_LAUNCH_BLOCKING: 1
       workingDirectory: tests
@@ -162,11 +181,11 @@ jobs:
         python .github/assistant.py set-oldest-versions --req_files='["requirements/integrate.txt"]'
         cat requirements/integrate.txt
       displayName: 'Set oldest versions'
-      condition: eq(variables['torch-ver'], '1.9.0')
+      condition: eq(variables['torch-ver'], '1.8.1')
 
     - bash: |
         pip install -q -r requirements/integrate.txt
-        pip install -e .  # force reinstall TM as it could be overwritten by integration's depenencies
+        pip install -e .  # force reinstall TM as it could be overwritten by integration's dependencies
         pip list
       displayName: 'Install integrations'
 

.github/workflows/ci-checks.yml

@@ -26,3 +26,8 @@ jobs:
       actions-ref: main
       artifact-name: dist-packages-${{ github.sha }}
       import-name: "torchmetrics"
+      testing-matrix: |
+        {
+          "os": ["ubuntu-20.04", "macos-11", "windows-2022"],
+          "python-version": ["3.8", "3.10"]
+        }

.github/workflows/ci-integrate.yml

@@ -28,10 +28,9 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-20.04, macOS-11, windows-2022]
-        python-version: ['3.7', '3.10']
+        python-version: ['3.8', '3.10']
         requires: ['oldest', 'latest']
         exclude:
-          - {python-version: '3.7', requires: 'latest'}
           - {python-version: '3.10', requires: 'oldest'}
           - {python-version: '3.10', os: 'windows'}  # todo: https://discuss.pytorch.org/t/numpy-is-not-available-error/146192
         include:

.github/workflows/ci-tests-full.yml

@@ -44,11 +44,8 @@ jobs:
           - {os: 'windows-2022', python-version: '3.9', pytorch-version: '1.13.1'}
           - {os: 'windows-2022', python-version: '3.10', pytorch-version: '2.0.0'}
           # the oldest configurations
-          - {os: 'ubuntu-20.04', python-version: '3.7', pytorch-version: '1.8.1', requires: 'oldest'}
           - {os: 'ubuntu-20.04', python-version: '3.8', pytorch-version: '1.8.1', requires: 'oldest'}
-          - {os: 'macOS-11', python-version: '3.7', pytorch-version: '1.8.1', requires: 'oldest'}
           - {os: 'macOS-11', python-version: '3.8', pytorch-version: '1.8.1', requires: 'oldest'}
-          - {os: 'windows-2019', python-version: '3.7', pytorch-version: '1.8.1', requires: 'oldest'}
           - {os: 'windows-2019', python-version: '3.8', pytorch-version: '1.8.1', requires: 'oldest'}
     env:
       PYTORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html"

CHANGELOG.md

@@ -115,7 +115,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Removed
 
--
+- Support for python 3.7 ([#1640](https://github.com/Lightning-AI/metrics/pull/1640))
 
 
 ### Fixed

requirements.txt

@@ -1,7 +1,7 @@
 # NOTE: the upper bound for the package version is only set for CI stability, and it is dropped while installing this package
 #  in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment
 
-numpy >=1.17.2
+numpy >=1.18.5
 torch >=1.8.1, <=2.0.0
 typing-extensions; python_version < '3.9'
 packaging  # hotfix for utils, can be dropped with lit-utils >=0.5

requirements/adjust-versions.py

@@ -82,7 +82,7 @@ def adjust(requires: str, torch_version: Optional[str] = None) -> str:
         requirements_path, torch_version = sys.argv[1:]
     else:
         requirements_path, torch_version = sys.argv[1], None
-    logging.info(f"requirements_path='{requirements_path}' with torch_version='{torch_version}'")
+    logging.info(f"requirements_path='{requirements_path}' with arg torch_version='{torch_version}'")
 
     with open(requirements_path) as fp:
         requirements = fp.read()

requirements/classification_test.txt

@@ -1,6 +1,6 @@
 # NOTE: the upper bound for the package version is only set for CI stability, and it is dropped while installing this package
 #  in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment
 
-pandas >1.0.0, <=1.5.3
+pandas >=1.4.0, <=1.5.3
 netcal >1.0.0, <=1.3.5  # calibration_error
 fairlearn # group_fairness

requirements/multimodal.txt

@@ -1,4 +1,4 @@
 # NOTE: the upper bound for the package version is only set for CI stability, and it is dropped while installing this package
 #  in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment
 
-transformers >=4.10.0, <4.27.4
+transformers >=4.10.0, <4.27.5

requirements/test.txt

@@ -14,4 +14,4 @@ requests <=2.28.2
 fire <=0.5.0
 
 cloudpickle >1.3, <=2.2.1
-scikit-learn >1.0, <1.2.3
+scikit-learn >=1.1.1, <1.2.3

requirements/text_test.txt

@@ -4,6 +4,6 @@
 jiwer >=2.3.0, <=3.0.1
 rouge-score >0.1.0, <=0.1.2
 bert_score ==0.3.13
-transformers >4.4.0, <4.27.4
+transformers >4.4.0, <4.27.5
 huggingface-hub <0.14  # hotfix, failing SDR for latest PT 1.11
 sacrebleu >=2.0.0, <=2.3.1

setup.py

@@ -201,7 +201,7 @@ def _prepare_extras(skip_files: Tuple[str] = ("devel.txt", "doctest.txt", "integ
         include_package_data=True,
         zip_safe=False,
         keywords=["deep learning", "machine learning", "pytorch", "metrics", "AI"],
-        python_requires=">=3.7",
+        python_requires=">=3.8",
         setup_requires=[],
         install_requires=BASE_REQUIREMENTS,
         extras_require=_prepare_extras(),
@@ -227,7 +227,6 @@ def _prepare_extras(skip_files: Tuple[str] = ("devel.txt", "doctest.txt", "integ
             # Specify the Python versions you support here. In particular, ensure
             # that you indicate whether you support Python 2, Python 3 or both.
             "Programming Language :: Python :: 3",
-            "Programming Language :: Python :: 3.7",
             "Programming Language :: Python :: 3.8",
             "Programming Language :: Python :: 3.9",
             "Programming Language :: Python :: 3.10",

tests/unittests/classification/test_precision_recall_curve.py

@@ -55,7 +55,6 @@ class TestBinaryPrecisionRecallCurve(MetricTester):
 
     @pytest.mark.parametrize("ignore_index", [None, -1, 0])
     @pytest.mark.parametrize("ddp", [True, False])
-    @pytest.mark.skipif(compare_version("sklearn", operator.lt, "1.1.0"), reason="Restricted to latest `sklearn`")
     def test_binary_precision_recall_curve(self, input, ddp, ignore_index):
         """Test class implementation of metric."""
         preds, target = input
@@ -74,7 +73,6 @@ def test_binary_precision_recall_curve(self, input, ddp, ignore_index):
         )
 
     @pytest.mark.parametrize("ignore_index", [None, -1, 0])
-    @pytest.mark.skipif(compare_version("sklearn", operator.lt, "1.1.0"), reason="Restricted to latest `sklearn`")
     def test_binary_precision_recall_curve_functional(self, input, ignore_index):
         """Test functional implementation of metric."""
         preds, target = input
@@ -182,7 +180,6 @@ class TestMulticlassPrecisionRecallCurve(MetricTester):
 
     @pytest.mark.parametrize("ignore_index", [None, -1])
     @pytest.mark.parametrize("ddp", [True, False])
-    @pytest.mark.skipif(compare_version("sklearn", operator.lt, "1.1.0"), reason="Restricted to latest `sklearn`")
     def test_multiclass_precision_recall_curve(self, input, ddp, ignore_index):
         """Test class implementation of metric."""
         preds, target = input
@@ -202,7 +199,6 @@ def test_multiclass_precision_recall_curve(self, input, ddp, ignore_index):
         )
 
     @pytest.mark.parametrize("ignore_index", [None, -1])
-    @pytest.mark.skipif(compare_version("sklearn", operator.lt, "1.1.0"), reason="Restricted to latest `sklearn`")
     def test_multiclass_precision_recall_curve_functional(self, input, ignore_index):
         """Test functional implementation of metric."""
         preds, target = input
@@ -304,7 +300,6 @@ class TestMultilabelPrecisionRecallCurve(MetricTester):
 
     @pytest.mark.parametrize("ignore_index", [None, -1, 0])
     @pytest.mark.parametrize("ddp", [True, False])
-    @pytest.mark.skipif(compare_version("sklearn", operator.lt, "1.1.0"), reason="Restricted to latest `sklearn`")
     def test_multilabel_precision_recall_curve(self, input, ddp, ignore_index):
         """Test class implementation of metric."""
         preds, target = input
@@ -324,7 +319,6 @@ def test_multilabel_precision_recall_curve(self, input, ddp, ignore_index):
         )
 
     @pytest.mark.parametrize("ignore_index", [None, -1, 0])
-    @pytest.mark.skipif(compare_version("sklearn", operator.lt, "1.1.0"), reason="Restricted to latest `sklearn`")
     def test_multilabel_precision_recall_curve_functional(self, input, ignore_index):
         """Test functional implementation of metric."""
         preds, target = input

tests/unittests/nominal/test_cramers.py

@@ -90,9 +90,6 @@ def _dython_cramers_v_matrix(matrix, bias_correction, nan_strategy, nan_replace_
 
 
 @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`")
-@pytest.mark.skipif(  # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7
-    torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas"
-)
 @pytest.mark.parametrize(
     "preds, target",
     [
@@ -167,9 +164,6 @@ def test_cramers_v_differentiability(self, preds, target, bias_correction, nan_s
 
 
 @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`")
-@pytest.mark.skipif(  # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7
-    torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas"
-)
 @pytest.mark.parametrize("bias_correction", [False, True])
 @pytest.mark.parametrize(("nan_strategy", "nan_replace_value"), [("replace", 1.0), ("drop", None)])
 def test_cramers_v_matrix(cramers_matrix_input, bias_correction, nan_strategy, nan_replace_value):

tests/unittests/nominal/test_pearson.py

@@ -77,9 +77,6 @@ def _pd_pearsons_t_matrix(matrix):
 
 
 @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`")
-@pytest.mark.skipif(  # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7
-    torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas"
-)
 @pytest.mark.parametrize(
     "preds, target",
     [
@@ -124,9 +121,6 @@ def test_pearsons_t_differentiability(self, preds, target):
 
 
 @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`")
-@pytest.mark.skipif(  # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7
-    torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas"
-)
 def test_pearsons_contingency_coefficient_matrix(pearson_matrix_input):
     """Test matrix version of metric works as expected."""
     tm_score = pearsons_contingency_coefficient_matrix(pearson_matrix_input)

tests/unittests/nominal/test_theils_u.py

@@ -88,9 +88,6 @@ def _dython_theils_u_matrix(matrix, nan_strategy, nan_replace_value):
 
 
 @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`")
-@pytest.mark.skipif(  # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7
-    torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas"
-)
 @pytest.mark.parametrize(
     "preds, target",
     [
@@ -159,9 +156,6 @@ def test_theils_u_differentiability(self, preds, target, nan_strategy, nan_repla
 
 
 @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`")
-@pytest.mark.skipif(  # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7
-    torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas"
-)
 @pytest.mark.parametrize(("nan_strategy", "nan_replace_value"), [("replace", 1.0), ("drop", None)])
 def test_theils_u_matrix(theils_u_matrix_input, nan_strategy, nan_replace_value):
     """Test matrix version of metric works as expected."""

tests/unittests/nominal/test_tschuprows.py

@@ -74,9 +74,6 @@ def _pd_tschuprows_t_matrix(matrix):
 
 
 @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`")
-@pytest.mark.skipif(  # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7
-    torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas"
-)
 @pytest.mark.parametrize(
     "preds, target",
     [
@@ -122,9 +119,6 @@ def test_tschuprows_t_differentiability(self, preds, target):
 
 
 @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`")
-@pytest.mark.skipif(  # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7
-    torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas"
-)
 def test_tschuprows_t_matrix(tschuprows_matrix_input):
     """Test matrix version of metric works as expected."""
     tm_score = tschuprows_t_matrix(tschuprows_matrix_input, bias_correction=False)