Merge branch 'master' into lite/dtype-conversion

Lightning-AI · Sep 29, 2022 · 1e30af3 · 1e30af3
2 parents f4d0b33 + c8059d4
commit 1e30af3
Show file tree

Hide file tree

Showing 77 changed files with 1,108 additions and 526 deletions.
diff --git a/.azure/app-cloud-e2e.yml b/.azure/app-cloud-e2e.yml
@@ -97,8 +97,15 @@ jobs:
         path: $(pip_cache_dir)
       displayName: Cache pip
 
-    - bash: python -m pip install -e . --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
-      displayName: 'Install lightning'
+    - bash: git restore . && python -m pip install -e . --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
+      displayName: 'Install lightning app'
+      env:
+        PACKAGE_NAME: app
+
+    - bash: git restore . && python -m pip install -e . --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
+      displayName: 'Install pytorch lightning'
+      env:
+        PACKAGE_NAME: pytorch
 
     - bash: python -m pip install -r requirements/app/test.txt -r requirements/app/ui.txt
       displayName: 'Install dependencies'

diff --git a/.azure/gpu-tests-lite.yml b/.azure/gpu-tests-lite.yml
@@ -63,11 +63,13 @@ jobs:
 
     - bash: |
         set -e
-        TORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
-        CUDA_VERSION_MM=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))")
+        PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
         python ./requirements/pytorch/adjust-versions.py requirements/lite/base.txt ${PYTORCH_VERSION}
+
+        CUDA_VERSION_MM=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))")
         pip install -e .[strategies] --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html
         pip install --requirement requirements/pytorch/devel.txt --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html
+
         pip list
       env:
         PACKAGE_NAME: pytorch

diff --git a/.azure/gpu-tests.yml b/.azure/gpu-tests.yml
@@ -91,15 +91,19 @@ jobs:
         set -e
         python -c "fname = 'requirements/pytorch/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'horovod' not in line] ; open(fname, 'w').writelines(lines)"
         python -c "fname = 'requirements/pytorch/strategies.txt' ; lines = [line for line in open(fname).readlines() if 'bagua' not in line] ; open(fname, 'w').writelines(lines)"
-        TORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
-        CUDA_VERSION_MM=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))")
-        CUDA_VERSION_BAGUA=$(python -c "print([ver for ver in [116,113,111,102] if $CUDA_VERSION_MM >= ver][0])")
+
+        PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__.split('+')[0])")
         python ./requirements/pytorch/adjust-versions.py requirements/pytorch/base.txt ${PYTORCH_VERSION}
         python ./requirements/pytorch/adjust-versions.py requirements/pytorch/extra.txt ${PYTORCH_VERSION}
         python ./requirements/pytorch/adjust-versions.py requirements/pytorch/examples.txt ${PYTORCH_VERSION}
-        pip install "bagua-cuda$CUDA_VERSION_BAGUA"
+
+        CUDA_VERSION_MM=$(python -c "import torch ; print(''.join(map(str, torch.version.cuda.split('.')[:2])))")
         pip install -e .[strategies] --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html
         pip install --requirement requirements/pytorch/devel.txt --find-links https://download.pytorch.org/whl/cu${CUDA_VERSION_MM}/torch_stable.html
+
+        CUDA_VERSION_BAGUA=$(python -c "print([ver for ver in [116,113,111,102] if $CUDA_VERSION_MM >= ver][0])")
+        pip install "bagua-cuda$CUDA_VERSION_BAGUA"
+
         pip list
       env:
         PACKAGE_NAME: pytorch

diff --git a/.azure/hpu-tests.yml b/.azure/hpu-tests.yml
@@ -39,7 +39,7 @@ jobs:
     cancelTimeoutInMinutes: "2"
     pool: intel-hpus
     container:
-      image: "vault.habana.ai/gaudi-docker/1.6.0/ubuntu20.04/habanalabs/pytorch-installer-1.12.0:latest"
+      image: "vault.habana.ai/gaudi-docker/1.6.1/ubuntu20.04/habanalabs/pytorch-installer-1.12.0:latest"
       options: "--runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host --shm-size=4g --name cd-container -v /usr/bin/docker:/tmp/docker:ro"
     workspace:
       clean: all

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -8,71 +8,69 @@
 * @williamfalcon @borda @tchaton @awaelchli @kaushikb11 @rohitgr7
 
 # CI/CD and configs
-/.github/       @borda @carmocca @akihironitta @tchaton
-/.circleci/     @borda @carmocca @akihironitta @tchaton
-/.azure/        @borda @carmocca @akihironitta @tchaton
-/dockers/       @borda @carmocca @akihironitta @tchaton
-*.yml           @borda @carmocca @akihironitta @tchaton
+/.github/       @borda @carmocca @akihironitta @otaj
+/.circleci/     @borda @carmocca @akihironitta @otaj
+/.azure/        @borda @carmocca @akihironitta @otaj
+/dockers/       @borda @carmocca @akihironitta @otaj
+*.yml           @borda @carmocca @akihironitta @otaj
 
 # Docs
-/docs/                                      @edenlightning @tchaton @borda @awaelchli @RobertLaurella @Felonious-Spellfire
+/docs/                                      @edenlightning @tchaton @borda @awaelchli @RobertLaurella @rohitgr7
 /.github/*.md                               @edenlightning @williamfalcon @borda
-/.github/ISSUE_TEMPLATE/                    @edenlightning @borda @tchaton
+/.github/ISSUE_TEMPLATE/                    @edenlightning @borda @tchaton @awaelchli
 /docs/source-pytorch/conf.py                @borda @awaelchli @carmocca
 /docs/source-pytorch/index.rst              @williamfalcon
 /docs/source-pytorch/levels                 @williamfalcon @RobertLaurella
 /docs/source-pytorch/expertise_levels       @williamfalcon @RobertLaurella
-/docs/source-app/                           @williamfalcon @Felonious-Spellfire @RobertLaurella @tchaton @awaelchli
+/docs/source-app/                           @williamfalcon @RobertLaurella @tchaton @awaelchli
 /docs/source-app/conf.py                    @borda @awaelchli @carmocca
 /docs/source-app/index.rst                  @williamfalcon
-/docs/source-app/expertise_levels           @williamfalcon @Felonious-Spellfire @RobertLaurella
+/docs/source-app/expertise_levels           @williamfalcon @RobertLaurella
 
-# Packages
-/src/pytorch_lightning                      @borda @awaelchli @carmocca @justusschock @rohitgr7 @otaj
-/src/pytorch_lightning/accelerators         @williamfalcon @tchaton @SeanNaren @awaelchli @justusschock @kaushikb11
-/src/pytorch_lightning/callbacks            @williamfalcon @tchaton @carmocca @kaushikb11
-/src/pytorch_lightning/core                 @tchaton @borda @carmocca @justusschock @kaushikb11
-/src/pytorch_lightning/distributed          @williamfalcon @tchaton @awaelchli @kaushikb11
-/src/pytorch_lightning/lite                 @tchaton @awaelchli @carmocca
-/src/pytorch_lightning/loggers              @tchaton @awaelchli @borda
-/src/pytorch_lightning/loggers/wandb.py     @borisdayma @otaj @rohitgr7
+# PyTorch Lightning
+/src/pytorch_lightning                      @awaelchli @carmocca @rohitgr7 @otaj @justusschock
+/src/pytorch_lightning/accelerators         @awaelchli @carmocca @justusschock @kaushikb11
+/src/pytorch_lightning/callbacks            @borda @awaelchli @carmocca @justusschock
+/src/pytorch_lightning/core                 @borda @carmocca @justusschock
+/src/pytorch_lightning/distributed          @awaelchli @carmocca @justusschock
+/src/pytorch_lightning/lite                 @awaelchli @carmocca @justusschock  # remove in 1.10
+/src/pytorch_lightning/loggers              @borda @awaelchli @carmocca
+/src/pytorch_lightning/loggers/wandb.py     @borisdayma @awaelchli @otaj @rohitgr7
 /src/pytorch_lightning/loggers/neptune.py   @shnela @HubertJaworski @pkasprzyk @pitercl @Raalsky @aniezurawski @kamil-kaczmarek
-/src/pytorch_lightning/loops                @tchaton @awaelchli @justusschock @carmocca
-/src/pytorch_lightning/plugins              @tchaton @awaelchli @justusschock
-/src/pytorch_lightning/profilers            @williamfalcon @tchaton @borda @carmocca
+/src/pytorch_lightning/loops                @awaelchli @justusschock @carmocca
+/src/pytorch_lightning/plugins              @awaelchli @carmocca @justusschock
+/src/pytorch_lightning/profilers            @tchaton @borda @carmocca
 /src/pytorch_lightning/profilers/pytorch.py @nbcsm @guotuofeng @carmocca
-/src/pytorch_lightning/strategies           @tchaton @SeanNaren @awaelchli @justusschock @kaushikb11
-/src/pytorch_lightning/trainer              @williamfalcon @borda @tchaton @carmocca @awaelchli @justusschock @kaushikb11
-/src/pytorch_lightning/trainer/connectors   @tchaton @carmocca @awaelchli
-/src/pytorch_lightning/tuner                @SkafteNicki @borda @awaelchli
-/src/pytorch_lightning/utilities            @borda @tchaton @carmocca
+/src/pytorch_lightning/strategies           @awaelchli @carmocca @justusschock @kaushikb11
+/src/pytorch_lightning/trainer              @williamfalcon @borda @tchaton @awaelchli @carmocca @justusschock
+/src/pytorch_lightning/trainer/connectors   @awaelchli @carmocca @justusschock @otaj
+/src/pytorch_lightning/trainer/conectors/logger_connector @awaelchli @carmocca @rohitgr7
+/src/pytorch_lightning/tuner                @SkafteNicki @borda @awaelchli @rohitgr7
+/src/pytorch_lightning/utilities            @borda @awaelchli @carmocca
+/tests/tests_pytorch                        @awaelchli @carmocca @rohitgr7 @otaj @justusschock
 
+# Core APIs
+/src/pytorch_lightning/callbacks/callback.py @williamfalcon @awaelchli @carmocca
+/src/pytorch_lightning/core/datamodule.py    @williamFalcon @awaelchli @carmocca
+/src/pytorch_lightning/trainer/trainer.py    @williamfalcon @tchaton @awaelchli @carmocca
+/src/pytorch_lightning/core/hooks.py         @williamfalcon @tchaton @awaelchli @carmocca
+/src/pytorch_lightning/core/module.py        @williamfalcon @tchaton @awaelchli @carmocca
+
+# Lightning Lite
+/src/lightning_lite                         @awaelchli @carmocca @justusschock
+/tests/tests_lite                           @awaelchli @carmocca @justusschock
+
+# Lightning App
 /src/lightning_app                          @tchaton @manskx @lantiga
 /src/lightning_app/cli/pl-app-template      @tchaton @awaelchli @lantiga
 /src/lightning_app/core                     @tchaton @awaelchli @manskx
 /src/lightning_app/core/queues.py           @tchaton @hhsecond @manskx
 /src/lightning_app/runners/cloud.py         @tchaton @hhsecond @lantiga
 /src/lightning_app/testing                  @tchaton @manskx @lantiga
 /src/lightning_app/__about__.py             @nohalon @edenlightning @lantiga
-
-/src/lightning_lite                         @awaelchli @carmocca @justusschock
-
-# Examples
-/examples/app_*                             @tchaton @awaelchli @manskx @hhsecond
-
-# App tests
 /tests/tests_app                            @tchaton @awaelchli @manskx @hhsecond
 /tests/tests_app_examples                   @tchaton @awaelchli @manskx @hhsecond
-
-# Lite tests
-/tests/tests_lite                           @awaelchli @carmocca @justusschock
-
-# API
-/src/pytorch_lightning/callbacks/callback.py @williamfalcon @awaelchli @ananthsub @carmocca
-/src/pytorch_lightning/core/datamodule.py    @williamFalcon @awaelchli @ananthsub @carmocca
-/src/pytorch_lightning/trainer/trainer.py    @williamfalcon @tchaton @awaelchli @carmocca
-/src/pytorch_lightning/core/hooks.py         @williamfalcon @tchaton @awaelchli @carmocca
-/src/pytorch_lightning/core/module.py        @williamfalcon @tchaton @awaelchli @carmocca
+/examples/app_*                             @tchaton @awaelchli @manskx @hhsecond
 
 /.github/CODEOWNERS                  @williamfalcon
 /SECURITY.md                         @williamfalcon

diff --git a/.github/ISSUE_TEMPLATE/1_bug_report.yaml b/.github/ISSUE_TEMPLATE/1_bug_report.yaml
@@ -1,52 +1,47 @@
 name: Report a bug
 description: Any errors that are not UI related.
-labels: ["needs triage", "bug"]
+labels: ["needs triage"]
 body:
   - type: markdown
     attributes:
       value: >
-        WAIT!
-
         Before you go any further. Is this really a **🐛 bug**?
 
         If it's a question about how Lightning works, have a look at our [Lightning documentation](https://lightning.ai/lightning-docs/)
         or ask a question on our [Community Slack](https://pytorch-lightning.slack.com).
 
-        If it really is a **🐛 bug**, please fill out this form as completely as you can.
-
-  - type: checkboxes
-    id: checks
-    attributes:
-      label: First check
-      description: Please confirm and check all the following options.
-      options:
-        - label: I'm sure this is a bug.
-          required: true
-        - label: I've added a descriptive title to this bug.
-          required: true
-        - label: I've provided clear instructions on how to reproduce the bug.
-          required: true
-        - label: I've added a code sample.
-          required: true
-        - label: I've provided any other important info that is required.
-          required: true
-
   - type: textarea
     attributes:
       label: Bug description
-      description: A clear and concise description of the 🐛bug🐛.
+      description: A description of the 🐛bug🐛.
+      placeholder: |
+        A clear and concise description of what the bug is.
+
+        Include steps to reproduce, the expected behaviour, and the actual behaviour.
+
+        ```
+        The error message you got, with the full traceback if available.
+        ```
+
     validations:
       required: true
 
+  - type: markdown
+    attributes:
+      value: "**Note: The rest of this form is optional, but filling it out may help us to provide better support.**"
+
   - type: textarea
     attributes:
       label: How to reproduce the bug
       description: >
         Provide steps and example code here.
-      placeholder: "# Insert code here"
+      placeholder: |
+        ```python
+        # Sample code to reproduce the problem
+        ```
       render: python
     validations:
-      required: true
+      required: false
 
   - type: textarea
     attributes:
@@ -65,9 +60,18 @@ body:
 
   - type: textarea
     attributes:
-      label: Important info
-      description: >
-        Provide all the following info
+      label: Environment
+      description: |
+        Please provide details about your enviornment by copying and pasting the output from our environment collection script:
+        https://raw.githubusercontent.com/Lightning-AI/lightning/master/requirements/collect_env_details.py
+        (For security purposes, please check the contents of the script before running it)
+        You can get the script and run it with:
+        ```console
+        wget https://raw.githubusercontent.com/Lightning-AI/lightning/master/requirements/collect_env_details.py
+        python collect_env_details.py
+        ```
+
+        You can also fill out the list below manually.
       value: |
         ```
 
@@ -84,7 +88,7 @@ body:
 
         ```
     validations:
-      required: true
+      required: false
 
   - type: textarea
     attributes:

diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml
@@ -29,7 +29,6 @@ subprojects:
       - "src/pytorch_lightning/**"
       - "tests/tests_pytorch/**"
       - "setup.cfg"  # includes pytest config
-      - ".github/workflows/docs-*.yml"
     checks:
       # Note: updates here should be applied to the lightning_lite group
       - "pl-conda (3.8, 1.10)"

diff --git a/.github/workflows/ci-circleci.yml b/.github/workflows/ci-circleci.yml
@@ -25,6 +25,7 @@ jobs:
   # https://github.com/marketplace/actions/trigger-circleci-pipeline
   trigger-circleci:
     runs-on: ubuntu-latest
+    if: ${{ github.repository_owner == 'Lightning-AI' }}  # skip for forks as secrets cannot be shared
     steps:
       - uses: CircleCI-Public/[email protected]
         env:

diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-checks.yml
@@ -43,14 +43,13 @@ jobs:
       - name: Install dependencies
         env:
           FREEZE_REQUIREMENTS: 1
-          PACKAGE_NAME: ${{ matrix.pkg }}
         run: |
           sudo apt-get update
           sudo apt-get install -y cmake pandoc
           pip --version
           # python -m pip install --upgrade --user pip
-          pip install -e . --quiet -r requirements/${{ matrix.pkg }}/docs.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
-          pip install -r requirements/${{ matrix.pkg }}/devel.txt
+          pip install -r requirements/${{ matrix.pkg }}/docs.txt -r requirements/${{ matrix.pkg }}/devel.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
+          pip install -e .
           pip list
         shell: bash
 
@@ -92,14 +91,11 @@ jobs:
       - name: Install dependencies
         env:
           FREEZE_REQUIREMENTS: 1
-          PACKAGE_NAME: ${{ matrix.pkg }}
         run: |
           sudo apt-get update
-          sudo apt-get install -y cmake pandoc
+          sudo apt-get install -y cmake pandoc texlive-latex-extra dvipng texlive-pictures
           pip --version
           pip install -e . --quiet -r requirements/${{ matrix.pkg }}/docs.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
-          # install Texlive, see https://linuxconfig.org/how-to-install-latex-on-ubuntu-20-04-focal-fossa-linux
-          sudo apt-get update && sudo apt-get install -y texlive-latex-extra dvipng texlive-pictures
           pip list
         shell: bash
 

diff --git a/dockers/base-ipu/Dockerfile b/dockers/base-ipu/Dockerfile
@@ -18,6 +18,7 @@ LABEL maintainer="Lightning-AI <https://github.com/Lightning-AI>"
 
 ARG PYTHON_VERSION=3.9
 ARG CONDA_VERSION=4.9.2
+ARG PYTORCH_VERSION=1.9
 
 SHELL ["/bin/bash", "-c"]
 
@@ -56,13 +57,12 @@ ENV \
 
 COPY environment.yml environment.yml
 
-RUN conda create -y --name $CONDA_ENV python=${PYTHON_VERSION} pytorch=${PYTORCH_VERSION} cudatoolkit=${CUDA_VERSION} -c pytorch && \
-    conda init bash && \
+RUN conda init bash && \
     python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'python>=[\d\.]+', 'python=${PYTHON_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
-    python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'- pytorch[>=]+[\d\.]+', '# - pytorch=${PYTORCH_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
+    python -c "import re ; fname = 'environment.yml' ; req = re.sub(r'- pytorch[>=]+[\d\.]+', '- pytorch=${PYTORCH_VERSION}', open(fname).read()) ; open(fname, 'w').write(req)" && \
     python -c "fname = 'environment.yml' ; req = open(fname).readlines() ; open(fname, 'w').writelines([ln for ln in req if not any(n in ln for n in ['pytorch>', 'horovod'])])" && \
     cat environment.yml && \
-    conda env update --file environment.yml && \
+    conda env create -f environment.yml && \
     conda clean -ya && \
     rm environment.yml
 

diff --git a/dockers/ci-runner-hpu/Dockerfile b/dockers/ci-runner-hpu/Dockerfile
@@ -16,7 +16,7 @@
 #        gaudi-docker-agent:latest
 
 ARG DIST="latest"
-ARG GAUDI_VERSION="1.6.0"
+ARG GAUDI_VERSION="1.6.1"
 ARG PYTORCH_INSTALLER_VERSION="1.12.0"
 FROM vault.habana.ai/gaudi-docker/${GAUDI_VERSION}/ubuntu20.04/habanalabs/pytorch-installer-${PYTORCH_INSTALLER_VERSION}:${DIST}