CI: precommit - docformatter (#8584)

* CI: precommit - docformatter
* fix deprecated

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.pre-commit-config.yaml

@@ -52,6 +52,12 @@ repos:
         args: [--py36-plus]
         name: Upgrade code
 
+  - repo: https://github.com/myint/docformatter
+    rev: v1.4
+    hooks:
+      - id: docformatter
+        args: [--in-place, --wrap-summaries=115, --wrap-descriptions=120]
+
   - repo: https://github.com/asottile/yesqa
     rev: v1.2.3
     hooks:

benchmarks/test_basic_parity.py

@@ -59,9 +59,7 @@ def assert_parity_absolute(pl_values, pt_values, norm_by: float = 1, max_diff: f
 def test_pytorch_parity(
     tmpdir, cls_model: LightningModule, max_diff_speed: float, max_diff_memory: float, num_epochs: int, num_runs: int
 ):
-    """
-    Verify that the same  pytorch and lightning models achieve the same results
-    """
+    """Verify that the same  pytorch and lightning models achieve the same results."""
     lightning = measure_loops(cls_model, kind="PT Lightning", num_epochs=num_epochs, num_runs=num_runs)
     vanilla = measure_loops(cls_model, kind="Vanilla PT", num_epochs=num_epochs, num_runs=num_runs)
 
@@ -88,9 +86,7 @@ def _hook_memory():
 
 
 def measure_loops(cls_model, kind, num_runs=10, num_epochs=10):
-    """
-    Returns an array with the last loss from each epoch for each run
-    """
+    """Returns an array with the last loss from each epoch for each run."""
     hist_losses = []
     hist_durations = []
     hist_memory = []

benchmarks/test_sharded_parity.py

@@ -25,9 +25,7 @@
 
 
 class SeedTrainLoaderModel(BoringModel):
-    """
-    Overrides training loader to ensure we enforce the same seed for all DDP processes.
-    """
+    """Overrides training loader to ensure we enforce the same seed for all DDP processes."""
 
     def train_dataloader(self):
         seed_everything(42)
@@ -87,8 +85,7 @@ def configure_optimizers(self):
 
 
 def record_ddp_fit_model_stats(trainer, model, use_cuda):
-    """
-    Helper to calculate wall clock time for fit + max allocated memory.
+    """Helper to calculate wall clock time for fit + max allocated memory.
 
     Args:
         trainer: The trainer object.
@@ -123,9 +120,8 @@ def plugin_parity_test(
     precision: int = 32,
     max_percent_speed_diff: float = 0.1,
 ):
-    """
-    Ensures that the trained model is identical to the standard DDP implementation.
-    Also checks for speed/memory regressions, we should expect always less memory but performance to fluctuate.
+    """Ensures that the trained model is identical to the standard DDP implementation. Also checks for speed/memory
+    regressions, we should expect always less memory but performance to fluctuate.
 
     Args:
         model_cls: Model class to use for test.
@@ -134,7 +130,6 @@ def plugin_parity_test(
         precision: Whether to use AMP or normal FP32 training.
         max_percent_speed_diff: The maximum speed difference compared to normal DDP training.
         This is more a safety net for variability in CI which can vary in speed, not for benchmarking.
-
     """
 
     # Train normal DDP

docs/source/api_references.rst

@@ -1,6 +1,8 @@
 API References
 ==============
 
+.. include:: links.rst
+
 Accelerator API
 ---------------
 

docs/source/conf.py

@@ -293,7 +293,7 @@ def setup(app):
 # Ignoring Third-party packages
 # https://stackoverflow.com/questions/15889621/sphinx-how-to-exclude-imports-in-automodule
 def package_list_from_file(file):
-    """List up package name (not containing version and extras) from a package list file"""
+    """List up package name (not containing version and extras) from a package list file."""
     mocked_packages = []
     with open(file) as fp:
         for ln in fp.readlines():

docs/source/extensions/plugins.rst

@@ -4,6 +4,8 @@
 Plugins
 #######
 
+.. include:: ../links.rst
+
 Plugins allow custom integrations to the internals of the Trainer such as a custom precision or
 distributed implementation.
 

docs/source/links.rst

@@ -0,0 +1,2 @@
+.. _PyTorchJob: https://www.kubeflow.org/docs/components/training/pytorch/
+.. _Kubeflow: https://www.kubeflow.org

pl_examples/basic_examples/autoencoder.py

@@ -11,11 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
-MNIST autoencoder example.
+"""MNIST autoencoder example.
 
-To run:
-python autoencoder.py --trainer.max_epochs=50
+To run: python autoencoder.py --trainer.max_epochs=50
 """
 
 import torch

pl_examples/basic_examples/backbone_image_classifier.py

@@ -11,11 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
-MNIST backbone image classifier example.
+"""MNIST backbone image classifier example.
 
-To run:
-python backbone_image_classifier.py --trainer.max_epochs=50
+To run: python backbone_image_classifier.py --trainer.max_epochs=50
 """
 from typing import Optional
 

pl_examples/basic_examples/dali_image_classifier.py

@@ -45,9 +45,7 @@
 
 
 class ExternalMNISTInputIterator:
-    """
-    This iterator class wraps torchvision's MNIST dataset and returns the images and labels in batches
-    """
+    """This iterator class wraps torchvision's MNIST dataset and returns the images and labels in batches."""
 
     def __init__(self, mnist_ds, batch_size):
         self.batch_size = batch_size
@@ -73,9 +71,7 @@ def __next__(self):
 
 
 class ExternalSourcePipeline(Pipeline):
-    """
-    This DALI pipeline class just contains the MNIST iterator
-    """
+    """This DALI pipeline class just contains the MNIST iterator."""
 
     def __init__(self, batch_size, eii, num_threads, device_id):
         super().__init__(batch_size, num_threads, device_id, seed=12)
@@ -88,10 +84,8 @@ def define_graph(self):
 
 
 class DALIClassificationLoader(DALIClassificationIterator):
-    """
-    This class extends DALI's original `DALIClassificationIterator` with the `__len__()` function
-     so that we can call `len()` on it
-    """
+    """This class extends DALI's original `DALIClassificationIterator` with the `__len__()` function so that we can
+    call `len()` on it."""
 
     def __init__(
         self,

pl_examples/basic_examples/mnist_datamodule.py

@@ -41,8 +41,7 @@
 
 
 class MNISTDataModule(LightningDataModule):
-    """
-    Standard MNIST, train, val, test splits and transforms
+    """Standard MNIST, train, val, test splits and transforms.
 
     >>> MNISTDataModule()  # doctest: +ELLIPSIS
     <...mnist_datamodule.MNISTDataModule object at ...>
@@ -100,14 +99,14 @@ def prepare_data(self):
         MNIST(self.data_dir, train=False, download=True)
 
     def setup(self, stage: Optional[str] = None):
-        """Split the train and valid dataset"""
+        """Split the train and valid dataset."""
         extra = dict(transform=self.default_transforms) if self.default_transforms else {}
         dataset = MNIST(self.data_dir, train=True, download=False, **extra)
         train_length = len(dataset)
         self.dataset_train, self.dataset_val = random_split(dataset, [train_length - self.val_split, self.val_split])
 
     def train_dataloader(self):
-        """MNIST train set removes a subset to use for validation"""
+        """MNIST train set removes a subset to use for validation."""
         loader = DataLoader(
             self.dataset_train,
             batch_size=self.batch_size,
@@ -119,7 +118,7 @@ def train_dataloader(self):
         return loader
 
     def val_dataloader(self):
-        """MNIST val set uses a subset of the training set for validation"""
+        """MNIST val set uses a subset of the training set for validation."""
         loader = DataLoader(
             self.dataset_val,
             batch_size=self.batch_size,
@@ -131,7 +130,7 @@ def val_dataloader(self):
         return loader
 
     def test_dataloader(self):
-        """MNIST test set uses the test split"""
+        """MNIST test set uses the test split."""
         extra = dict(transform=self.test_transforms) if self.test_transforms else {}
         dataset = MNIST(self.data_dir, train=False, download=False, **extra)
         loader = DataLoader(

pl_examples/basic_examples/profiler_example.py

@@ -11,9 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
-This script will generate 2 traces: one for `training_step` and one for `validation_step`.
-The traces can be visualized in 2 ways:
+"""This script will generate 2 traces: one for `training_step` and one for `validation_step`. The traces can be
+visualized in 2 ways:
+
 * With Chrome:
     1. Open Chrome and copy/paste this url: `chrome://tracing/`.
     2. Once tracing opens, click on `Load` at the top-right and load one of the generated traces.

pl_examples/basic_examples/simple_image_classifier.py

@@ -11,11 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
-MNIST simple image classifier example.
+"""MNIST simple image classifier example.
 
-To run:
-python simple_image_classifier.py --trainer.max_epochs=50
+To run: python simple_image_classifier.py --trainer.max_epochs=50
 """
 
 import torch

pl_examples/domain_templates/computer_vision_fine_tuning.py

@@ -11,12 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Computer vision example on Transfer Learning.
-This computer vision example illustrates how one could fine-tune a pre-trained
-network (by default, a ResNet50 is used) using pytorch-lightning. For the sake
-of this example, the 'cats and dogs dataset' (~60MB, see `DATA_URL` below) and
-the proposed network (denoted by `TransferLearningModel`, see below) is
-trained for 15 epochs.
+"""Computer vision example on Transfer Learning. This computer vision example illustrates how one could fine-tune a
+pre-trained network (by default, a ResNet50 is used) using pytorch-lightning. For the sake of this example, the
+'cats and dogs dataset' (~60MB, see `DATA_URL` below) and the proposed network (denoted by `TransferLearningModel`,
+see below) is trained for 15 epochs.
 
 The training consists of three stages.
 
@@ -91,7 +89,7 @@ def finetune_function(self, pl_module: pl.LightningModule, epoch: int, optimizer
 
 class CatDogImageDataModule(LightningDataModule):
     def __init__(self, dl_path: Union[str, Path] = "data", num_workers: int = 0, batch_size: int = 8):
-        """CatDogImageDataModule
+        """CatDogImageDataModule.
 
         Args:
             dl_path: root directory where to download the data
@@ -166,7 +164,7 @@ def __init__(
         num_workers: int = 6,
         **kwargs,
     ) -> None:
-        """TransferLearningModel
+        """TransferLearningModel.
 
         Args:
             backbone: Name (as in ``torchvision.models``) of the feature extractor
@@ -208,7 +206,10 @@ def __build_model(self):
         self.loss_func = F.binary_cross_entropy_with_logits
 
     def forward(self, x):
-        """Forward pass. Returns logits."""
+        """Forward pass.
+
+        Returns logits.
+        """
 
         # 1. Feature extraction:
         x = self.feature_extractor(x)

pl_examples/domain_templates/generative_adversarial_net.py

@@ -11,9 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
-To run this template just do:
-python generative_adversarial_net.py
+"""To run this template just do: python generative_adversarial_net.py.
 
 After a few epochs, launch TensorBoard to see the images being generated at every batch:
 

pl_examples/domain_templates/imagenet.py

@@ -11,8 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""
-This example is largely adapted from https://github.com/pytorch/examples/blob/master/imagenet/main.py
+"""This example is largely adapted from https://github.com/pytorch/examples/blob/master/imagenet/main.py.
 
 Before you can run this example, you will need to download the ImageNet dataset manually from the
 `official website <http://image-net.org/download>`_ and place it into a folder `path/to/imagenet`.
@@ -28,7 +27,6 @@
 .. code-block: bash
 
     python imagenet.py --help
-
 """
 import os
 from argparse import ArgumentParser, Namespace
@@ -112,7 +110,7 @@ def validation_step(self, batch, batch_idx):
 
     @staticmethod
     def __accuracy(output, target, topk=(1,)):
-        """Computes the accuracy over the k top predictions for the specified values of k"""
+        """Computes the accuracy over the k top predictions for the specified values of k."""
         with torch.no_grad():
             maxk = max(topk)
             batch_size = target.size(0)