From e11c16daace0436cd9f87c55ad13bad939627253 Mon Sep 17 00:00:00 2001 From: samlurye Date: Wed, 21 Jul 2021 14:55:21 -0400 Subject: [PATCH 01/24] Move LightningModule.summarize() off of LightningModule to utilities Deprecate the method `LightningModule.summarize` and move it to its own function in `pytorch_lightning/utilities/model_summary.py`. Additionally, move `LayerSummary` and `ModelSummary`, along with all of their supporting functions, from `pytorch_lightning/core/memory.py` into the same `model_summary.py` file. --- pytorch_lightning/core/lightning.py | 29 +- pytorch_lightning/core/memory.py | 418 +--------------- pytorch_lightning/trainer/trainer.py | 4 +- pytorch_lightning/utilities/model_summary.py | 471 ++++++++++++++++++ tests/deprecated_api/test_remove_1-6.py | 2 +- tests/deprecated_api/test_remove_1-7.py | 23 + .../test_model_summary.py} | 38 +- 7 files changed, 528 insertions(+), 457 deletions(-) create mode 100644 pytorch_lightning/utilities/model_summary.py create mode 100644 tests/deprecated_api/test_remove_1-7.py rename tests/{core/test_memory.py => utilities/test_model_summary.py} (92%) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 1f6272cb2e37f..a7a87590abcf3 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -34,7 +34,6 @@ from pytorch_lightning.core.grads import GradInformation from pytorch_lightning.core.hooks import CheckpointHooks, DataHooks, ModelHooks -from pytorch_lightning.core.memory import ModelSummary from pytorch_lightning.core.mixins import DeviceDtypeModuleMixin, HyperparametersMixin from pytorch_lightning.core.optimizer import LightningOptimizer from pytorch_lightning.core.saving import ModelIO @@ -44,6 +43,7 @@ from pytorch_lightning.utilities.cloud_io import get_filesystem from pytorch_lightning.utilities.distributed import distributed_available, sync_ddp from pytorch_lightning.utilities.exceptions import MisconfigurationException +from pytorch_lightning.utilities.model_summary import ModelSummary, summarize from pytorch_lightning.utilities.parsing import collect_init_args from pytorch_lightning.utilities.signature_utils import is_param_in_hook_signature from pytorch_lightning.utilities.types import _METRIC_COLLECTION, EPOCH_OUTPUT, STEP_OUTPUT @@ -1685,6 +1685,10 @@ def summarize(self, mode: Optional[str] = "top", max_depth: Optional[int] = None """ Summarize this LightningModule. + .. deprecated:: v1.5 + This method was deprecated in v1.5 in favor of `pytorch_lightning.utilities.model_summary.summarize` + and will be removed in v1.7. + Args: mode: Can be either ``'top'`` (summarize only direct submodules) or ``'full'`` (summarize all layers). @@ -1697,24 +1701,13 @@ def summarize(self, mode: Optional[str] = "top", max_depth: Optional[int] = None Return: The model summary object """ - model_summary = None - - # temporary mapping from mode to max_depth - if max_depth is None: - if mode in ModelSummary.MODES: - max_depth = ModelSummary.MODES[mode] - rank_zero_deprecation( - f"Argument `mode` in `LightningModule.summarize` is deprecated in v1.4" - f" and will be removed in v1.6. Use `max_depth={max_depth}` to replicate `mode={mode}` behavior." - ) - model_summary = ModelSummary(self, max_depth=max_depth) - elif mode is not None: - raise MisconfigurationException(f"`mode` can be None, {', '.join(ModelSummary.MODES)}, got {mode}") - else: - model_summary = ModelSummary(self, max_depth=max_depth) + warning_cache.deprecation( + "The `LightningModule.summarize` method is deprecated in v1.5 and will be removed in v1.7. " + "Use `pytorch_lightning.utilities.model_summary.summarize` instead.", + stacklevel=6, + ) - log.info("\n" + str(model_summary)) - return model_summary + return summarize(self, mode, max_depth) def freeze(self) -> None: r""" diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index bba42d6997be3..2e5a45fc4b9e7 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -15,367 +15,7 @@ import os import shutil import subprocess -from collections import OrderedDict -from typing import Any, Dict, List, Optional, Tuple, Union - -import numpy as np -import torch -import torch.nn as nn -from torch import Tensor -from torch.utils.hooks import RemovableHandle - -from pytorch_lightning.utilities import AMPType, DeviceType -from pytorch_lightning.utilities.imports import _TORCH_GREATER_EQUAL_1_8 -from pytorch_lightning.utilities.warnings import WarningCache - -warning_cache = WarningCache() - -PARAMETER_NUM_UNITS = [" ", "K", "M", "B", "T"] -UNKNOWN_SIZE = "?" - - -class LayerSummary(object): - """ - Summary class for a single layer in a :class:`~pytorch_lightning.core.lightning.LightningModule`. - It collects the following information: - - - Type of the layer (e.g. Linear, BatchNorm1d, ...) - - Input shape - - Output shape - - Number of parameters - - The input and output shapes are only known after the example input array was - passed through the model. - - Example:: - - >>> model = torch.nn.Conv2d(3, 8, 3) - >>> summary = LayerSummary(model) - >>> summary.num_parameters - 224 - >>> summary.layer_type - 'Conv2d' - >>> output = model(torch.rand(1, 3, 5, 5)) - >>> summary.in_size - [1, 3, 5, 5] - >>> summary.out_size - [1, 8, 3, 3] - - Args: - module: A module to summarize - - """ - - def __init__(self, module: nn.Module): - super().__init__() - self._module = module - self._hook_handle = self._register_hook() - self._in_size = None - self._out_size = None - - def __del__(self): - self.detach_hook() - - def _register_hook(self) -> Optional[RemovableHandle]: - """ - Registers a hook on the module that computes the input- and output size(s) on the first forward pass. - If the hook is called, it will remove itself from the from the module, meaning that - recursive models will only record their input- and output shapes once. - Registering hooks on :class:`~torch.jit.ScriptModule` is not supported. - - Return: - A handle for the installed hook, or ``None`` if registering the hook is not possible. - """ - - def hook(module, inp, out): - if len(inp) == 1: - inp = inp[0] - self._in_size = parse_batch_shape(inp) - self._out_size = parse_batch_shape(out) - self._hook_handle.remove() - - handle = None - if not isinstance(self._module, torch.jit.ScriptModule): - handle = self._module.register_forward_hook(hook) - return handle - - def detach_hook(self): - """ - Removes the forward hook if it was not already removed in the forward pass. - Will be called after the summary is created. - """ - if self._hook_handle is not None: - self._hook_handle.remove() - - @property - def in_size(self) -> Union[str, List]: - return self._in_size or UNKNOWN_SIZE - - @property - def out_size(self) -> Union[str, List]: - return self._out_size or UNKNOWN_SIZE - - @property - def layer_type(self) -> str: - """ Returns the class name of the module. """ - return str(self._module.__class__.__name__) - - @property - def num_parameters(self) -> int: - """ Returns the number of parameters in this module. """ - return sum(np.prod(p.shape) if not _is_lazy_weight_tensor(p) else 0 for p in self._module.parameters()) - - -class ModelSummary(object): - """ - Generates a summary of all layers in a :class:`~pytorch_lightning.core.lightning.LightningModule`. - - Args: - model: The model to summarize (also referred to as the root module). - mode: Can be one of - - - `top` (default): only the top-level modules will be recorded (the children of the root module) - - `full`: summarizes all layers and their submodules in the root module - - .. deprecated:: v1.4 - This parameter was deprecated in v1.4 in favor of `max_depth` and will be removed in v1.6. - - max_depth: Maximum depth of modules to show. Use -1 to show all modules or 0 to show no - summary. Defaults to 1. - - The string representation of this summary prints a table with columns containing - the name, type and number of parameters for each layer. - - The root module may also have an attribute ``example_input_array`` as shown in the example below. - If present, the root module will be called with it as input to determine the - intermediate input- and output shapes of all layers. Supported are tensors and - nested lists and tuples of tensors. All other types of inputs will be skipped and show as `?` - in the summary table. The summary will also display `?` for layers not used in the forward pass. - - Example:: - - >>> import pytorch_lightning as pl - >>> class LitModel(pl.LightningModule): - ... - ... def __init__(self): - ... super().__init__() - ... self.net = nn.Sequential(nn.Linear(256, 512), nn.BatchNorm1d(512)) - ... self.example_input_array = torch.zeros(10, 256) # optional - ... - ... def forward(self, x): - ... return self.net(x) - ... - >>> model = LitModel() - >>> ModelSummary(model, max_depth=1) # doctest: +NORMALIZE_WHITESPACE - | Name | Type | Params | In sizes | Out sizes - ------------------------------------------------------------ - 0 | net | Sequential | 132 K | [10, 256] | [10, 512] - ------------------------------------------------------------ - 132 K Trainable params - 0 Non-trainable params - 132 K Total params - 0.530 Total estimated model params size (MB) - >>> ModelSummary(model, max_depth=-1) # doctest: +NORMALIZE_WHITESPACE - | Name | Type | Params | In sizes | Out sizes - -------------------------------------------------------------- - 0 | net | Sequential | 132 K | [10, 256] | [10, 512] - 1 | net.0 | Linear | 131 K | [10, 256] | [10, 512] - 2 | net.1 | BatchNorm1d | 1.0 K | [10, 512] | [10, 512] - -------------------------------------------------------------- - 132 K Trainable params - 0 Non-trainable params - 132 K Total params - 0.530 Total estimated model params size (MB) - """ - - MODES = dict(top=1, full=-1) # TODO: remove in v1.6 - - def __init__(self, model, mode: Optional[str] = None, max_depth: Optional[int] = 1): - self._model = model - - # temporary mapping from mode to max_depth - if max_depth is None or mode is not None: - if mode in ModelSummary.MODES: - max_depth = ModelSummary.MODES[mode] - from pytorch_lightning.utilities import rank_zero_deprecation - rank_zero_deprecation( - f"Argument `mode` in `ModelSummary` is deprecated in v1.4" - f" and will be removed in v1.6. Use `max_depth={max_depth}` to replicate `mode={mode}` behaviour." - ) - else: - from pytorch_lightning.utilities.exceptions import MisconfigurationException - raise MisconfigurationException(f"`mode` can be {', '.join(ModelSummary.MODES)}, got {mode}.") - - if not isinstance(max_depth, int) or max_depth < -1: - raise ValueError(f"`max_depth` can be -1, 0 or > 0, got {max_depth}.") - - self._max_depth = max_depth - self._layer_summary = self.summarize() - # 1 byte -> 8 bits - # TODO: how do we compute precisin_megabytes in case of mixed precision? - precision = self._model.precision if isinstance(self._model.precision, int) else 32 - self._precision_megabytes = (precision / 8.0) * 1e-6 - - @property - def named_modules(self) -> List[Tuple[str, nn.Module]]: - if self._max_depth == 0: - mods = [] - elif self._max_depth == 1: - # the children are the top-level modules - mods = self._model.named_children() - else: - mods = self._model.named_modules() - mods = list(mods)[1:] # do not include root module (LightningModule) - return list(mods) - - @property - def layer_names(self) -> List[str]: - return list(self._layer_summary.keys()) - - @property - def layer_types(self) -> List[str]: - return [layer.layer_type for layer in self._layer_summary.values()] - - @property - def in_sizes(self) -> List: - return [layer.in_size for layer in self._layer_summary.values()] - - @property - def out_sizes(self) -> List: - return [layer.out_size for layer in self._layer_summary.values()] - - @property - def param_nums(self) -> List[int]: - return [layer.num_parameters for layer in self._layer_summary.values()] - - @property - def total_parameters(self) -> int: - return sum(p.numel() if not _is_lazy_weight_tensor(p) else 0 for p in self._model.parameters()) - - @property - def trainable_parameters(self) -> int: - return sum( - p.numel() if not _is_lazy_weight_tensor(p) else 0 for p in self._model.parameters() if p.requires_grad - ) - - @property - def model_size(self) -> float: - # todo: seems it does not work with quantized models - it returns 0.0 - return self.total_parameters * self._precision_megabytes - - def summarize(self) -> Dict[str, LayerSummary]: - summary = OrderedDict((name, LayerSummary(module)) for name, module in self.named_modules) - if self._model.example_input_array is not None: - self._forward_example_input() - for layer in summary.values(): - layer.detach_hook() - - if self._max_depth >= 1: - # remove summary entries with depth > max_depth - for k in [k for k in summary if k.count(".") >= self._max_depth]: - del summary[k] - - return summary - - def _forward_example_input(self) -> None: - """ Run the example input through each layer to get input- and output sizes. """ - model = self._model - trainer = self._model.trainer - - input_ = model.example_input_array - input_ = model._apply_batch_transfer_handler(input_) - - if trainer is not None and trainer.amp_backend == AMPType.NATIVE and trainer._device_type != DeviceType.TPU: - model.forward = torch.cuda.amp.autocast()(model.forward) - - mode = model.training - model.eval() - with torch.no_grad(): - # let the model hooks collect the input- and output shapes - if isinstance(input_, (list, tuple)): - model(*input_) - elif isinstance(input_, dict): - model(**input_) - else: - model(input_) - model.train(mode) # restore mode of module - - def __str__(self): - """ - Makes a summary listing with: - - Layer Name, Layer Type, Number of Parameters, Input Sizes, Output Sizes, Model Size - """ - arrays = [ - [" ", list(map(str, range(len(self._layer_summary))))], - ["Name", self.layer_names], - ["Type", self.layer_types], - ["Params", list(map(get_human_readable_count, self.param_nums))], - ] - if self._model.example_input_array is not None: - arrays.append(["In sizes", self.in_sizes]) - arrays.append(["Out sizes", self.out_sizes]) - total_parameters = self.total_parameters - trainable_parameters = self.trainable_parameters - model_size = self.model_size - - return _format_summary_table(total_parameters, trainable_parameters, model_size, *arrays) - - def __repr__(self): - return str(self) - - -def parse_batch_shape(batch: Any) -> Union[str, List]: - if hasattr(batch, "shape"): - return list(batch.shape) - - if isinstance(batch, (list, tuple)): - shape = [parse_batch_shape(el) for el in batch] - return shape - - return UNKNOWN_SIZE - - -def _format_summary_table(total_parameters: int, trainable_parameters: int, model_size: float, *cols) -> str: - """ - Takes in a number of arrays, each specifying a column in - the summary table, and combines them all into one big - string defining the summary table that are nicely formatted. - """ - n_rows = len(cols[0][1]) - n_cols = 1 + len(cols) - - # Get formatting width of each column - col_widths = [] - for c in cols: - col_width = max(len(str(a)) for a in c[1]) if n_rows else 0 - col_width = max(col_width, len(c[0])) # minimum length is header length - col_widths.append(col_width) - - # Formatting - s = "{:<{}}" - total_width = sum(col_widths) + 3 * n_cols - header = [s.format(c[0], l) for c, l in zip(cols, col_widths)] - - # Summary = header + divider + Rest of table - summary = " | ".join(header) + "\n" + "-" * total_width - for i in range(n_rows): - line = [] - for c, l in zip(cols, col_widths): - line.append(s.format(str(c[1][i]), l)) - summary += "\n" + " | ".join(line) - summary += "\n" + "-" * total_width - - summary += "\n" + s.format(get_human_readable_count(trainable_parameters), 10) - summary += "Trainable params" - summary += "\n" + s.format(get_human_readable_count(total_parameters - trainable_parameters), 10) - summary += "Non-trainable params" - summary += "\n" + s.format(get_human_readable_count(total_parameters), 10) - summary += "Total params" - summary += "\n" + s.format(get_formatted_model_size(model_size), 10) - summary += "Total estimated model params size (MB)" - - return summary +from typing import Dict, Union def get_memory_profile(mode: str) -> Union[Dict[str, int], Dict[int, int]]: @@ -427,59 +67,3 @@ def get_gpu_memory_map() -> Dict[str, int]: gpu_memory = [float(x) for x in result.stdout.strip().split(os.linesep)] gpu_memory_map = {f"gpu_id: {gpu_id}/memory.used (MB)": memory for gpu_id, memory in enumerate(gpu_memory)} return gpu_memory_map - - -def get_formatted_model_size(total_model_size: float) -> float: - return f"{total_model_size:,.3f}" - - -def get_human_readable_count(number: int) -> str: - """ - Abbreviates an integer number with K, M, B, T for thousands, millions, - billions and trillions, respectively. - - Examples: - >>> get_human_readable_count(123) - '123 ' - >>> get_human_readable_count(1234) # (one thousand) - '1.2 K' - >>> get_human_readable_count(2e6) # (two million) - '2.0 M' - >>> get_human_readable_count(3e9) # (three billion) - '3.0 B' - >>> get_human_readable_count(4e14) # (four hundred trillion) - '400 T' - >>> get_human_readable_count(5e15) # (more than trillion) - '5,000 T' - - Args: - number: a positive integer number - - Return: - A string formatted according to the pattern described above. - - """ - assert number >= 0 - labels = PARAMETER_NUM_UNITS - num_digits = int(np.floor(np.log10(number)) + 1 if number > 0 else 1) - num_groups = int(np.ceil(num_digits / 3)) - num_groups = min(num_groups, len(labels)) # don't abbreviate beyond trillions - shift = -3 * (num_groups - 1) - number = number * (10**shift) - index = num_groups - 1 - if index < 1 or number >= 100: - return f"{int(number):,d} {labels[index]}" - - return f"{number:,.1f} {labels[index]}" - - -def _is_lazy_weight_tensor(p: Tensor) -> bool: - if _TORCH_GREATER_EQUAL_1_8: - from torch.nn.parameter import UninitializedParameter - if isinstance(p, UninitializedParameter): - warning_cache.warn( - "A layer with UninitializedParameter was found. " - "Thus, the total number of parameters detected may be inaccurate." - ) - return True - return False diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index f6342b5e8e458..b0adb0eaceafd 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -27,7 +27,6 @@ from pytorch_lightning.accelerators import Accelerator, IPUAccelerator from pytorch_lightning.callbacks import Callback from pytorch_lightning.core.datamodule import LightningDataModule -from pytorch_lightning.core.memory import ModelSummary from pytorch_lightning.loggers import LightningLoggerBase from pytorch_lightning.loops import TrainingBatchLoop, TrainingEpochLoop from pytorch_lightning.loops.dataloader.evaluation_loop import EvaluationLoop @@ -82,6 +81,7 @@ from pytorch_lightning.utilities.exceptions import MisconfigurationException from pytorch_lightning.utilities.imports import _fault_tolerant_enabled from pytorch_lightning.utilities.model_helpers import is_overridden +from pytorch_lightning.utilities.model_summary import ModelSummary, summarize from pytorch_lightning.utilities.seed import reset_seed from pytorch_lightning.utilities.types import _EVALUATE_OUTPUT, _PREDICT_OUTPUT, EVAL_DATALOADERS, TRAIN_DATALOADERS @@ -1004,7 +1004,7 @@ def _pre_training_routine(self): # print model summary if self.is_global_zero and self.weights_summary is not None and not self.testing: max_depth = ModelSummary.MODES[self.weights_summary] - ref_model.summarize(max_depth=max_depth) + summarize(ref_model, max_depth=max_depth) # on pretrain routine end self.on_pretrain_routine_end() diff --git a/pytorch_lightning/utilities/model_summary.py b/pytorch_lightning/utilities/model_summary.py new file mode 100644 index 0000000000000..1a90483882668 --- /dev/null +++ b/pytorch_lightning/utilities/model_summary.py @@ -0,0 +1,471 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from collections import OrderedDict +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn as nn +from torch import Tensor + +from pytorch_lightning.utilities import AMPType, DeviceType, rank_zero_deprecation +from pytorch_lightning.utilities.exceptions import MisconfigurationException +from pytorch_lightning.utilities.imports import _TORCH_GREATER_EQUAL_1_8 +from pytorch_lightning.utilities.warnings import WarningCache +from torch.utils.hooks import RemovableHandle + +warning_cache = WarningCache() +log = logging.getLogger(__name__) + +PARAMETER_NUM_UNITS = [" ", "K", "M", "B", "T"] +UNKNOWN_SIZE = "?" + + +class LayerSummary(object): + """ + Summary class for a single layer in a :class:`~pytorch_lightning.core.lightning.LightningModule`. + It collects the following information: + + - Type of the layer (e.g. Linear, BatchNorm1d, ...) + - Input shape + - Output shape + - Number of parameters + + The input and output shapes are only known after the example input array was + passed through the model. + + Example:: + + >>> model = torch.nn.Conv2d(3, 8, 3) + >>> summary = LayerSummary(model) + >>> summary.num_parameters + 224 + >>> summary.layer_type + 'Conv2d' + >>> output = model(torch.rand(1, 3, 5, 5)) + >>> summary.in_size + [1, 3, 5, 5] + >>> summary.out_size + [1, 8, 3, 3] + + Args: + module: A module to summarize + + """ + + def __init__(self, module: nn.Module): + super().__init__() + self._module = module + self._hook_handle = self._register_hook() + self._in_size = None + self._out_size = None + + def __del__(self): + self.detach_hook() + + def _register_hook(self) -> Optional[RemovableHandle]: + """ + Registers a hook on the module that computes the input- and output size(s) on the first forward pass. + If the hook is called, it will remove itself from the from the module, meaning that + recursive models will only record their input- and output shapes once. + Registering hooks on :class:`~torch.jit.ScriptModule` is not supported. + + Return: + A handle for the installed hook, or ``None`` if registering the hook is not possible. + """ + + def hook(module, inp, out): + if len(inp) == 1: + inp = inp[0] + self._in_size = parse_batch_shape(inp) + self._out_size = parse_batch_shape(out) + self._hook_handle.remove() + + handle = None + if not isinstance(self._module, torch.jit.ScriptModule): + handle = self._module.register_forward_hook(hook) + return handle + + def detach_hook(self): + """ + Removes the forward hook if it was not already removed in the forward pass. + Will be called after the summary is created. + """ + if self._hook_handle is not None: + self._hook_handle.remove() + + @property + def in_size(self) -> Union[str, List]: + return self._in_size or UNKNOWN_SIZE + + @property + def out_size(self) -> Union[str, List]: + return self._out_size or UNKNOWN_SIZE + + @property + def layer_type(self) -> str: + """ Returns the class name of the module. """ + return str(self._module.__class__.__name__) + + @property + def num_parameters(self) -> int: + """ Returns the number of parameters in this module. """ + return sum(np.prod(p.shape) if not _is_lazy_weight_tensor(p) else 0 for p in self._module.parameters()) + + +class ModelSummary(object): + """ + Generates a summary of all layers in a :class:`~pytorch_lightning.core.lightning.LightningModule`. + + Args: + model: The model to summarize (also referred to as the root module). + mode: Can be one of + + - `top` (default): only the top-level modules will be recorded (the children of the root module) + - `full`: summarizes all layers and their submodules in the root module + + .. deprecated:: v1.4 + This parameter was deprecated in v1.4 in favor of `max_depth` and will be removed in v1.6. + + max_depth: Maximum depth of modules to show. Use -1 to show all modules or 0 to show no + summary. Defaults to 1. + + The string representation of this summary prints a table with columns containing + the name, type and number of parameters for each layer. + + The root module may also have an attribute ``example_input_array`` as shown in the example below. + If present, the root module will be called with it as input to determine the + intermediate input- and output shapes of all layers. Supported are tensors and + nested lists and tuples of tensors. All other types of inputs will be skipped and show as `?` + in the summary table. The summary will also display `?` for layers not used in the forward pass. + + Example:: + + >>> import pytorch_lightning as pl + >>> class LitModel(pl.LightningModule): + ... + ... def __init__(self): + ... super().__init__() + ... self.net = nn.Sequential(nn.Linear(256, 512), nn.BatchNorm1d(512)) + ... self.example_input_array = torch.zeros(10, 256) # optional + ... + ... def forward(self, x): + ... return self.net(x) + ... + >>> model = LitModel() + >>> ModelSummary(model, max_depth=1) # doctest: +NORMALIZE_WHITESPACE + | Name | Type | Params | In sizes | Out sizes + ------------------------------------------------------------ + 0 | net | Sequential | 132 K | [10, 256] | [10, 512] + ------------------------------------------------------------ + 132 K Trainable params + 0 Non-trainable params + 132 K Total params + 0.530 Total estimated model params size (MB) + >>> ModelSummary(model, max_depth=-1) # doctest: +NORMALIZE_WHITESPACE + | Name | Type | Params | In sizes | Out sizes + -------------------------------------------------------------- + 0 | net | Sequential | 132 K | [10, 256] | [10, 512] + 1 | net.0 | Linear | 131 K | [10, 256] | [10, 512] + 2 | net.1 | BatchNorm1d | 1.0 K | [10, 512] | [10, 512] + -------------------------------------------------------------- + 132 K Trainable params + 0 Non-trainable params + 132 K Total params + 0.530 Total estimated model params size (MB) + """ + + MODES = dict(top=1, full=-1) # TODO: remove in v1.6 + + def __init__(self, model, mode: Optional[str] = None, max_depth: Optional[int] = 1): + self._model = model + + # temporary mapping from mode to max_depth + if max_depth is None or mode is not None: + if mode in ModelSummary.MODES: + max_depth = ModelSummary.MODES[mode] + from pytorch_lightning.utilities import rank_zero_deprecation + rank_zero_deprecation( + f"Argument `mode` in `ModelSummary` is deprecated in v1.4" + f" and will be removed in v1.6. Use `max_depth={max_depth}` to replicate `mode={mode}` behaviour." + ) + else: + from pytorch_lightning.utilities.exceptions import MisconfigurationException + raise MisconfigurationException(f"`mode` can be {', '.join(ModelSummary.MODES)}, got {mode}.") + + if not isinstance(max_depth, int) or max_depth < -1: + raise ValueError(f"`max_depth` can be -1, 0 or > 0, got {max_depth}.") + + self._max_depth = max_depth + self._layer_summary = self.summarize() + # 1 byte -> 8 bits + # TODO: how do we compute precisin_megabytes in case of mixed precision? + precision = self._model.precision if isinstance(self._model.precision, int) else 32 + self._precision_megabytes = (precision / 8.0) * 1e-6 + + @property + def named_modules(self) -> List[Tuple[str, nn.Module]]: + if self._max_depth == 0: + mods = [] + elif self._max_depth == 1: + # the children are the top-level modules + mods = self._model.named_children() + else: + mods = self._model.named_modules() + mods = list(mods)[1:] # do not include root module (LightningModule) + return list(mods) + + @property + def layer_names(self) -> List[str]: + return list(self._layer_summary.keys()) + + @property + def layer_types(self) -> List[str]: + return [layer.layer_type for layer in self._layer_summary.values()] + + @property + def in_sizes(self) -> List: + return [layer.in_size for layer in self._layer_summary.values()] + + @property + def out_sizes(self) -> List: + return [layer.out_size for layer in self._layer_summary.values()] + + @property + def param_nums(self) -> List[int]: + return [layer.num_parameters for layer in self._layer_summary.values()] + + @property + def total_parameters(self) -> int: + return sum(p.numel() if not _is_lazy_weight_tensor(p) else 0 for p in self._model.parameters()) + + @property + def trainable_parameters(self) -> int: + return sum( + p.numel() if not _is_lazy_weight_tensor(p) else 0 for p in self._model.parameters() if p.requires_grad + ) + + @property + def model_size(self) -> float: + # todo: seems it does not work with quantized models - it returns 0.0 + return self.total_parameters * self._precision_megabytes + + def summarize(self) -> Dict[str, LayerSummary]: + summary = OrderedDict((name, LayerSummary(module)) for name, module in self.named_modules) + if self._model.example_input_array is not None: + self._forward_example_input() + for layer in summary.values(): + layer.detach_hook() + + if self._max_depth >= 1: + # remove summary entries with depth > max_depth + for k in [k for k in summary if k.count(".") >= self._max_depth]: + del summary[k] + + return summary + + def _forward_example_input(self) -> None: + """ Run the example input through each layer to get input- and output sizes. """ + model = self._model + trainer = self._model.trainer + + input_ = model.example_input_array + input_ = model._apply_batch_transfer_handler(input_) + + if trainer is not None and trainer.amp_backend == AMPType.NATIVE and trainer._device_type != DeviceType.TPU: + model.forward = torch.cuda.amp.autocast()(model.forward) + + mode = model.training + model.eval() + with torch.no_grad(): + # let the model hooks collect the input- and output shapes + if isinstance(input_, (list, tuple)): + model(*input_) + elif isinstance(input_, dict): + model(**input_) + else: + model(input_) + model.train(mode) # restore mode of module + + def __str__(self): + """ + Makes a summary listing with: + + Layer Name, Layer Type, Number of Parameters, Input Sizes, Output Sizes, Model Size + """ + arrays = [ + [" ", list(map(str, range(len(self._layer_summary))))], + ["Name", self.layer_names], + ["Type", self.layer_types], + ["Params", list(map(get_human_readable_count, self.param_nums))], + ] + if self._model.example_input_array is not None: + arrays.append(["In sizes", self.in_sizes]) + arrays.append(["Out sizes", self.out_sizes]) + total_parameters = self.total_parameters + trainable_parameters = self.trainable_parameters + model_size = self.model_size + + return _format_summary_table(total_parameters, trainable_parameters, model_size, *arrays) + + def __repr__(self): + return str(self) + + +def parse_batch_shape(batch: Any) -> Union[str, List]: + if hasattr(batch, "shape"): + return list(batch.shape) + + if isinstance(batch, (list, tuple)): + shape = [parse_batch_shape(el) for el in batch] + return shape + + return UNKNOWN_SIZE + + +def _format_summary_table(total_parameters: int, trainable_parameters: int, model_size: float, *cols) -> str: + """ + Takes in a number of arrays, each specifying a column in + the summary table, and combines them all into one big + string defining the summary table that are nicely formatted. + """ + n_rows = len(cols[0][1]) + n_cols = 1 + len(cols) + + # Get formatting width of each column + col_widths = [] + for c in cols: + col_width = max(len(str(a)) for a in c[1]) if n_rows else 0 + col_width = max(col_width, len(c[0])) # minimum length is header length + col_widths.append(col_width) + + # Formatting + s = "{:<{}}" + total_width = sum(col_widths) + 3 * n_cols + header = [s.format(c[0], l) for c, l in zip(cols, col_widths)] + + # Summary = header + divider + Rest of table + summary = " | ".join(header) + "\n" + "-" * total_width + for i in range(n_rows): + line = [] + for c, l in zip(cols, col_widths): + line.append(s.format(str(c[1][i]), l)) + summary += "\n" + " | ".join(line) + summary += "\n" + "-" * total_width + + summary += "\n" + s.format(get_human_readable_count(trainable_parameters), 10) + summary += "Trainable params" + summary += "\n" + s.format(get_human_readable_count(total_parameters - trainable_parameters), 10) + summary += "Non-trainable params" + summary += "\n" + s.format(get_human_readable_count(total_parameters), 10) + summary += "Total params" + summary += "\n" + s.format(get_formatted_model_size(model_size), 10) + summary += "Total estimated model params size (MB)" + + return summary + + +def get_formatted_model_size(total_model_size: float) -> float: + return f"{total_model_size:,.3f}" + + +def get_human_readable_count(number: int) -> str: + """ + Abbreviates an integer number with K, M, B, T for thousands, millions, + billions and trillions, respectively. + + Examples: + >>> get_human_readable_count(123) + '123 ' + >>> get_human_readable_count(1234) # (one thousand) + '1.2 K' + >>> get_human_readable_count(2e6) # (two million) + '2.0 M' + >>> get_human_readable_count(3e9) # (three billion) + '3.0 B' + >>> get_human_readable_count(4e14) # (four hundred trillion) + '400 T' + >>> get_human_readable_count(5e15) # (more than trillion) + '5,000 T' + + Args: + number: a positive integer number + + Return: + A string formatted according to the pattern described above. + + """ + assert number >= 0 + labels = PARAMETER_NUM_UNITS + num_digits = int(np.floor(np.log10(number)) + 1 if number > 0 else 1) + num_groups = int(np.ceil(num_digits / 3)) + num_groups = min(num_groups, len(labels)) # don't abbreviate beyond trillions + shift = -3 * (num_groups - 1) + number = number * (10**shift) + index = num_groups - 1 + if index < 1 or number >= 100: + return f"{int(number):,d} {labels[index]}" + + return f"{number:,.1f} {labels[index]}" + + +def _is_lazy_weight_tensor(p: Tensor) -> bool: + if _TORCH_GREATER_EQUAL_1_8: + from torch.nn.parameter import UninitializedParameter + if isinstance(p, UninitializedParameter): + warning_cache.warn( + "A layer with UninitializedParameter was found. " + "Thus, the total number of parameters detected may be inaccurate." + ) + return True + return False + + +def summarize(lightning_module, mode: Optional[str] = "top", max_depth: Optional[int] = None) -> Optional[ModelSummary]: + """ + Summarize the LightningModule specified by `lightning_module`. + + Args: + lightning_module: `LightningModule` to summarize. + mode: Can be either ``'top'`` (summarize only direct submodules) or ``'full'`` (summarize all layers). + + .. deprecated:: v1.4 + This parameter was deprecated in v1.4 in favor of `max_depth` and will be removed in v1.6. + + max_depth: The maximum depth of layer nesting that the summary will include. A value of 0 turns the + layer summary off. Default: 1. + + Return: + The model summary object + """ + model_summary = None + + # temporary mapping from mode to max_depth + if max_depth is None: + if mode in ModelSummary.MODES: + max_depth = ModelSummary.MODES[mode] + rank_zero_deprecation( + f"Argument `mode` in `LightningModule.summarize` is deprecated in v1.4" + f" and will be removed in v1.6. Use `max_depth={max_depth}` to replicate `mode={mode}` behavior." + ) + model_summary = ModelSummary(lightning_module, max_depth=max_depth) + elif mode is not None: + raise MisconfigurationException(f"`mode` can be None, {', '.join(ModelSummary.MODES)}, got {mode}") + else: + model_summary = ModelSummary(lightning_module, max_depth=max_depth) + + log.info("\n" + str(model_summary)) + return model_summary diff --git a/tests/deprecated_api/test_remove_1-6.py b/tests/deprecated_api/test_remove_1-6.py index 73365f8d022f2..406af6ac0d9db 100644 --- a/tests/deprecated_api/test_remove_1-6.py +++ b/tests/deprecated_api/test_remove_1-6.py @@ -17,10 +17,10 @@ from pytorch_lightning import Trainer from pytorch_lightning.callbacks import ModelCheckpoint from pytorch_lightning.callbacks.early_stopping import EarlyStopping -from pytorch_lightning.core.memory import ModelSummary from pytorch_lightning.plugins.training_type import DDPPlugin, DDPSpawnPlugin from pytorch_lightning.utilities.distributed import rank_zero_deprecation, rank_zero_warn from pytorch_lightning.utilities.model_helpers import is_overridden +from pytorch_lightning.utilities.model_summary import ModelSummary from tests.helpers import BoringDataModule, BoringModel diff --git a/tests/deprecated_api/test_remove_1-7.py b/tests/deprecated_api/test_remove_1-7.py new file mode 100644 index 0000000000000..bc65a62041995 --- /dev/null +++ b/tests/deprecated_api/test_remove_1-7.py @@ -0,0 +1,23 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Test deprecated functionality which will be removed in v1.7.0 """ +import pytest + +from tests.helpers import BoringModel + + +def test_v1_7_0_deprecated_lightning_module_summarize(tmpdir): + model = BoringModel() + with pytest.deprecated_call(match="The `LightningModule.summarize` method is deprecated in v1.5"): + model.summarize(max_depth=1) diff --git a/tests/core/test_memory.py b/tests/utilities/test_model_summary.py similarity index 92% rename from tests/core/test_memory.py rename to tests/utilities/test_model_summary.py index 96e1bfaec14cb..f2b0cb6659170 100644 --- a/tests/core/test_memory.py +++ b/tests/utilities/test_model_summary.py @@ -16,9 +16,9 @@ import torch.nn as nn from pytorch_lightning import LightningModule, Trainer -from pytorch_lightning.core.memory import ModelSummary, UNKNOWN_SIZE from pytorch_lightning.utilities import _TORCH_GREATER_EQUAL_1_9 from pytorch_lightning.utilities.exceptions import MisconfigurationException +from pytorch_lightning.utilities.model_summary import ModelSummary, summarize, UNKNOWN_SIZE from tests.helpers import BoringModel from tests.helpers.advanced_models import ParityModuleRNN from tests.helpers.runif import RunIf @@ -140,7 +140,7 @@ def forward(self, inp): def test_invalid_weights_summmary(): """ Test that invalid value for weights_summary raises an error. """ with pytest.raises(MisconfigurationException, match='`mode` can be None, .* got temp'): - UnorderedModel().summarize(mode='temp') + summarize(UnorderedModel, mode='temp') with pytest.raises(MisconfigurationException, match='`weights_summary` can be None, .* got temp'): Trainer(weights_summary='temp') @@ -150,7 +150,7 @@ def test_invalid_weights_summmary(): def test_empty_model_summary_shapes(mode: str): """ Test that the summary works for models that have no submodules. """ model = EmptyModule() - summary = model.summarize(mode=mode) + summary = summarize(model, mode=mode) assert summary.in_sizes == [] assert summary.out_sizes == [] assert summary.param_nums == [] @@ -167,7 +167,7 @@ def test_linear_model_summary_shapes(device, mode): """ Test that the model summary correctly computes the input- and output shapes. """ model = UnorderedModel().to(device) model.train() - summary = model.summarize(mode=mode) + summary = summarize(model, mode=mode) assert summary.in_sizes == [ [2, 10], # layer 2 [2, 7], # combine @@ -189,7 +189,7 @@ def test_linear_model_summary_shapes(device, mode): def test_mixed_dtype_model_summary(): """ Test that the model summary works with models that have mixed input- and parameter dtypes. """ model = MixedDtypeModel() - summary = model.summarize() + summary = summarize(model, ) assert summary.in_sizes == [ [2, 3], # embed [2, 3, 20], # reduce @@ -224,7 +224,7 @@ def test_rnn_summary_shapes(mode): model.example_input_array = torch.zeros(b, t, 10) - summary = model.summarize(mode=mode) + summary = summarize(model, mode=mode) assert summary.in_sizes == [ [b, t, i], # rnn [b, t, h], # linear @@ -239,7 +239,7 @@ def test_rnn_summary_shapes(mode): def test_summary_parameter_count(mode): """ Test that the summary counts the number of parameters in every submodule. """ model = UnorderedModel() - summary = model.summarize(mode=mode) + summary = summarize(model, mode=mode) assert summary.param_nums == [ model.layer2.weight.numel() + model.layer2.bias.numel(), model.combine.weight.numel() + model.combine.bias.numel(), @@ -253,7 +253,7 @@ def test_summary_parameter_count(mode): def test_summary_layer_types(mode): """ Test that the summary displays the layer names correctly. """ model = UnorderedModel() - summary = model.summarize(mode=mode) + summary = summarize(model, mode=mode) assert summary.layer_types == [ 'Linear', 'Linear', @@ -266,7 +266,7 @@ def test_summary_layer_types(mode): @pytest.mark.parametrize('mode', ["full", "top"]) def test_summary_with_scripted_modules(mode): model = PartialScriptModel() - summary = model.summarize(mode=mode) + summary = summarize(model, mode=mode) assert summary.layer_types == ["RecursiveScriptModule", "Linear"] assert summary.in_sizes == [UNKNOWN_SIZE, [2, 3]] assert summary.out_sizes == [UNKNOWN_SIZE, [2, 2]] @@ -302,7 +302,7 @@ def forward(self, *args, **kwargs): model = DummyLightningModule() model.example_input_array = example_input - summary = model.summarize(mode=mode) + summary = summarize(model, mode=mode) assert summary.in_sizes == [expected_size] @@ -310,7 +310,7 @@ def forward(self, *args, **kwargs): def test_model_size(mode): """ Test model size is calculated correctly. """ model = PreCalculatedModel() - summary = model.summarize(mode=mode) + summary = summarize(model, mode=mode) assert model.pre_calculated_model_size == summary.model_size @@ -318,7 +318,7 @@ def test_model_size(mode): def test_empty_model_size(mode): """ Test empty model size is zero. """ model = EmptyModule() - summary = model.summarize(mode=mode) + summary = summarize(model, mode=mode) assert 0.0 == summary.model_size @@ -336,7 +336,7 @@ def test_model_size_precision(tmpdir): precision=32, ) trainer.fit(model) - summary = model.summarize() + summary = summarize(model, ) assert model.pre_calculated_model_size == summary.model_size @@ -362,15 +362,15 @@ def test_lazy_model_summary(): def test_max_depth_equals_mode_interface(): - """Test model.summarize(full/top) interface mapping matches max_depth""" + """Test summarize(model, full/top) interface mapping matches max_depth""" model = DeepNestedModel() - summary_top = model.summarize(mode="top") - summary_0 = model.summarize(max_depth=1) + summary_top = summarize(model, mode="top") + summary_0 = summarize(model, max_depth=1) assert str(summary_top) == str(summary_0) - summary_full = model.summarize(mode="full") - summary_minus1 = model.summarize(max_depth=-1) + summary_full = summarize(model, mode="full") + summary_minus1 = summarize(model, max_depth=-1) assert str(summary_full) == str(summary_minus1) @@ -387,4 +387,4 @@ def test_max_depth_param(max_depth): @pytest.mark.parametrize('max_depth', [-99, -2, "invalid"]) def test_raise_invalid_max_depth_value(max_depth): with pytest.raises(ValueError, match=f"`max_depth` can be -1, 0 or > 0, got {max_depth}"): - DeepNestedModel().summarize(max_depth=max_depth) + summarize(DeepNestedModel(), max_depth=max_depth) From 2d1ce57c5e54af5aad5ddf0df3e52d0276c71122 Mon Sep 17 00:00:00 2001 From: samlurye Date: Wed, 21 Jul 2021 15:24:28 -0400 Subject: [PATCH 02/24] Update CHANGELOG.md with changes from e11c16da --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b260e3401d44..a21c1f266d9e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -329,6 +329,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Deprecated +- Deprecated `LightningModule.summarize()` in favor of `pytorch_lightning.utilities.model_summary.summarize()` + + - Deprecated `LightningModule.loaded_optimizer_states_dict` ([#8229](https://github.com/PyTorchLightning/pytorch-lightning/pull/8229)) From e43daac9ac1740bd2884a5832a30133c66e3401f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 21 Jul 2021 19:54:41 +0000 Subject: [PATCH 03/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytorch_lightning/utilities/model_summary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/utilities/model_summary.py b/pytorch_lightning/utilities/model_summary.py index 1a90483882668..a0453f535e993 100644 --- a/pytorch_lightning/utilities/model_summary.py +++ b/pytorch_lightning/utilities/model_summary.py @@ -20,12 +20,12 @@ import torch import torch.nn as nn from torch import Tensor +from torch.utils.hooks import RemovableHandle from pytorch_lightning.utilities import AMPType, DeviceType, rank_zero_deprecation from pytorch_lightning.utilities.exceptions import MisconfigurationException from pytorch_lightning.utilities.imports import _TORCH_GREATER_EQUAL_1_8 from pytorch_lightning.utilities.warnings import WarningCache -from torch.utils.hooks import RemovableHandle warning_cache = WarningCache() log = logging.getLogger(__name__) From 1befd4a37cecc31251d59aae9ad48ebf38694826 Mon Sep 17 00:00:00 2001 From: samlurye Date: Wed, 21 Jul 2021 17:01:16 -0400 Subject: [PATCH 04/24] Fix some code check and testing issues --- pytorch_lightning/utilities/model_summary.py | 2 -- tests/deprecated_api/test_remove_1-7.py | 8 ++++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/utilities/model_summary.py b/pytorch_lightning/utilities/model_summary.py index 1a90483882668..bd56b41911cf5 100644 --- a/pytorch_lightning/utilities/model_summary.py +++ b/pytorch_lightning/utilities/model_summary.py @@ -197,13 +197,11 @@ def __init__(self, model, mode: Optional[str] = None, max_depth: Optional[int] = if max_depth is None or mode is not None: if mode in ModelSummary.MODES: max_depth = ModelSummary.MODES[mode] - from pytorch_lightning.utilities import rank_zero_deprecation rank_zero_deprecation( f"Argument `mode` in `ModelSummary` is deprecated in v1.4" f" and will be removed in v1.6. Use `max_depth={max_depth}` to replicate `mode={mode}` behaviour." ) else: - from pytorch_lightning.utilities.exceptions import MisconfigurationException raise MisconfigurationException(f"`mode` can be {', '.join(ModelSummary.MODES)}, got {mode}.") if not isinstance(max_depth, int) or max_depth < -1: diff --git a/tests/deprecated_api/test_remove_1-7.py b/tests/deprecated_api/test_remove_1-7.py index bc65a62041995..2d792c29d1efe 100644 --- a/tests/deprecated_api/test_remove_1-7.py +++ b/tests/deprecated_api/test_remove_1-7.py @@ -18,6 +18,10 @@ def test_v1_7_0_deprecated_lightning_module_summarize(tmpdir): + from pytorch_lightning.core.lightning import warning_cache model = BoringModel() - with pytest.deprecated_call(match="The `LightningModule.summarize` method is deprecated in v1.5"): - model.summarize(max_depth=1) + model.summarize(max_depth=1) + assert any( + "The `LightningModule.summarize` method is deprecated in v1.5" in w for w in warning_cache + ) + warning_cache.clear() From 3686b05675f4a238758bbc24ba44a758ba4f1dc6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 21 Jul 2021 21:04:28 +0000 Subject: [PATCH 05/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/deprecated_api/test_remove_1-7.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/deprecated_api/test_remove_1-7.py b/tests/deprecated_api/test_remove_1-7.py index 2d792c29d1efe..3b77356196141 100644 --- a/tests/deprecated_api/test_remove_1-7.py +++ b/tests/deprecated_api/test_remove_1-7.py @@ -21,7 +21,5 @@ def test_v1_7_0_deprecated_lightning_module_summarize(tmpdir): from pytorch_lightning.core.lightning import warning_cache model = BoringModel() model.summarize(max_depth=1) - assert any( - "The `LightningModule.summarize` method is deprecated in v1.5" in w for w in warning_cache - ) + assert any("The `LightningModule.summarize` method is deprecated in v1.5" in w for w in warning_cache) warning_cache.clear() From 0d8ea7d7b9f7fb86a06874cd27a0f9badbe5bd1d Mon Sep 17 00:00:00 2001 From: samlurye Date: Wed, 21 Jul 2021 18:31:44 -0400 Subject: [PATCH 06/24] Update tests/utilities/test_model_summary.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrian Wälchli --- tests/utilities/test_model_summary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utilities/test_model_summary.py b/tests/utilities/test_model_summary.py index f2b0cb6659170..dc0ce425d3dbf 100644 --- a/tests/utilities/test_model_summary.py +++ b/tests/utilities/test_model_summary.py @@ -189,7 +189,7 @@ def test_linear_model_summary_shapes(device, mode): def test_mixed_dtype_model_summary(): """ Test that the model summary works with models that have mixed input- and parameter dtypes. """ model = MixedDtypeModel() - summary = summarize(model, ) + summary = summarize(model) assert summary.in_sizes == [ [2, 3], # embed [2, 3, 20], # reduce From da8cd599a845fa64c025675c21fc136794994f09 Mon Sep 17 00:00:00 2001 From: samlurye Date: Wed, 21 Jul 2021 18:31:52 -0400 Subject: [PATCH 07/24] Update tests/utilities/test_model_summary.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrian Wälchli --- tests/utilities/test_model_summary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utilities/test_model_summary.py b/tests/utilities/test_model_summary.py index dc0ce425d3dbf..58e721b605777 100644 --- a/tests/utilities/test_model_summary.py +++ b/tests/utilities/test_model_summary.py @@ -336,7 +336,7 @@ def test_model_size_precision(tmpdir): precision=32, ) trainer.fit(model) - summary = summarize(model, ) + summary = summarize(model) assert model.pre_calculated_model_size == summary.model_size From 4194ca924e71e16e872bef809226abad0d844785 Mon Sep 17 00:00:00 2001 From: samlurye Date: Wed, 21 Jul 2021 18:32:08 -0400 Subject: [PATCH 08/24] Update pytorch_lightning/core/lightning.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrian Wälchli --- pytorch_lightning/core/lightning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index a7a87590abcf3..dfa21f4061f75 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1686,7 +1686,7 @@ def summarize(self, mode: Optional[str] = "top", max_depth: Optional[int] = None Summarize this LightningModule. .. deprecated:: v1.5 - This method was deprecated in v1.5 in favor of `pytorch_lightning.utilities.model_summary.summarize` + This method was deprecated in v1.5 in favor of :func:`pytorch_lightning.utilities.model_summary.summarize` and will be removed in v1.7. Args: From 0ba157339dc6a1848dac5f806ece7080e63fbc25 Mon Sep 17 00:00:00 2001 From: samlurye Date: Wed, 21 Jul 2021 18:51:57 -0400 Subject: [PATCH 09/24] Add backward compatibility for LayerSummary and ModelSummary --- pytorch_lightning/core/lightning.py | 6 +++--- pytorch_lightning/core/memory.py | 11 +++++++++++ tests/deprecated_api/test_remove_1-7.py | 9 ++++++++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index dfa21f4061f75..12f0ad1baff92 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1685,9 +1685,9 @@ def summarize(self, mode: Optional[str] = "top", max_depth: Optional[int] = None """ Summarize this LightningModule. - .. deprecated:: v1.5 - This method was deprecated in v1.5 in favor of :func:`pytorch_lightning.utilities.model_summary.summarize` - and will be removed in v1.7. + .. deprecated:: v1.5 + This method was deprecated in v1.5 in favor of `pytorch_lightning.utilities.model_summary.summarize` + and will be removed in v1.7. Args: mode: Can be either ``'top'`` (summarize only direct submodules) or ``'full'`` (summarize all layers). diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index 2e5a45fc4b9e7..01292f7312fcd 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -17,6 +17,17 @@ import subprocess from typing import Dict, Union +from pytorch_lightning.utilities import rank_zero_deprecation + +rank_zero_deprecation( + "pytorch_lightning.core.memory.LayerSummary and " + "pytorch_lightning.core.memory.ModelSummary have been moved " + "to pytorch_lightning.utilities.model_summary since v1.5 and " + "will be removed in v1.7." +) + +from pytorch_lightning.utilities.model_summary import LayerSummary, ModelSummary + def get_memory_profile(mode: str) -> Union[Dict[str, int], Dict[int, int]]: """ Get a profile of the current memory usage. diff --git a/tests/deprecated_api/test_remove_1-7.py b/tests/deprecated_api/test_remove_1-7.py index 3b77356196141..e29c5bcfd3642 100644 --- a/tests/deprecated_api/test_remove_1-7.py +++ b/tests/deprecated_api/test_remove_1-7.py @@ -12,14 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. """ Test deprecated functionality which will be removed in v1.7.0 """ + import pytest +from tests.deprecated_api import _soft_unimport_module from tests.helpers import BoringModel - def test_v1_7_0_deprecated_lightning_module_summarize(tmpdir): from pytorch_lightning.core.lightning import warning_cache model = BoringModel() model.summarize(max_depth=1) assert any("The `LightningModule.summarize` method is deprecated in v1.5" in w for w in warning_cache) warning_cache.clear() + + +def test_v1_7_0_moved_model_summary_and_layer_summary(tmpdir): + _soft_unimport_module('pytorch_lightning.core.memory') + with pytest.deprecated_call(match="to pytorch_lightning.utilities.model_summary since v1.5"): + from pytorch_lightning.core.memory import LayerSummary, ModelSummary # noqa: F811 F401 From eb3ed28639901ff05571f8d907fdca88ad31f295 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 21 Jul 2021 22:53:12 +0000 Subject: [PATCH 10/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/deprecated_api/test_remove_1-7.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/deprecated_api/test_remove_1-7.py b/tests/deprecated_api/test_remove_1-7.py index e29c5bcfd3642..2df8647f24669 100644 --- a/tests/deprecated_api/test_remove_1-7.py +++ b/tests/deprecated_api/test_remove_1-7.py @@ -18,6 +18,7 @@ from tests.deprecated_api import _soft_unimport_module from tests.helpers import BoringModel + def test_v1_7_0_deprecated_lightning_module_summarize(tmpdir): from pytorch_lightning.core.lightning import warning_cache model = BoringModel() @@ -29,4 +30,4 @@ def test_v1_7_0_deprecated_lightning_module_summarize(tmpdir): def test_v1_7_0_moved_model_summary_and_layer_summary(tmpdir): _soft_unimport_module('pytorch_lightning.core.memory') with pytest.deprecated_call(match="to pytorch_lightning.utilities.model_summary since v1.5"): - from pytorch_lightning.core.memory import LayerSummary, ModelSummary # noqa: F811 F401 + from pytorch_lightning.core.memory import LayerSummary, ModelSummary # noqa: F811 F401 From 3e93e2caaa967cab8e43701cc3bbf9d454bd3de2 Mon Sep 17 00:00:00 2001 From: samlurye Date: Thu, 22 Jul 2021 14:52:59 -0400 Subject: [PATCH 11/24] Move all functions out of core/memory.py --- pytorch_lightning/core/memory.py | 66 ++++--------------- .../logger_connector/logger_connector.py | 3 +- pytorch_lightning/utilities/memory.py | 55 ++++++++++++++++ pytorch_lightning/utilities/model_summary.py | 3 +- tests/accelerators/test_ddp_spawn.py | 2 +- tests/accelerators/test_dp.py | 2 +- tests/deprecated_api/test_remove_1-7.py | 6 ++ 7 files changed, 79 insertions(+), 58 deletions(-) diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index 01292f7312fcd..7526201d18619 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -19,6 +19,17 @@ from pytorch_lightning.utilities import rank_zero_deprecation +rank_zero_deprecation( + "pytorch_lightning.core.memory.get_memory_profile and " + "pytorch_lightning.core.memory.get_gpu_memory_map have been moved " + "to pytorch_lightning.utilities.memory since v1.5 and will be " + "removed in v1.7." +) + +# To support backward compatibility as get_memory_profile and get_gpu_memory_map have been +# moved +from pytorch_lightning.utilities.memory import get_memory_profile, get_gpu_memory_map + rank_zero_deprecation( "pytorch_lightning.core.memory.LayerSummary and " "pytorch_lightning.core.memory.ModelSummary have been moved " @@ -26,55 +37,6 @@ "will be removed in v1.7." ) -from pytorch_lightning.utilities.model_summary import LayerSummary, ModelSummary - - -def get_memory_profile(mode: str) -> Union[Dict[str, int], Dict[int, int]]: - """ Get a profile of the current memory usage. - - Args: - mode: There are two modes: - - - 'all' means return memory for all gpus - - 'min_max' means return memory for max and min - - Return: - A dictionary in which the keys are device ids as integers and - values are memory usage as integers in MB. - If mode is 'min_max', the dictionary will also contain two additional keys: - - - 'min_gpu_mem': the minimum memory usage in MB - - 'max_gpu_mem': the maximum memory usage in MB - """ - memory_map = get_gpu_memory_map() - - if mode == "min_max": - min_index, min_memory = min(memory_map.items(), key=lambda item: item[1]) - max_index, max_memory = max(memory_map.items(), key=lambda item: item[1]) - - memory_map = {"min_gpu_mem": min_memory, "max_gpu_mem": max_memory} - - return memory_map - - -def get_gpu_memory_map() -> Dict[str, int]: - """ - Get the current gpu usage. - - Return: - A dictionary in which the keys are device ids as integers and - values are memory usage as integers in MB. - """ - result = subprocess.run( - [shutil.which("nvidia-smi"), "--query-gpu=memory.used", "--format=csv,nounits,noheader"], - encoding="utf-8", - # capture_output=True, # valid for python version >=3.7 - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, # for backward compatibility with python version 3.6 - check=True, - ) - - # Convert lines into a dictionary - gpu_memory = [float(x) for x in result.stdout.strip().split(os.linesep)] - gpu_memory_map = {f"gpu_id: {gpu_id}/memory.used (MB)": memory for gpu_id, memory in enumerate(gpu_memory)} - return gpu_memory_map +# To support backward compatibility as LayerSummary and ModelSummary have been +# moved +from pytorch_lightning.utilities.model_summary import LayerSummary, ModelSummary # noqa: E402 F401 # isort: skip diff --git a/pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py b/pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py index 265f8e6b3ebdb..dc56bbd24d47b 100644 --- a/pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py +++ b/pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py @@ -18,11 +18,10 @@ import torch import pytorch_lightning as pl -from pytorch_lightning.core import memory from pytorch_lightning.loggers import LightningLoggerBase, LoggerCollection, TensorBoardLogger from pytorch_lightning.trainer.connectors.logger_connector.result import _METRIC, MetricSource from pytorch_lightning.trainer.states import RunningStage, TrainerFn -from pytorch_lightning.utilities import DeviceType +from pytorch_lightning.utilities import DeviceType, memory from pytorch_lightning.utilities.apply_func import apply_to_collection, move_data_to_device from pytorch_lightning.utilities.metrics import metrics_to_scalars from pytorch_lightning.utilities.types import _EVALUATE_OUTPUT diff --git a/pytorch_lightning/utilities/memory.py b/pytorch_lightning/utilities/memory.py index 0ae88e8995614..42c4fd106bb0f 100644 --- a/pytorch_lightning/utilities/memory.py +++ b/pytorch_lightning/utilities/memory.py @@ -13,6 +13,10 @@ # limitations under the License. import gc +import os +import shutil +import subprocess +from typing import Dict, Union import torch @@ -83,3 +87,54 @@ def garbage_collection_cuda(): if not is_oom_error(exception): # Only handle OOM errors raise + + +def get_memory_profile(mode: str) -> Union[Dict[str, int], Dict[int, int]]: + """ Get a profile of the current memory usage. + + Args: + mode: There are two modes: + + - 'all' means return memory for all gpus + - 'min_max' means return memory for max and min + + Return: + A dictionary in which the keys are device ids as integers and + values are memory usage as integers in MB. + If mode is 'min_max', the dictionary will also contain two additional keys: + + - 'min_gpu_mem': the minimum memory usage in MB + - 'max_gpu_mem': the maximum memory usage in MB + """ + memory_map = get_gpu_memory_map() + + if mode == "min_max": + min_index, min_memory = min(memory_map.items(), key=lambda item: item[1]) + max_index, max_memory = max(memory_map.items(), key=lambda item: item[1]) + + memory_map = {"min_gpu_mem": min_memory, "max_gpu_mem": max_memory} + + return memory_map + + +def get_gpu_memory_map() -> Dict[str, int]: + """ + Get the current gpu usage. + + Return: + A dictionary in which the keys are device ids as integers and + values are memory usage as integers in MB. + """ + result = subprocess.run( + [shutil.which("nvidia-smi"), "--query-gpu=memory.used", "--format=csv,nounits,noheader"], + encoding="utf-8", + # capture_output=True, # valid for python version >=3.7 + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, # for backward compatibility with python version 3.6 + check=True, + ) + + # Convert lines into a dictionary + gpu_memory = [float(x) for x in result.stdout.strip().split(os.linesep)] + gpu_memory_map = {f"gpu_id: {gpu_id}/memory.used (MB)": memory for gpu_id, memory in enumerate(gpu_memory)} + return gpu_memory_map diff --git a/pytorch_lightning/utilities/model_summary.py b/pytorch_lightning/utilities/model_summary.py index df78f65cd1676..ddd1cf66c602a 100644 --- a/pytorch_lightning/utilities/model_summary.py +++ b/pytorch_lightning/utilities/model_summary.py @@ -432,7 +432,7 @@ def _is_lazy_weight_tensor(p: Tensor) -> bool: return False -def summarize(lightning_module, mode: Optional[str] = "top", max_depth: Optional[int] = None) -> Optional[ModelSummary]: +def summarize(lightning_module: "pl.LightningModule", mode: Optional[str] = "top", max_depth: Optional[int] = None) -> Optional[ModelSummary]: """ Summarize the LightningModule specified by `lightning_module`. @@ -465,5 +465,4 @@ def summarize(lightning_module, mode: Optional[str] = "top", max_depth: Optional else: model_summary = ModelSummary(lightning_module, max_depth=max_depth) - log.info("\n" + str(model_summary)) return model_summary diff --git a/tests/accelerators/test_ddp_spawn.py b/tests/accelerators/test_ddp_spawn.py index fdd5de2523c1d..a9d082917b422 100644 --- a/tests/accelerators/test_ddp_spawn.py +++ b/tests/accelerators/test_ddp_spawn.py @@ -14,8 +14,8 @@ import tests.helpers.pipelines as tpipes import tests.helpers.utils as tutils from pytorch_lightning.callbacks import EarlyStopping -from pytorch_lightning.core import memory from pytorch_lightning.trainer import Trainer +from pytorch_lightning.utilites import memory from tests.helpers import BoringModel from tests.helpers.datamodules import ClassifDataModule from tests.helpers.runif import RunIf diff --git a/tests/accelerators/test_dp.py b/tests/accelerators/test_dp.py index ab46aba3119fb..9aa280e9c3642 100644 --- a/tests/accelerators/test_dp.py +++ b/tests/accelerators/test_dp.py @@ -21,7 +21,7 @@ import tests.helpers.utils as tutils from pytorch_lightning import Trainer from pytorch_lightning.callbacks import EarlyStopping -from pytorch_lightning.core import memory +from pytorch_lightning.utilities import memory from pytorch_lightning.utilities.exceptions import MisconfigurationException from tests.helpers import BoringModel, RandomDataset from tests.helpers.datamodules import ClassifDataModule diff --git a/tests/deprecated_api/test_remove_1-7.py b/tests/deprecated_api/test_remove_1-7.py index 2df8647f24669..d96ebda59c523 100644 --- a/tests/deprecated_api/test_remove_1-7.py +++ b/tests/deprecated_api/test_remove_1-7.py @@ -31,3 +31,9 @@ def test_v1_7_0_moved_model_summary_and_layer_summary(tmpdir): _soft_unimport_module('pytorch_lightning.core.memory') with pytest.deprecated_call(match="to pytorch_lightning.utilities.model_summary since v1.5"): from pytorch_lightning.core.memory import LayerSummary, ModelSummary # noqa: F811 F401 + + +def test_v1_7_0_moved_get_memory_profile_and_get_gpu_memory_map(tmpdir): + _soft_unimport_module('pytorch_lightning.core.memory') + with pytest.deprecated_call(match="to pytorch_lightning.utilities.memory since v1.5"): + from pytorch_lightning.core.memory import get_memory_profile, get_gpu_memory_map # noqa: F811 F401 From 3cce6e0a20084a5f538db757939242c95b038c10 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 22 Jul 2021 18:54:15 +0000 Subject: [PATCH 12/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytorch_lightning/core/memory.py | 4 ++-- pytorch_lightning/utilities/model_summary.py | 4 +++- tests/deprecated_api/test_remove_1-7.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index 7526201d18619..20b5aaa1c5bbb 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -28,7 +28,7 @@ # To support backward compatibility as get_memory_profile and get_gpu_memory_map have been # moved -from pytorch_lightning.utilities.memory import get_memory_profile, get_gpu_memory_map +from pytorch_lightning.utilities.memory import get_gpu_memory_map, get_memory_profile rank_zero_deprecation( "pytorch_lightning.core.memory.LayerSummary and " @@ -39,4 +39,4 @@ # To support backward compatibility as LayerSummary and ModelSummary have been # moved -from pytorch_lightning.utilities.model_summary import LayerSummary, ModelSummary # noqa: E402 F401 # isort: skip +from pytorch_lightning.utilities.model_summary import LayerSummary, ModelSummary # noqa: E402 F401 # isort: skip diff --git a/pytorch_lightning/utilities/model_summary.py b/pytorch_lightning/utilities/model_summary.py index ddd1cf66c602a..a6437b7151609 100644 --- a/pytorch_lightning/utilities/model_summary.py +++ b/pytorch_lightning/utilities/model_summary.py @@ -432,7 +432,9 @@ def _is_lazy_weight_tensor(p: Tensor) -> bool: return False -def summarize(lightning_module: "pl.LightningModule", mode: Optional[str] = "top", max_depth: Optional[int] = None) -> Optional[ModelSummary]: +def summarize(lightning_module: "pl.LightningModule", + mode: Optional[str] = "top", + max_depth: Optional[int] = None) -> Optional[ModelSummary]: """ Summarize the LightningModule specified by `lightning_module`. diff --git a/tests/deprecated_api/test_remove_1-7.py b/tests/deprecated_api/test_remove_1-7.py index d96ebda59c523..edf29a2473600 100644 --- a/tests/deprecated_api/test_remove_1-7.py +++ b/tests/deprecated_api/test_remove_1-7.py @@ -36,4 +36,4 @@ def test_v1_7_0_moved_model_summary_and_layer_summary(tmpdir): def test_v1_7_0_moved_get_memory_profile_and_get_gpu_memory_map(tmpdir): _soft_unimport_module('pytorch_lightning.core.memory') with pytest.deprecated_call(match="to pytorch_lightning.utilities.memory since v1.5"): - from pytorch_lightning.core.memory import get_memory_profile, get_gpu_memory_map # noqa: F811 F401 + from pytorch_lightning.core.memory import get_gpu_memory_map, get_memory_profile # noqa: F811 F401 From caca34f5d89dba90315a906efbf2f16501bbfd31 Mon Sep 17 00:00:00 2001 From: samlurye Date: Thu, 22 Jul 2021 14:56:46 -0400 Subject: [PATCH 13/24] Remove unused import logging from model_summary.py --- pytorch_lightning/utilities/model_summary.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pytorch_lightning/utilities/model_summary.py b/pytorch_lightning/utilities/model_summary.py index ddd1cf66c602a..4a9dd25be410a 100644 --- a/pytorch_lightning/utilities/model_summary.py +++ b/pytorch_lightning/utilities/model_summary.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging from collections import OrderedDict from typing import Any, Dict, List, Optional, Tuple, Union @@ -28,7 +27,6 @@ from pytorch_lightning.utilities.warnings import WarningCache warning_cache = WarningCache() -log = logging.getLogger(__name__) PARAMETER_NUM_UNITS = [" ", "K", "M", "B", "T"] UNKNOWN_SIZE = "?" From 48e81d3c70df1fddd1f4e3e4b72da33118b1e28f Mon Sep 17 00:00:00 2001 From: ananthsub Date: Tue, 27 Jul 2021 13:11:48 -0700 Subject: [PATCH 14/24] Update tests/accelerators/test_ddp_spawn.py --- tests/accelerators/test_ddp_spawn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/accelerators/test_ddp_spawn.py b/tests/accelerators/test_ddp_spawn.py index a9d082917b422..3fb719c978a6c 100644 --- a/tests/accelerators/test_ddp_spawn.py +++ b/tests/accelerators/test_ddp_spawn.py @@ -15,7 +15,7 @@ import tests.helpers.utils as tutils from pytorch_lightning.callbacks import EarlyStopping from pytorch_lightning.trainer import Trainer -from pytorch_lightning.utilites import memory +from pytorch_lightning.utilities import memory from tests.helpers import BoringModel from tests.helpers.datamodules import ClassifDataModule from tests.helpers.runif import RunIf From 5a0c40714478f3650276c6c6e5d05657ae7f2ced Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 27 Jul 2021 21:17:42 +0000 Subject: [PATCH 15/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../functional/precision_recall_curve.py | 3 +- pytorch_lightning/utilities/model_summary.py | 10 +- tests/utilities/test_model_summary.py | 93 ++++++------------- 3 files changed, 34 insertions(+), 72 deletions(-) diff --git a/pytorch_lightning/metrics/functional/precision_recall_curve.py b/pytorch_lightning/metrics/functional/precision_recall_curve.py index 93b203fae129b..93914c146e82f 100644 --- a/pytorch_lightning/metrics/functional/precision_recall_curve.py +++ b/pytorch_lightning/metrics/functional/precision_recall_curve.py @@ -27,8 +27,7 @@ def precision_recall_curve( pos_label: Optional[int] = None, sample_weights: Optional[Sequence] = None, ) -> Union[ - Tuple[torch.Tensor, torch.Tensor, torch.Tensor], - Tuple[List[torch.Tensor], List[torch.Tensor], List[torch.Tensor]], + Tuple[torch.Tensor, torch.Tensor, torch.Tensor], Tuple[List[torch.Tensor], List[torch.Tensor], List[torch.Tensor]], ]: """ .. deprecated:: diff --git a/pytorch_lightning/utilities/model_summary.py b/pytorch_lightning/utilities/model_summary.py index d0f91f8e96dca..fdff1204d2a4c 100644 --- a/pytorch_lightning/utilities/model_summary.py +++ b/pytorch_lightning/utilities/model_summary.py @@ -32,7 +32,7 @@ UNKNOWN_SIZE = "?" -class LayerSummary(object): +class LayerSummary: """ Summary class for a single layer in a :class:`~pytorch_lightning.core.lightning.LightningModule`. It collects the following information: @@ -115,16 +115,16 @@ def out_size(self) -> Union[str, List]: @property def layer_type(self) -> str: - """ Returns the class name of the module. """ + """Returns the class name of the module.""" return str(self._module.__class__.__name__) @property def num_parameters(self) -> int: - """ Returns the number of parameters in this module. """ + """Returns the number of parameters in this module.""" return sum(np.prod(p.shape) if not _is_lazy_weight_tensor(p) else 0 for p in self._module.parameters()) -class ModelSummary(object): +class ModelSummary: """ Generates a summary of all layers in a :class:`~pytorch_lightning.core.lightning.LightningModule`. @@ -274,7 +274,7 @@ def summarize(self) -> Dict[str, LayerSummary]: return summary def _forward_example_input(self) -> None: - """ Run the example input through each layer to get input- and output sizes. """ + """Run the example input through each layer to get input- and output sizes.""" model = self._model trainer = self._model.trainer diff --git a/tests/utilities/test_model_summary.py b/tests/utilities/test_model_summary.py index d38bef02bb417..0d993bee18ff2 100644 --- a/tests/utilities/test_model_summary.py +++ b/tests/utilities/test_model_summary.py @@ -25,7 +25,7 @@ class EmptyModule(LightningModule): - """ A module that has no layers """ + """A module that has no layers""" def __init__(self): super().__init__() @@ -37,7 +37,7 @@ def forward(self, *args, **kwargs): class PreCalculatedModel(BoringModel): - """ A model with precalculated total params size in MB for FP16 and FP32. """ + """A model with precalculated total params size in MB for FP16 and FP32.""" def __init__(self, precision: int = 32): super().__init__() @@ -54,7 +54,7 @@ def forward(self, x): class UnorderedModel(LightningModule): - """ A model in which the layers not defined in order of execution """ + """A model in which the layers not defined in order of execution""" def __init__(self): super().__init__() @@ -77,7 +77,7 @@ def forward(self, x, y): class MixedDtypeModel(LightningModule): - """ The parameters and inputs of this model have different dtypes. """ + """The parameters and inputs of this model have different dtypes.""" def __init__(self): super().__init__() @@ -90,7 +90,7 @@ def forward(self, x): class PartialScriptModel(LightningModule): - """ A model which contains scripted layers. """ + """A model which contains scripted layers.""" def __init__(self): super().__init__() @@ -103,7 +103,7 @@ def forward(self, x): class LazyModel(LightningModule): - """ A model which contains lazy layers with unintialized parameters. """ + """A model which contains lazy layers with unintialized parameters.""" def __init__(self): super().__init__() @@ -115,7 +115,7 @@ def forward(self, inp): class DeepNestedModel(LightningModule): - """ A model with deep nested layers. """ + """A model with deep nested layers.""" def __init__(self): super().__init__() @@ -138,7 +138,7 @@ def forward(self, inp): def test_invalid_weights_summmary(): - """ Test that invalid value for weights_summary raises an error. """ + """Test that invalid value for weights_summary raises an error.""" with pytest.raises(MisconfigurationException, match="`mode` can be None, .* got temp"): summarize(UnorderedModel, mode="temp") @@ -148,7 +148,7 @@ def test_invalid_weights_summmary(): @pytest.mark.parametrize("mode", ["full", "top"]) def test_empty_model_summary_shapes(mode: str): - """ Test that the summary works for models that have no submodules. """ + """Test that the summary works for models that have no submodules.""" model = EmptyModule() summary = summarize(model, mode=mode) assert summary.in_sizes == [] @@ -160,52 +160,30 @@ def test_empty_model_summary_shapes(mode: str): @pytest.mark.parametrize("mode", ["full", "top"]) @pytest.mark.parametrize( ["device"], - [ - pytest.param(torch.device("cpu")), - pytest.param(torch.device("cuda", 0)), - pytest.param(torch.device("cuda", 0)), - ], + [pytest.param(torch.device("cpu")), pytest.param(torch.device("cuda", 0)), pytest.param(torch.device("cuda", 0))], ) def test_linear_model_summary_shapes(device, mode): - """ Test that the model summary correctly computes the input- and output shapes. """ + """Test that the model summary correctly computes the input- and output shapes.""" model = UnorderedModel().to(device) model.train() summary = summarize(model, mode=mode) - assert summary.in_sizes == [ - [2, 10], # layer 2 - [2, 7], # combine - [2, 3], # layer 1 - [2, 7], # relu - UNKNOWN_SIZE, - ] - assert summary.out_sizes == [ - [2, 2], # layer 2 - [2, 9], # combine - [2, 5], # layer 1 - [2, 7], # relu - UNKNOWN_SIZE, - ] + assert summary.in_sizes == [[2, 10], [2, 7], [2, 3], [2, 7], UNKNOWN_SIZE] # layer 2 # combine # layer 1 # relu + assert summary.out_sizes == [[2, 2], [2, 9], [2, 5], [2, 7], UNKNOWN_SIZE] # layer 2 # combine # layer 1 # relu assert model.training assert model.device == device def test_mixed_dtype_model_summary(): - """ Test that the model summary works with models that have mixed input- and parameter dtypes. """ + """Test that the model summary works with models that have mixed input- and parameter dtypes.""" model = MixedDtypeModel() summary = summarize(model) - assert summary.in_sizes == [ - [2, 3], # embed - [2, 3, 20], # reduce - ] - assert summary.out_sizes == [ - [2, 3, 20], # embed - [2, 3, 1], # reduce - ] + assert summary.in_sizes == [[2, 3], [2, 3, 20]] # embed # reduce + assert summary.out_sizes == [[2, 3, 20], [2, 3, 1]] # embed # reduce @pytest.mark.parametrize("max_depth", [-1, 0]) def test_hooks_removed_after_summarize(max_depth): - """ Test that all hooks were properly removed after summary, even ones that were not run. """ + """Test that all hooks were properly removed after summary, even ones that were not run.""" model = UnorderedModel() summary = ModelSummary(model, max_depth=max_depth) # hooks should be removed @@ -216,7 +194,7 @@ def test_hooks_removed_after_summarize(max_depth): @pytest.mark.parametrize("mode", ["full", "top"]) def test_rnn_summary_shapes(mode): - """ Test that the model summary works for RNNs. """ + """Test that the model summary works for RNNs.""" model = ParityModuleRNN() b = 3 @@ -228,16 +206,13 @@ def test_rnn_summary_shapes(mode): model.example_input_array = torch.zeros(b, t, 10) summary = summarize(model, mode=mode) - assert summary.in_sizes == [ - [b, t, i], # rnn - [b, t, h], # linear - ] + assert summary.in_sizes == [[b, t, i], [b, t, h]] # rnn # linear assert summary.out_sizes == [[[b, t, h], [[1, b, h], [1, b, h]]], [b, t, o]] # rnn # linear @pytest.mark.parametrize("mode", ["full", "top"]) def test_summary_parameter_count(mode): - """ Test that the summary counts the number of parameters in every submodule. """ + """Test that the summary counts the number of parameters in every submodule.""" model = UnorderedModel() summary = summarize(model, mode=mode) assert summary.param_nums == [ @@ -251,16 +226,10 @@ def test_summary_parameter_count(mode): @pytest.mark.parametrize("mode", ["full", "top"]) def test_summary_layer_types(mode): - """ Test that the summary displays the layer names correctly. """ + """Test that the summary displays the layer names correctly.""" model = UnorderedModel() summary = summarize(model, mode=mode) - assert summary.layer_types == [ - "Linear", - "Linear", - "Linear", - "ReLU", - "Conv2d", - ] + assert summary.layer_types == ["Linear", "Linear", "Linear", "ReLU", "Conv2d"] @pytest.mark.parametrize("mode", ["full", "top"]) @@ -286,7 +255,7 @@ def test_summary_with_scripted_modules(mode): ], ) def test_example_input_array_types(example_input, expected_size, mode): - """ Test the types of example inputs supported for display in the summary. """ + """Test the types of example inputs supported for display in the summary.""" class DummyModule(nn.Module): def forward(self, *args, **kwargs): @@ -309,7 +278,7 @@ def forward(self, *args, **kwargs): @pytest.mark.parametrize("mode", ["full", "top"]) def test_model_size(mode): - """ Test model size is calculated correctly. """ + """Test model size is calculated correctly.""" model = PreCalculatedModel() summary = summarize(model, mode=mode) assert model.pre_calculated_model_size == summary.model_size @@ -317,7 +286,7 @@ def test_model_size(mode): @pytest.mark.parametrize("mode", ["full", "top"]) def test_empty_model_size(mode): - """ Test empty model size is zero. """ + """Test empty model size is zero.""" model = EmptyModule() summary = summarize(model, mode=mode) assert 0.0 == summary.model_size @@ -325,17 +294,11 @@ def test_empty_model_size(mode): @RunIf(min_gpus=1, amp_native=True) def test_model_size_precision(tmpdir): - """ Test model size for half and full precision. """ + """Test model size for half and full precision.""" model = PreCalculatedModel() # fit model - trainer = Trainer( - default_root_dir=tmpdir, - gpus=1, - max_steps=1, - max_epochs=1, - precision=32, - ) + trainer = Trainer(default_root_dir=tmpdir, gpus=1, max_steps=1, max_epochs=1, precision=32) trainer.fit(model) summary = summarize(model) assert model.pre_calculated_model_size == summary.model_size @@ -343,7 +306,7 @@ def test_model_size_precision(tmpdir): @RunIf(min_torch="1.8") def test_lazy_model_summary(): - """ Test that the model summary can work with lazy layers. """ + """Test that the model summary can work with lazy layers.""" lazy_model = LazyModel() summary = ModelSummary(lazy_model) From 9e5b8b9b1d7e71239daf99ba25152b5934af56aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 27 Jul 2021 23:28:44 +0200 Subject: [PATCH 16/24] add import for the typehint in summarize() --- pytorch_lightning/utilities/model_summary.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytorch_lightning/utilities/model_summary.py b/pytorch_lightning/utilities/model_summary.py index fdff1204d2a4c..7ffd0a17a30fe 100644 --- a/pytorch_lightning/utilities/model_summary.py +++ b/pytorch_lightning/utilities/model_summary.py @@ -21,6 +21,7 @@ from torch import Tensor from torch.utils.hooks import RemovableHandle +import pytorch_lightning as pl from pytorch_lightning.utilities import AMPType, DeviceType, rank_zero_deprecation from pytorch_lightning.utilities.exceptions import MisconfigurationException from pytorch_lightning.utilities.imports import _TORCH_GREATER_EQUAL_1_8 From 783072cdf521b072f8bfd7ee5ef1e7c6c6d6f60f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 27 Jul 2021 23:58:23 +0200 Subject: [PATCH 17/24] fix unused imports --- pytorch_lightning/core/memory.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index 20b5aaa1c5bbb..3ebd4fc1de248 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -11,12 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -import os -import shutil -import subprocess -from typing import Dict, Union - from pytorch_lightning.utilities import rank_zero_deprecation rank_zero_deprecation( @@ -26,9 +20,8 @@ "removed in v1.7." ) -# To support backward compatibility as get_memory_profile and get_gpu_memory_map have been -# moved -from pytorch_lightning.utilities.memory import get_gpu_memory_map, get_memory_profile +# To support backward compatibility as get_memory_profile and get_gpu_memory_map have been moved +from pytorch_lightning.utilities.memory import get_gpu_memory_map, get_memory_profile # noqa: E402 F401 # isort: skip rank_zero_deprecation( "pytorch_lightning.core.memory.LayerSummary and " @@ -37,6 +30,5 @@ "will be removed in v1.7." ) -# To support backward compatibility as LayerSummary and ModelSummary have been -# moved +# To support backward compatibility as LayerSummary and ModelSummary have been moved from pytorch_lightning.utilities.model_summary import LayerSummary, ModelSummary # noqa: E402 F401 # isort: skip From 0c3cdc877243c5363ab3b2055053a40685fc06b4 Mon Sep 17 00:00:00 2001 From: samlurye Date: Fri, 30 Jul 2021 09:40:22 -0400 Subject: [PATCH 18/24] Fix typo in test_ddp_spawn.py --- tests/accelerators/test_ddp_spawn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/accelerators/test_ddp_spawn.py b/tests/accelerators/test_ddp_spawn.py index a9d082917b422..3fb719c978a6c 100644 --- a/tests/accelerators/test_ddp_spawn.py +++ b/tests/accelerators/test_ddp_spawn.py @@ -15,7 +15,7 @@ import tests.helpers.utils as tutils from pytorch_lightning.callbacks import EarlyStopping from pytorch_lightning.trainer import Trainer -from pytorch_lightning.utilites import memory +from pytorch_lightning.utilities import memory from tests.helpers import BoringModel from tests.helpers.datamodules import ClassifDataModule from tests.helpers.runif import RunIf From 85870d6904dedf723d1b98ff963fb44e1ca896a3 Mon Sep 17 00:00:00 2001 From: samlurye Date: Fri, 30 Jul 2021 11:34:19 -0400 Subject: [PATCH 19/24] Add back log.info call --- pytorch_lightning/utilities/model_summary.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/utilities/model_summary.py b/pytorch_lightning/utilities/model_summary.py index 7ffd0a17a30fe..4834da54220ef 100644 --- a/pytorch_lightning/utilities/model_summary.py +++ b/pytorch_lightning/utilities/model_summary.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging from collections import OrderedDict from typing import Any, Dict, List, Optional, Tuple, Union @@ -27,6 +28,7 @@ from pytorch_lightning.utilities.imports import _TORCH_GREATER_EQUAL_1_8 from pytorch_lightning.utilities.warnings import WarningCache +log = logging.getLogger(__name__) warning_cache = WarningCache() PARAMETER_NUM_UNITS = [" ", "K", "M", "B", "T"] @@ -451,7 +453,6 @@ def summarize( Return: The model summary object """ - model_summary = None # temporary mapping from mode to max_depth if max_depth is None: @@ -466,5 +467,5 @@ def summarize( raise MisconfigurationException(f"`mode` can be None, {', '.join(ModelSummary.MODES)}, got {mode}") else: model_summary = ModelSummary(lightning_module, max_depth=max_depth) - + log.info("\n" + str(model_summary)) return model_summary From ea27e983f914453b98695b371c1319d943c56ba2 Mon Sep 17 00:00:00 2001 From: samlurye Date: Fri, 30 Jul 2021 11:35:47 -0400 Subject: [PATCH 20/24] Update pytorch_lightning/core/memory.py Co-authored-by: Jirka Borovec --- pytorch_lightning/core/memory.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index 3ebd4fc1de248..136f979063757 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -14,10 +14,9 @@ from pytorch_lightning.utilities import rank_zero_deprecation rank_zero_deprecation( - "pytorch_lightning.core.memory.get_memory_profile and " - "pytorch_lightning.core.memory.get_gpu_memory_map have been moved " - "to pytorch_lightning.utilities.memory since v1.5 and will be " - "removed in v1.7." + "`pytorch_lightning.core.memory.get_memory_profile` and" + " `pytorch_lightning.core.memory.get_gpu_memory_map` have been moved" + " to `pytorch_lightning.utilities.memory` since v1.5 and will be removed in v1.7." ) # To support backward compatibility as get_memory_profile and get_gpu_memory_map have been moved From 432c6e1b45f2be2cd899543b75751a8cac0d2617 Mon Sep 17 00:00:00 2001 From: samlurye Date: Fri, 30 Jul 2021 11:36:09 -0400 Subject: [PATCH 21/24] Update pytorch_lightning/core/memory.py Co-authored-by: Jirka Borovec --- pytorch_lightning/core/memory.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index 136f979063757..7957fdecf8432 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -23,10 +23,9 @@ from pytorch_lightning.utilities.memory import get_gpu_memory_map, get_memory_profile # noqa: E402 F401 # isort: skip rank_zero_deprecation( - "pytorch_lightning.core.memory.LayerSummary and " - "pytorch_lightning.core.memory.ModelSummary have been moved " - "to pytorch_lightning.utilities.model_summary since v1.5 and " - "will be removed in v1.7." + "`pytorch_lightning.core.memory.LayerSummary` and" + " `pytorch_lightning.core.memory.ModelSummary` have been moved" + " to `pytorch_lightning.utilities.model_summary` since v1.5 and will be removed in v1.7." ) # To support backward compatibility as LayerSummary and ModelSummary have been moved From b7f47b8729e7d478d6b64f6379860a709bebc477 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Jul 2021 15:41:35 +0000 Subject: [PATCH 22/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytorch_lightning/core/lightning.py | 2 +- pytorch_lightning/utilities/memory.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 4636a29eec6fe..e4d563338ed76 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -42,8 +42,8 @@ from pytorch_lightning.utilities.cloud_io import get_filesystem from pytorch_lightning.utilities.distributed import distributed_available, sync_ddp from pytorch_lightning.utilities.exceptions import MisconfigurationException -from pytorch_lightning.utilities.model_summary import ModelSummary, summarize from pytorch_lightning.utilities.memory import get_model_size_mb +from pytorch_lightning.utilities.model_summary import ModelSummary, summarize from pytorch_lightning.utilities.parsing import collect_init_args from pytorch_lightning.utilities.signature_utils import is_param_in_hook_signature from pytorch_lightning.utilities.types import _METRIC_COLLECTION, EPOCH_OUTPUT, STEP_OUTPUT diff --git a/pytorch_lightning/utilities/memory.py b/pytorch_lightning/utilities/memory.py index 328eb3c6dcb4e..ff8cac116427f 100644 --- a/pytorch_lightning/utilities/memory.py +++ b/pytorch_lightning/utilities/memory.py @@ -16,8 +16,8 @@ import os import shutil import subprocess -from typing import Dict, Union import uuid +from typing import Dict, Union import torch from torch.nn import Module @@ -145,7 +145,7 @@ def get_gpu_memory_map() -> Dict[str, int]: gpu_memory_map = {f"gpu_id: {gpu_id}/memory.used (MB)": memory for gpu_id, memory in enumerate(gpu_memory)} return gpu_memory_map - + def get_model_size_mb(model: Module) -> float: """ Calculates the size of a Module in megabytes by saving the model to a temporary file and reading its size. @@ -162,4 +162,3 @@ def get_model_size_mb(model: Module) -> float: size_mb = os.path.getsize(tmp_name) / 1e6 os.remove(tmp_name) return size_mb - From 9dc913a590adc1d6d76fd010099f36e1d6ca8d5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 3 Aug 2021 23:28:30 +0200 Subject: [PATCH 23/24] fix ticks in warning message test --- tests/deprecated_api/test_remove_1-7.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/deprecated_api/test_remove_1-7.py b/tests/deprecated_api/test_remove_1-7.py index c69983011ba92..3be195e14bcf9 100644 --- a/tests/deprecated_api/test_remove_1-7.py +++ b/tests/deprecated_api/test_remove_1-7.py @@ -30,13 +30,13 @@ def test_v1_7_0_deprecated_lightning_module_summarize(tmpdir): def test_v1_7_0_moved_model_summary_and_layer_summary(tmpdir): _soft_unimport_module("pytorch_lightning.core.memory") - with pytest.deprecated_call(match="to pytorch_lightning.utilities.model_summary since v1.5"): + with pytest.deprecated_call(match="to `pytorch_lightning.utilities.model_summary` since v1.5"): from pytorch_lightning.core.memory import LayerSummary, ModelSummary # noqa: F811 F401 def test_v1_7_0_moved_get_memory_profile_and_get_gpu_memory_map(tmpdir): _soft_unimport_module("pytorch_lightning.core.memory") - with pytest.deprecated_call(match="to pytorch_lightning.utilities.memory since v1.5"): + with pytest.deprecated_call(match="to `pytorch_lightning.utilities.memory` since v1.5"): from pytorch_lightning.core.memory import get_gpu_memory_map, get_memory_profile # noqa: F811 F401 From 2dbffab75f118d8761036d50c1d6281c69c5df3a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Aug 2021 21:29:28 +0000 Subject: [PATCH 24/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytorch_lightning/core/memory.py | 4 ++-- tests/deprecated_api/test_remove_1-7.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index 7957fdecf8432..d61164e6d88cb 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -20,7 +20,7 @@ ) # To support backward compatibility as get_memory_profile and get_gpu_memory_map have been moved -from pytorch_lightning.utilities.memory import get_gpu_memory_map, get_memory_profile # noqa: E402 F401 # isort: skip +from pytorch_lightning.utilities.memory import get_gpu_memory_map, get_memory_profile # noqa: E402, F401 # isort: skip rank_zero_deprecation( "`pytorch_lightning.core.memory.LayerSummary` and" @@ -29,4 +29,4 @@ ) # To support backward compatibility as LayerSummary and ModelSummary have been moved -from pytorch_lightning.utilities.model_summary import LayerSummary, ModelSummary # noqa: E402 F401 # isort: skip +from pytorch_lightning.utilities.model_summary import LayerSummary, ModelSummary # noqa: E402, F401 # isort: skip diff --git a/tests/deprecated_api/test_remove_1-7.py b/tests/deprecated_api/test_remove_1-7.py index 3be195e14bcf9..38b1e50150063 100644 --- a/tests/deprecated_api/test_remove_1-7.py +++ b/tests/deprecated_api/test_remove_1-7.py @@ -31,13 +31,13 @@ def test_v1_7_0_deprecated_lightning_module_summarize(tmpdir): def test_v1_7_0_moved_model_summary_and_layer_summary(tmpdir): _soft_unimport_module("pytorch_lightning.core.memory") with pytest.deprecated_call(match="to `pytorch_lightning.utilities.model_summary` since v1.5"): - from pytorch_lightning.core.memory import LayerSummary, ModelSummary # noqa: F811 F401 + from pytorch_lightning.core.memory import LayerSummary, ModelSummary # noqa: F401 def test_v1_7_0_moved_get_memory_profile_and_get_gpu_memory_map(tmpdir): _soft_unimport_module("pytorch_lightning.core.memory") with pytest.deprecated_call(match="to `pytorch_lightning.utilities.memory` since v1.5"): - from pytorch_lightning.core.memory import get_gpu_memory_map, get_memory_profile # noqa: F811 F401 + from pytorch_lightning.core.memory import get_gpu_memory_map, get_memory_profile # noqa: F401 def test_v1_7_0_deprecated_model_size():