diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b68175b8272a..df68ddcf5192 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,3 +54,9 @@ repos: --check-filenames, --check-hidden ] + +- repo: https://github.com/pycqa/flake8 + rev: 4.0.1 + hooks: + - id: flake8 + args: ['--ignore=E,F403,F405,F541,F841,W', '--select=E9,F,W6', '--per-file-ignores=__init__.py:F401'] diff --git a/benchmarks/communication/all_gather.py b/benchmarks/communication/all_gather.py index 993e605b855a..d99d2aa0e4c9 100644 --- a/benchmarks/communication/all_gather.py +++ b/benchmarks/communication/all_gather.py @@ -1,12 +1,7 @@ -import torch from benchmarks.communication.utils import * from benchmarks.communication.constants import * import time -import argparse -import os - -import math # Run all_gather and print metrics diff --git a/benchmarks/communication/all_reduce.py b/benchmarks/communication/all_reduce.py index a2902e53c162..e31f51733609 100644 --- a/benchmarks/communication/all_reduce.py +++ b/benchmarks/communication/all_reduce.py @@ -1,11 +1,7 @@ -import torch from benchmarks.communication.utils import * from benchmarks.communication.constants import * import time -import argparse -import os -import math def timed_all_reduce(input, args): diff --git a/benchmarks/communication/all_to_all.py b/benchmarks/communication/all_to_all.py index e0f1c5925e27..6ee99a48ee62 100644 --- a/benchmarks/communication/all_to_all.py +++ b/benchmarks/communication/all_to_all.py @@ -1,11 +1,7 @@ -import torch from benchmarks.communication.utils import * from benchmarks.communication.constants import * import time -import argparse -import os -import math def timed_all_to_all(input, output, args): diff --git a/benchmarks/communication/broadcast.py b/benchmarks/communication/broadcast.py index 80aef15faf97..e9d89779ec66 100644 --- a/benchmarks/communication/broadcast.py +++ b/benchmarks/communication/broadcast.py @@ -3,9 +3,6 @@ from benchmarks.communication.constants import * import time -import argparse -import os -import math def timed_broadcast(input, args): diff --git a/benchmarks/communication/constants.py b/benchmarks/communication/constants.py index 3276594b777c..4b3356894b5f 100644 --- a/benchmarks/communication/constants.py +++ b/benchmarks/communication/constants.py @@ -1,5 +1,3 @@ -import torch - DEFAULT_WARMUPS = 5 DEFAULT_TRIALS = 50 DEFAULT_TYPE = 'float' diff --git a/benchmarks/communication/pt2pt.py b/benchmarks/communication/pt2pt.py index e23ab1bb3e83..cb99b20b9097 100644 --- a/benchmarks/communication/pt2pt.py +++ b/benchmarks/communication/pt2pt.py @@ -1,11 +1,7 @@ -import torch from benchmarks.communication.utils import * from benchmarks.communication.constants import * import time -import argparse -import os -import math def timed_pt2pt(input, args): diff --git a/benchmarks/communication/run_all.py b/benchmarks/communication/run_all.py index 246280767a63..9d0f0f5f191f 100644 --- a/benchmarks/communication/run_all.py +++ b/benchmarks/communication/run_all.py @@ -1,4 +1,3 @@ -import torch from benchmarks.communication.utils import * from benchmarks.communication.all_reduce import run_all_reduce from benchmarks.communication.all_gather import run_all_gather @@ -7,10 +6,6 @@ from benchmarks.communication.broadcast import run_broadcast from benchmarks.communication.constants import * -import time -import argparse -import os - # For importing def main(args, rank): diff --git a/bin/ds_bench b/bin/ds_bench index 5364d57d6953..bfacbc8e25c8 100755 --- a/bin/ds_bench +++ b/bin/ds_bench @@ -3,7 +3,6 @@ from benchmarks.communication.run_all import main from benchmarks.communication.constants import * from benchmarks.communication.utils import * -import argparse import os import sys diff --git a/csrc/aio/py_test/aio_bench_perf_sweep.py b/csrc/aio/py_test/aio_bench_perf_sweep.py index be6cd74f7ac6..f16e962b1648 100644 --- a/csrc/aio/py_test/aio_bench_perf_sweep.py +++ b/csrc/aio/py_test/aio_bench_perf_sweep.py @@ -277,7 +277,6 @@ def script_path(): def async_io_setup(): - import deepspeed from deepspeed.ops.aio import AsyncIOBuilder return AsyncIOBuilder().is_compatible() diff --git a/csrc/aio/py_test/parse_aio_stats.py b/csrc/aio/py_test/parse_aio_stats.py index 1921973e4f73..2a3e64944bef 100755 --- a/csrc/aio/py_test/parse_aio_stats.py +++ b/csrc/aio/py_test/parse_aio_stats.py @@ -7,7 +7,6 @@ import os import argparse -import re READ_SPEED = 'read_speed' WRITE_SPEED = 'write_speed' diff --git a/csrc/aio/py_test/test_ds_aio.py b/csrc/aio/py_test/test_ds_aio.py index f97d3e676c03..f83f8e78fb65 100755 --- a/csrc/aio/py_test/test_ds_aio.py +++ b/csrc/aio/py_test/test_ds_aio.py @@ -6,11 +6,7 @@ """ import os -import torch import argparse -import time -import sys -from multiprocessing import Pool import multiprocessing as mp from ds_aio_basic import aio_basic_multiprocessing from ds_aio_handle import aio_handle_multiprocessing diff --git a/csrc/aio/py_test/test_ds_aio_utils.py b/csrc/aio/py_test/test_ds_aio_utils.py index c68dfdddc233..a330e4cd1980 100755 --- a/csrc/aio/py_test/test_ds_aio_utils.py +++ b/csrc/aio/py_test/test_ds_aio_utils.py @@ -5,8 +5,6 @@ Functionality of swapping optimizer tensors to/from (NVMe) storage devices. """ -import os - BYTES_PER_GB = 1024**3 LOG_TIDS = [0] diff --git a/csrc/aio/py_test/validate_async_io.py b/csrc/aio/py_test/validate_async_io.py index ceae84c840da..4db25fe1bc33 100644 --- a/csrc/aio/py_test/validate_async_io.py +++ b/csrc/aio/py_test/validate_async_io.py @@ -4,6 +4,5 @@ Functionality of swapping optimizer tensors to/from (NVMe) storage devices. """ -import deepspeed from deepspeed.ops.aio import AsyncIOBuilder assert AsyncIOBuilder().is_compatible() diff --git a/deepspeed/autotuning/autotuner.py b/deepspeed/autotuning/autotuner.py index 4ff85e6d9717..99de3fdcab5f 100755 --- a/deepspeed/autotuning/autotuner.py +++ b/deepspeed/autotuning/autotuner.py @@ -1,10 +1,5 @@ -import copy -import json -import os -from random import sample import shutil import subprocess -import hjson import torch import time import datetime @@ -13,10 +8,9 @@ from ..runtime.config_utils import dict_raise_error_on_duplicate_keys from ..runtime.constants import * from ..runtime.zero.constants import * -from ..utils import logger from .config import DeepSpeedAutotuningConfig from .constants import * -from .scheduler import ResourceManager, run_experiment +from .scheduler import ResourceManager from .tuner import GridSearchTuner, RandomTuner, ModelBasedTuner from .utils import * diff --git a/deepspeed/autotuning/scheduler.py b/deepspeed/autotuning/scheduler.py index e3c4fbe7708b..4f91f3cc98df 100755 --- a/deepspeed/autotuning/scheduler.py +++ b/deepspeed/autotuning/scheduler.py @@ -1,17 +1,11 @@ import copy -from re import I from numpy import BUFSIZE -from deepspeed.env_report import SUCCESS -from enum import Flag import json -import os import subprocess import sys import threading import time -from pathlib import Path -from typing import List import hjson from tqdm import tqdm @@ -27,8 +21,6 @@ from deepspeed import comm as dist -from datetime import datetime - TIMEOUT = 5 diff --git a/deepspeed/autotuning/tuner/base_tuner.py b/deepspeed/autotuning/tuner/base_tuner.py index fbdb16dacb53..fe00e27457e7 100755 --- a/deepspeed/autotuning/tuner/base_tuner.py +++ b/deepspeed/autotuning/tuner/base_tuner.py @@ -1,12 +1,9 @@ -import atexit import sys from deepspeed.autotuning.constants import * from deepspeed.autotuning.utils import write_experiments from deepspeed.utils import logger -import json - class BaseTuner: def __init__(self, exps, resource_manager, metric): diff --git a/deepspeed/autotuning/tuner/cost_model.py b/deepspeed/autotuning/tuner/cost_model.py index c311659426bf..0cdcef6483b4 100755 --- a/deepspeed/autotuning/tuner/cost_model.py +++ b/deepspeed/autotuning/tuner/cost_model.py @@ -1,5 +1,3 @@ -import numpy as np - from .utils import * try: diff --git a/deepspeed/autotuning/tuner/index_based_tuner.py b/deepspeed/autotuning/tuner/index_based_tuner.py index f7233f2e8d75..1685d799d44b 100755 --- a/deepspeed/autotuning/tuner/index_based_tuner.py +++ b/deepspeed/autotuning/tuner/index_based_tuner.py @@ -1,7 +1,5 @@ import random -from deepspeed.utils import logger - from .base_tuner import BaseTuner diff --git a/deepspeed/autotuning/tuner/model_based_tuner.py b/deepspeed/autotuning/tuner/model_based_tuner.py index d8bc2b499f3d..c3e0b1ac3fd1 100755 --- a/deepspeed/autotuning/tuner/model_based_tuner.py +++ b/deepspeed/autotuning/tuner/model_based_tuner.py @@ -1,8 +1,6 @@ import hjson -import numpy as np -from deepspeed.utils import logger -from ..constants import AUTOTUNING, AUTOTUNING_METRIC_PATH, AUTOTUNING_METRIC_DEFAULT +from ..constants import AUTOTUNING, AUTOTUNING_METRIC_PATH from .base_tuner import BaseTuner from .cost_model import XGBoostCostModel from .utils import * diff --git a/deepspeed/autotuning/utils.py b/deepspeed/autotuning/utils.py index 5cfa92cd9911..41184b4dacd6 100644 --- a/deepspeed/autotuning/utils.py +++ b/deepspeed/autotuning/utils.py @@ -3,8 +3,6 @@ import os import json from deepspeed.runtime.constants import GRADIENT_ACCUMULATION_STEPS, TRAIN_MICRO_BATCH_SIZE_PER_GPU -import hjson -import sys import itertools import copy @@ -35,23 +33,11 @@ def was_interruptted(filename): return False -def was_interruptted(filename): - if not os.path.exists(filename): - return "stderr.log does not exist" - with open(filename) as f: - for line in f: - s = "KeyboardInterrupt" - idx = line.find(s) - if idx != -1: - return True - return False - - def find_replace_str(value, replace_dict): if not isinstance(value, str): return str(value) - matches = re.findall("\$[A-Za-z0-9_]+", value) + matches = re.findall(r"\$[A-Za-z0-9_]+", value) for var in matches: var_key = var.replace("$", "").lower() if var_key == "nvme_path": diff --git a/deepspeed/comm/comm.py b/deepspeed/comm/comm.py index bb88e6addcb8..ac7ea8f55c90 100644 --- a/deepspeed/comm/comm.py +++ b/deepspeed/comm/comm.py @@ -28,7 +28,6 @@ from enum import Enum import torch import os -import time from ..constants import TORCH_DISTRIBUTED_DEFAULT_PORT, default_pg_timeout from .constants import * @@ -48,7 +47,6 @@ class ReduceOp(Enum): from deepspeed.utils.comms_logging import CommsLogger from deepspeed.utils import timer, get_caller_func -from deepspeed.comm.backend import Backend from deepspeed.comm.torch import TorchBackend from deepspeed import utils diff --git a/deepspeed/comm/config.py b/deepspeed/comm/config.py index 2c8ceaed2df9..d0238331de9e 100644 --- a/deepspeed/comm/config.py +++ b/deepspeed/comm/config.py @@ -3,9 +3,7 @@ Licensed under the MIT license. """ -from typing import Optional -from deepspeed.runtime.config_utils import get_scalar_param -from pydantic import BaseModel, validator, ValidationError, create_model +from pydantic import BaseModel from .constants import * diff --git a/deepspeed/comm/torch.py b/deepspeed/comm/torch.py index 6b4af25abd88..5aec37dfd7e7 100644 --- a/deepspeed/comm/torch.py +++ b/deepspeed/comm/torch.py @@ -1,12 +1,7 @@ ''' Copyright 2021 The Microsoft DeepSpeed Team ''' -import os -import torch - from deepspeed import utils -from ..constants import TORCH_DISTRIBUTED_DEFAULT_PORT, default_pg_timeout -from datetime import timedelta from .utils import * from .backend import * diff --git a/deepspeed/comm/utils.py b/deepspeed/comm/utils.py index 1af067e91750..0b903b0b9b3f 100644 --- a/deepspeed/comm/utils.py +++ b/deepspeed/comm/utils.py @@ -1,5 +1,4 @@ import os -import enum import torch import inspect from deepspeed.utils import get_caller_func @@ -145,7 +144,7 @@ def get_msg_size_from_args(func, *args, **kwargs): # Sum of tensor sizes for list colls such as torch's all_to_all # NOTE: msg_size for list colls will not be the actual size transmitted by a given MPI/NCCL call within the coll op. Instead, it's the total amount of data transmitted. if type(tensor_arg) is list: - return sum(x.element_size() * x.nelement() for x in func_args['tensor_list']) + return sum(x.element_size() * x.nelement() for x in tensor_arg) else: return tensor_arg.element_size() * tensor_arg.nelement() diff --git a/deepspeed/compression/basic_layer.py b/deepspeed/compression/basic_layer.py index 5a5209ad7691..7dffc8a94790 100644 --- a/deepspeed/compression/basic_layer.py +++ b/deepspeed/compression/basic_layer.py @@ -1,4 +1,3 @@ -import copy import torch import math from torch import nn diff --git a/deepspeed/compression/compress.py b/deepspeed/compression/compress.py index 459b17075526..84ee53aab921 100644 --- a/deepspeed/compression/compress.py +++ b/deepspeed/compression/compress.py @@ -1,4 +1,3 @@ -import torch import re from .helper import compression_preparation, fix_compression, recursive_getattr, is_module_compressible from .config import get_compression_config @@ -7,8 +6,6 @@ import os import json -from deepspeed.utils import logger - def check_deepspeed_config(config): if isinstance(config, dict): diff --git a/deepspeed/elasticity/elasticity.py b/deepspeed/elasticity/elasticity.py index e678d5ed836e..c17dab0319d9 100644 --- a/deepspeed/elasticity/elasticity.py +++ b/deepspeed/elasticity/elasticity.py @@ -2,7 +2,6 @@ Copyright 2020 The Microsoft DeepSpeed Team """ import os -import re import json import numpy as np @@ -11,8 +10,7 @@ from .config import ElasticityConfig, ElasticityConfigError, ElasticityError, \ ElasticityIncompatibleWorldSize from .constants import ELASTICITY, ENABLED, ENABLED_DEFAULT, LATEST_ELASTICITY_VERSION, \ - MINIMUM_DEEPSPEED_VERSION, IGNORE_NON_ELASTIC_BATCH_INFO, \ - IGNORE_NON_ELASTIC_BATCH_INFO_DEFAULT, DEEPSPEED_ELASTICITY_CONFIG + MINIMUM_DEEPSPEED_VERSION, DEEPSPEED_ELASTICITY_CONFIG from ..git_version_info import version as __version__ from ..utils import logger diff --git a/deepspeed/env_report.py b/deepspeed/env_report.py index aad183665870..c5949a8c4d31 100644 --- a/deepspeed/env_report.py +++ b/deepspeed/env_report.py @@ -4,7 +4,6 @@ import argparse from .ops.op_builder import ALL_OPS from .git_version_info import installed_ops, torch_info -from .ops import __compatible_ops__ as compatible_ops GREEN = '\033[92m' RED = '\033[91m' @@ -54,7 +53,7 @@ def op_report(verbose=True): def ninja_installed(): try: - import ninja + import ninja # noqa: F401 except ImportError: return False return True diff --git a/deepspeed/git_version_info.py b/deepspeed/git_version_info.py index a806475c397b..5cd6d9f2f940 100644 --- a/deepspeed/git_version_info.py +++ b/deepspeed/git_version_info.py @@ -1,6 +1,6 @@ try: # This is populated by setup.py - from .git_version_info_installed import * + from .git_version_info_installed import * # noqa: F401 except ModuleNotFoundError: import os if os.path.isfile('version.txt'): diff --git a/deepspeed/inference/engine.py b/deepspeed/inference/engine.py index 9bfe954aa528..173f2587de70 100755 --- a/deepspeed/inference/engine.py +++ b/deepspeed/inference/engine.py @@ -5,7 +5,6 @@ import os from deepspeed import comm as dist -from deepspeed.utils import groups from deepspeed.utils.logging import log_dist from torch.nn.modules import Module @@ -18,7 +17,6 @@ from ..comm.comm import init_distributed from ..pipe import PipelineModule from ..moe.utils import has_moe_layers -from ..moe.layer import MoE from ..runtime.zero import GatheredParameters from ..module_inject import LinearAllreduce, LinearLayer, Normalize, ReplaceWithTensorSlicing diff --git a/deepspeed/launcher/multinode_runner.py b/deepspeed/launcher/multinode_runner.py index 27f32af30a2e..6fb187cfde23 100644 --- a/deepspeed/launcher/multinode_runner.py +++ b/deepspeed/launcher/multinode_runner.py @@ -3,7 +3,7 @@ import shutil import subprocess import warnings -from shlex import quote, split +from shlex import split from abc import ABC, abstractmethod from ..utils import logger diff --git a/deepspeed/module_inject/layers.py b/deepspeed/module_inject/layers.py index fa71c8d7d4d0..1ae5bd5ccf9f 100644 --- a/deepspeed/module_inject/layers.py +++ b/deepspeed/module_inject/layers.py @@ -1,6 +1,4 @@ import torch -import deepspeed -import deepspeed.ops.transformer as transformer_inference from deepspeed import comm as dist from torch import nn from torch.nn import functional as F diff --git a/deepspeed/module_inject/load_checkpoint.py b/deepspeed/module_inject/load_checkpoint.py index f6722deb582b..1d05b6e853f6 100644 --- a/deepspeed/module_inject/load_checkpoint.py +++ b/deepspeed/module_inject/load_checkpoint.py @@ -1,10 +1,7 @@ -import deepspeed -import torch from torch import nn -from torch.nn import functional as F import deepspeed.ops.transformer as transformer_inference from ..runtime.zero import GatheredParameters -from .layers import LinearAllreduce, LinearLayer, Normalize, EmbeddingLayer +from .layers import LinearLayer, Normalize, EmbeddingLayer def load_model_with_checkpoint(r_module, sd, mp_replace): diff --git a/deepspeed/module_inject/module_quantize.py b/deepspeed/module_inject/module_quantize.py index fde6990eba28..8485c0451214 100755 --- a/deepspeed/module_inject/module_quantize.py +++ b/deepspeed/module_inject/module_quantize.py @@ -1,6 +1,4 @@ -import copy import torch -import deepspeed def quantize_transformer_layer(orig_layer_impl, model, megatron=False, preln=False): diff --git a/deepspeed/module_inject/replace_module.py b/deepspeed/module_inject/replace_module.py index fe83b7b3b4cf..151abfacf004 100755 --- a/deepspeed/module_inject/replace_module.py +++ b/deepspeed/module_inject/replace_module.py @@ -1,19 +1,15 @@ -import copy import torch import tqdm import deepspeed import deepspeed.ops.transformer as transformer_inference -from .replace_policy import HFBertLayerPolicy, HFGPT2LayerPolicy, HFGPTJLayerPolicy, BLOOMLayerPolicy +from .replace_policy import HFBertLayerPolicy, HFGPT2LayerPolicy, BLOOMLayerPolicy from .replace_policy import replace_policies -from ..constants import INFERENCE_GENERIC_MODE, INFERENCE_SPECIALIZED_MODE from ..runtime.weight_quantizer import WeightQuantization -from torch import nn from deepspeed import comm as dist from torch import nn -from torch.nn import functional as F from ..runtime.zero import GatheredParameters -from .layers import LinearAllreduce, LinearLayer, Normalize, EmbeddingLayer +from .layers import LinearAllreduce, LinearLayer from .load_checkpoint import load_model_with_checkpoint @@ -452,10 +448,12 @@ def _transpose(x): if qkvw.is_meta or qkvw.ds_tensor.numel() < attn_block.attn_qkvw.numel(): pass else: - with GatheredParameters([attn_qkvw, - attn_qkvb, - attn_ow, - attn_ob], + with GatheredParameters([ + attn_block.attn_qkvw, + attn_block.attn_qkvb, + attn_block.attn_ow, + attn_block.attn_ob + ], modifier_rank=0): attn_block.attn_qkvw = mp_replace.copy( attn_block.attn_qkvw, @@ -680,12 +678,13 @@ def _replace(child, name, conv_linear_layer): def _slice_embedding(child, name, conv_linear_layer): mp_replace = ReplaceWithTensorSlicing(mp_group=mp_group) - new_weight = torch.empty((weight_shape[0], - weight_shape[1] // mp_size), + new_weight = torch.empty((child.weight.shape[0], + child.weight.shape[1] // mp_size), device=child.weight.device, dtype=child.weight.dtype) data = mp_replace.copy(new_weight, child.weight.ds_tensor.data) - new_embedding = nn.Embedding(weight_shape[0], weight_shape[1] // mp_size) + new_embedding = nn.Embedding(child.weight.shape[0], + child.weight.shape[1] // mp_size) new_embedding.weight.data.copy_(data) return new_embedding diff --git a/deepspeed/module_inject/replace_policy.py b/deepspeed/module_inject/replace_policy.py index e4b1195d9737..eeb6d613969b 100755 --- a/deepspeed/module_inject/replace_policy.py +++ b/deepspeed/module_inject/replace_policy.py @@ -215,7 +215,6 @@ def __init__(self, client_module, inference=True): MegatronLayerPolicy._orig_layer_class = None else: try: - import megatron from megatron.model.transformer import ParallelTransformerLayer MegatronLayerPolicy._orig_layer_class = ParallelTransformerLayer except ImportError: @@ -372,7 +371,6 @@ def __init__(self, client_module, inference=True, megatron_v2=True): GPTNEOXLayerPolicy._orig_layer_class = None else: try: - import transformers from transformers import GPTNeoXLayer GPTNEOXLayerPolicy._orig_layer_class = GPTNeoXLayer except ImportError: diff --git a/deepspeed/moe/layer.py b/deepspeed/moe/layer.py index 399f512bfead..fd847308b21e 100644 --- a/deepspeed/moe/layer.py +++ b/deepspeed/moe/layer.py @@ -2,16 +2,13 @@ Copyright 2020 The Microsoft DeepSpeed Team ''' -import torch.nn.init as init import torch -from deepspeed import comm as dist -from deepspeed.utils import logger, log_dist +from deepspeed.utils import log_dist from deepspeed.utils import groups from .sharded_moe import MOELayer, TopKGate from .experts import Experts -import copy import typing diff --git a/deepspeed/moe/sharded_moe.py b/deepspeed/moe/sharded_moe.py index d87d22dc7ff7..f9da6eb9d350 100644 --- a/deepspeed/moe/sharded_moe.py +++ b/deepspeed/moe/sharded_moe.py @@ -12,16 +12,13 @@ # This source code is licensed under the BSD license found in the # LICENSE file in the root directory of this source tree. -from deepspeed.utils.timer import ThroughputTimer, SynchronizedWallClockTimer -from deepspeed.utils import logger, log_dist -from typing import Callable, Dict, TYPE_CHECKING, Any, Optional, Tuple, Union, cast +from deepspeed.utils.timer import SynchronizedWallClockTimer +from deepspeed.utils import logger +from typing import Callable, Dict, TYPE_CHECKING, Any, Optional, Tuple -import time -from time import perf_counter import torch from torch import Tensor -from deepspeed import comm as dist -from torch.nn import Module, ModuleList +from torch.nn import Module import torch.nn.functional as F if TYPE_CHECKING: diff --git a/deepspeed/moe/utils.py b/deepspeed/moe/utils.py index 717947226165..043d2626d43c 100644 --- a/deepspeed/moe/utils.py +++ b/deepspeed/moe/utils.py @@ -1,6 +1,5 @@ from typing import List, Tuple, Dict import torch -from deepspeed.utils import groups from .layer import MoE diff --git a/deepspeed/monitor/config.py b/deepspeed/monitor/config.py index b854b8580023..709830f27e98 100644 --- a/deepspeed/monitor/config.py +++ b/deepspeed/monitor/config.py @@ -3,9 +3,7 @@ Licensed under the MIT license. """ -from typing import Optional -from deepspeed.runtime.config_utils import get_scalar_param -from pydantic import BaseModel, validator, ValidationError, create_model +from pydantic import BaseModel from .constants import * diff --git a/deepspeed/monitor/csv_monitor.py b/deepspeed/monitor/csv_monitor.py index b2b05260e445..1425f1d56f66 100644 --- a/deepspeed/monitor/csv_monitor.py +++ b/deepspeed/monitor/csv_monitor.py @@ -7,7 +7,6 @@ class csvMonitor(Monitor): def __init__(self, monitor_config): super().__init__(monitor_config) - import csv self.filenames = [] self.enabled = monitor_config.csv_monitor_config.enabled self.output_path = monitor_config.csv_monitor_config.output_path diff --git a/deepspeed/monitor/utils.py b/deepspeed/monitor/utils.py index f519a71823a9..0733268f3873 100644 --- a/deepspeed/monitor/utils.py +++ b/deepspeed/monitor/utils.py @@ -2,7 +2,7 @@ def check_tb_availability(): try: # torch.utils.tensorboard will fail if `tensorboard` is not available, # see their docs for more details: https://pytorch.org/docs/1.8.0/tensorboard.html - import tensorboard + import tensorboard # noqa: F401 except ImportError: print('If you want to use tensorboard logging, please `pip install tensorboard`') raise @@ -10,7 +10,7 @@ def check_tb_availability(): def check_wandb_availability(): try: - import wandb + import wandb # noqa: F401 except ImportError: print( 'If you want to use wandb logging, please `pip install wandb` and follow the instructions at https://docs.wandb.ai/quickstart' diff --git a/deepspeed/ops/adagrad/cpu_adagrad.py b/deepspeed/ops/adagrad/cpu_adagrad.py index e3f70a61539f..2527259b1382 100755 --- a/deepspeed/ops/adagrad/cpu_adagrad.py +++ b/deepspeed/ops/adagrad/cpu_adagrad.py @@ -2,10 +2,7 @@ Copyright 2020 The Microsoft DeepSpeed Team ''' -import math import torch -import time -from pathlib import Path from ..op_builder import CPUAdagradBuilder from deepspeed.utils.logging import should_log_le diff --git a/deepspeed/ops/adam/cpu_adam.py b/deepspeed/ops/adam/cpu_adam.py index e5f7aa70c364..911e4924dfbc 100755 --- a/deepspeed/ops/adam/cpu_adam.py +++ b/deepspeed/ops/adam/cpu_adam.py @@ -2,11 +2,8 @@ Copyright 2020 The Microsoft DeepSpeed Team ''' -import math import torch -import time from cpuinfo import get_cpu_info -from pathlib import Path from ..op_builder import CPUAdamBuilder from deepspeed.utils import logger from deepspeed.utils.logging import should_log_le diff --git a/deepspeed/ops/adam/fused_adam.py b/deepspeed/ops/adam/fused_adam.py index 3f3e5919feaf..5a1a1ddcaed3 100644 --- a/deepspeed/ops/adam/fused_adam.py +++ b/deepspeed/ops/adam/fused_adam.py @@ -6,7 +6,6 @@ ''' import torch -import importlib from .multi_tensor_apply import MultiTensorApply multi_tensor_applier = MultiTensorApply(2048 * 32) diff --git a/deepspeed/ops/adam/multi_tensor_apply.py b/deepspeed/ops/adam/multi_tensor_apply.py index 173288ee76ab..e837309be629 100644 --- a/deepspeed/ops/adam/multi_tensor_apply.py +++ b/deepspeed/ops/adam/multi_tensor_apply.py @@ -4,7 +4,6 @@ Copyright NVIDIA/apex This file is adapted from NVIDIA/apex, commit a109f85 ''' -import torch class MultiTensorApply(object): diff --git a/deepspeed/ops/quantizer/quantizer.py b/deepspeed/ops/quantizer/quantizer.py index cea9434049d9..6b25d02d87e7 100755 --- a/deepspeed/ops/quantizer/quantizer.py +++ b/deepspeed/ops/quantizer/quantizer.py @@ -1,12 +1,7 @@ ''' Copyright 2020 The Microsoft DeepSpeed Team ''' -import json -import math -import importlib import torch -from torch import nn -from torch.autograd import Function from ..op_builder import QuantizerBuilder diff --git a/deepspeed/ops/sparse_attention/matmul.py b/deepspeed/ops/sparse_attention/matmul.py index ea83f093c748..ca933b989036 100755 --- a/deepspeed/ops/sparse_attention/matmul.py +++ b/deepspeed/ops/sparse_attention/matmul.py @@ -1,9 +1,7 @@ # DeepSpeed note, code taken & adapted from commit 9aa94789f13ada713af36cfd8cca2fc9a7f6b79a # https://github.com/ptillet/torch-blocksparse/blob/master/torch_blocksparse/matmul.py import importlib -import warnings import torch -import math import triton import triton.language as tl @@ -339,7 +337,7 @@ def _sdd_matmul(a, a_inner, b_inner = a.shape[a_dim], b.shape[b_dim] if a_inner != b_inner: raise ValueError( - f"Size of tensor A along the {_dim_to_name(a_dim)} dim ({a_inner}) must match size " + f"Size of tensor A along the {_dim_to_name(a_dim)} dim ({a_inner}) must match size " # noqa: F821 f"of tensor B along the {_dim_to_name(b_dim)} dim ({b_inner})") if a_inner % 16 != 0: raise ValueError('Reduction size for SDD must be a multiple of 16') diff --git a/deepspeed/ops/sparse_attention/softmax.py b/deepspeed/ops/sparse_attention/softmax.py index 11d4583fd619..ce155105988f 100755 --- a/deepspeed/ops/sparse_attention/softmax.py +++ b/deepspeed/ops/sparse_attention/softmax.py @@ -1,14 +1,10 @@ # DeepSpeed note, code taken & adapted from commit 9aa94789f13ada713af36cfd8cca2fc9a7f6b79a # https://github.com/ptillet/torch-blocksparse/blob/master/torch_blocksparse/matmul.py -import warnings -import importlib import torch -import math import triton import triton.language as tl -import triton._C.libtriton as libtriton def next_power_of_2(n): diff --git a/deepspeed/ops/sparse_attention/sparse_attention_utils.py b/deepspeed/ops/sparse_attention/sparse_attention_utils.py index 700363b204af..90edb10fd84a 100644 --- a/deepspeed/ops/sparse_attention/sparse_attention_utils.py +++ b/deepspeed/ops/sparse_attention/sparse_attention_utils.py @@ -2,7 +2,7 @@ Copyright 2020 The Microsoft DeepSpeed Team """ -from torch import nn +import torch from torch.nn import functional as F from deepspeed.ops.sparse_attention import BertSparseSelfAttention, SparsityConfig ''' @@ -102,13 +102,13 @@ def replace_model_self_attention_with_sparse_self_attention( if hasattr(model, 'bert'): model.config.max_position_embeddings = max_position - replace_self_attention_layer_with_sparse_self_attention_layer( + model.replace_self_attention_layer_with_sparse_self_attention_layer( model.config, model.bert.encoder.layer, sparsity_config) elif hasattr(model, 'roberta'): model.config.max_position_embeddings = max_position + 2 - replace_self_attention_layer_with_sparse_self_attention_layer( + model.replace_self_attention_layer_with_sparse_self_attention_layer( model.config, model.roberta.encoder.layer, sparsity_config) @@ -155,7 +155,7 @@ def pad_to_block_size(block_size, position_ids, inputs_embeds, pad_token_id, - model_mbeddings): + model_embeddings): """This function pads input tokens and attention mask on sequence length dimension to be multiple of block size. This is a requirement for Sparse Transformer in which the self attention layer works on sequences of length multiple of block size. It needs to be called in your model, such as BertModel, right before you calculate the embedding outputs. diff --git a/deepspeed/ops/sparse_attention/sparse_self_attention.py b/deepspeed/ops/sparse_attention/sparse_self_attention.py index 5ccfa13198fd..46dedd0779f9 100644 --- a/deepspeed/ops/sparse_attention/sparse_self_attention.py +++ b/deepspeed/ops/sparse_attention/sparse_self_attention.py @@ -3,10 +3,8 @@ """ import torch.nn as nn -from torch.nn.functional import * import torch from torch import distributed as dist -from collections import namedtuple from deepspeed.ops.sparse_attention import SparsityConfig diff --git a/deepspeed/ops/transformer/inference/moe_inference.py b/deepspeed/ops/transformer/inference/moe_inference.py index 830110cc5f05..ca4b5b7a9702 100644 --- a/deepspeed/ops/transformer/inference/moe_inference.py +++ b/deepspeed/ops/transformer/inference/moe_inference.py @@ -3,11 +3,8 @@ ''' import json import math -import importlib import torch -from torch import nn from torch.autograd import Function -import time from ... import op_builder #from ...inference.engine import inference_cuda_module, specialized_mode # Cuda modules will be imported if needed @@ -18,8 +15,6 @@ from ....moe.sharded_moe import TopKGate from deepspeed import comm as dist -import torch.nn.functional as F - class DeepSpeedMoEInferenceConfig(DeepSpeedInferenceConfig): """Initialize the DeepSpeed Transformer Config. diff --git a/deepspeed/ops/transformer/inference/transformer_inference.py b/deepspeed/ops/transformer/inference/transformer_inference.py index 85857e0e8a4d..81762c10b014 100755 --- a/deepspeed/ops/transformer/inference/transformer_inference.py +++ b/deepspeed/ops/transformer/inference/transformer_inference.py @@ -3,11 +3,8 @@ ''' import json import math -import importlib import torch -from torch import nn from torch.autograd import Function -import time from ... import op_builder import torch.nn as nn from deepspeed import comm as dist @@ -338,7 +335,7 @@ def compute_attention(qkv_out, input_mask): torch.empty(1), num_attention_heads_per_partition, (1 / norm_factor if config.scale_attention else 1.0), - (not unfused_mode), + (not unfused_mode), # noqa: F821 config.triangular_masking, config.local_attention, config.window_size, @@ -346,21 +343,21 @@ def compute_attention(qkv_out, input_mask): else: attn_key_value = score_context_func( mixed_query, - (key_layer if unfused_mode else past_key.type_as(key_layer)), + (key_layer if unfused_mode else past_key.type_as(key_layer)), # noqa: F821 key_layer, ((1 - input_mask).half() * minus_inf) if input_mask.dtype == torch.int64 else input_mask, (value_layer - if unfused_mode else past_value.type_as(value_layer)), + if unfused_mode else past_value.type_as(value_layer)), # noqa: F821 value_layer, num_attention_heads_per_partition, (1 / norm_factor if config.scale_attention else 1.0), - (not unfused_mode), + (not unfused_mode), # noqa: F821 config.triangular_masking, config.local_attention, config.window_size, no_masking) - if unfused_mode: + if unfused_mode: # noqa: F821 context_layer, _, _ = attn_key_value else: context_layer, key_layer, value_layer = attn_key_value diff --git a/deepspeed/ops/transformer/transformer.py b/deepspeed/ops/transformer/transformer.py index 084587ba2a3b..7963d11774e3 100755 --- a/deepspeed/ops/transformer/transformer.py +++ b/deepspeed/ops/transformer/transformer.py @@ -3,7 +3,6 @@ ''' import json import math -import importlib import torch from torch import nn from torch.autograd import Function diff --git a/deepspeed/profiling/flops_profiler/profiler.py b/deepspeed/profiling/flops_profiler/profiler.py index 7fbfb19c777f..4dd73c189481 100644 --- a/deepspeed/profiling/flops_profiler/profiler.py +++ b/deepspeed/profiling/flops_profiler/profiler.py @@ -3,7 +3,7 @@ import torch.nn as nn import torch.nn.functional as F from functools import partial -from typing import Callable, List, Optional, Tuple +from typing import List, Optional from collections import OrderedDict import numpy as np @@ -251,7 +251,6 @@ def print_model_profile(self, return import sys import os.path - from os import path original_stdout = None f = None if output_file and output_file != "": diff --git a/deepspeed/runtime/bf16_optimizer.py b/deepspeed/runtime/bf16_optimizer.py index d5ffbac9d1d3..35f8bbd3473a 100644 --- a/deepspeed/runtime/bf16_optimizer.py +++ b/deepspeed/runtime/bf16_optimizer.py @@ -12,7 +12,6 @@ from packaging import version as pkg_version from deepspeed.git_version_info import version -from deepspeed.runtime.swap_tensor.partitioned_param_swapper import print_rank_0 from deepspeed.runtime.utils import (get_global_norm_of_tensors, clip_tensors_by_global_norm, DummyOptim, diff --git a/deepspeed/runtime/comm/coalesced_collectives.py b/deepspeed/runtime/comm/coalesced_collectives.py index 6a2b8e31516d..a18c3a55ad2d 100644 --- a/deepspeed/runtime/comm/coalesced_collectives.py +++ b/deepspeed/runtime/comm/coalesced_collectives.py @@ -12,7 +12,6 @@ import torch.nn.functional from deepspeed.utils import instrument_w_nvtx -from deepspeed.utils import logger def _torch_reduce_scatter_fn(input_tensor: Tensor, diff --git a/deepspeed/runtime/comm/nccl.py b/deepspeed/runtime/comm/nccl.py index ed80059a9067..c892316a3dc5 100644 --- a/deepspeed/runtime/comm/nccl.py +++ b/deepspeed/runtime/comm/nccl.py @@ -4,7 +4,6 @@ import torch from deepspeed import comm as dist -import time import cupy import numpy as np diff --git a/deepspeed/runtime/engine.py b/deepspeed/runtime/engine.py index bf37ab71adc3..39c214c791d9 100644 --- a/deepspeed/runtime/engine.py +++ b/deepspeed/runtime/engine.py @@ -5,9 +5,7 @@ import os import re import stat -import math import torch -import warnings import hashlib from collections import defaultdict, OrderedDict from shutil import copyfile @@ -17,21 +15,16 @@ from torch.optim import Optimizer from torch.optim.lr_scheduler import _LRScheduler -from typing import Callable, Dict, Optional, Union, Iterable -from deepspeed.checkpoint.utils import get_zero_ckpt_name_for_rank +from typing import Callable, Dict, Union, Iterable import deepspeed -from deepspeed.runtime.utils import see_memory_usage, get_ma_status, DummyOptim +from deepspeed.runtime.utils import see_memory_usage, DummyOptim from deepspeed.runtime.zero.stage_1_and_2 import DeepSpeedZeroOptimizer from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus from deepspeed.runtime.zero.utils import is_zero_supported_optimizer, ZeRORuntimeException from deepspeed.runtime.zero.parameter_offload import DeepSpeedZeRoOffload -from deepspeed.runtime.activation_checkpointing import ( - checkpointing as activation_checkpointing, -) - from deepspeed.runtime.fp16.fused_optimizer import FP16_Optimizer from deepspeed.runtime.fp16.unfused_optimizer import FP16_UnfusedOptimizer from deepspeed.runtime.bf16_optimizer import BF16_Optimizer @@ -47,23 +40,16 @@ from deepspeed.compression import compression_scheduler from deepspeed.compression.constants import \ - SHARED_PARAMETERS, \ WEIGHT_QUANTIZE_IN_FORWARD_ENABLED, \ - WEIGHT_QUANTIZATION, SHARED_PARAMETERS, WEIGHT_QUANTIZE_ENABLED, \ + WEIGHT_QUANTIZATION, SHARED_PARAMETERS, \ WEIGHT_QUANTIZE_ENABLED, \ - WEIGHT_QUANTIZE_SCHEDULE_OFFSET, \ WEIGHT_QUANTIZE_GROUPS, \ WEIGHT_QUANTIZE_FP16_MIXED_QUANTIZE, \ WEIGHT_QUANTIZE_CHANGE_RATIO, \ WEIGHT_QUANTIZE_TYPE, \ WEIGHT_QUANTIZE_ROUNDING, \ WEIGHT_QUANTIZE_VERBOSE, \ - WEIGHT_QUANTIZE_KERNEL, \ - ACTIVATION_QUANTIZATION, \ - SPARSE_PRUNING, \ - ROW_PRUNING, \ - HEAD_PRUNING, \ - CHANNEL_PRUNING + WEIGHT_QUANTIZE_KERNEL from deepspeed.runtime.zero.constants import \ ZERO_OPTIMIZATION_OPTIMIZER_STATES, ZERO_OPTIMIZATION_GRADIENTS, ZERO_OPTIMIZATION_WEIGHTS @@ -72,9 +58,7 @@ from deepspeed.runtime import lr_schedules from deepspeed.utils import groups -from deepspeed.runtime.utils import get_grad_norm from deepspeed.utils import logger, log_dist, instrument_w_nvtx -from deepspeed.comm.comm import init_distributed from deepspeed.utils.timer import ThroughputTimer, SynchronizedWallClockTimer from deepspeed.utils.debug import debug_extract_module_and_param_names from deepspeed.monitor.monitor import MonitorMaster @@ -86,7 +70,6 @@ from .pipe.module import PipelineModule from .utils import ensure_directory_exists, get_ma_status from ..ops.op_builder import UtilsBuilder -from ..ops.adam import DeepSpeedCPUAdam from ..ops.adam import FusedAdam from ..moe.sharded_moe import TopKGate, MOELayer from ..moe.layer import MoE diff --git a/deepspeed/runtime/fp16/fused_optimizer.py b/deepspeed/runtime/fp16/fused_optimizer.py index 479a0f7a2839..aeed2f4b18e1 100755 --- a/deepspeed/runtime/fp16/fused_optimizer.py +++ b/deepspeed/runtime/fp16/fused_optimizer.py @@ -181,7 +181,7 @@ def step_fused_adam(self, closure=None): apply_scale=False) # Stash unscaled gradient norm - self._global_grad_norm = scaled_global_grad_norm / self.cur_scale + self._global_grad_norm = scaled_grad_norm / self.cur_scale # norm is in fact norm*cur_scale self.optimizer.step(grads=[[g] for g in grads_groups_flat], diff --git a/deepspeed/runtime/fp16/onebit/adam.py b/deepspeed/runtime/fp16/onebit/adam.py index 71805176ae41..706d2a3dac1f 100644 --- a/deepspeed/runtime/fp16/onebit/adam.py +++ b/deepspeed/runtime/fp16/onebit/adam.py @@ -3,13 +3,9 @@ ''' import types import torch -import importlib import numpy as np -import time from deepspeed import comm as dist -from deepspeed.utils.logging import logger - class OnebitAdam(torch.optim.Optimizer): """Implements the 1-bit Adam algorithm. Currently GPU-only. @@ -37,7 +33,7 @@ class OnebitAdam(torch.optim.Optimizer): cuda_aware (boolean, required): Set True if the underlying MPI implementation supports CUDA-Aware communication. (default: False) comm_backend_name (string, optional): Set to 'mpi' if needed. (default: 'nccl') - .. _Adam\: A Method for Stochastic Optimization: + .. _Adam\\: A Method for Stochastic Optimization: https://arxiv.org/abs/1412.6980 .. _On the Convergence of Adam and Beyond: https://openreview.net/forum?id=ryQu7f-RZ diff --git a/deepspeed/runtime/fp16/onebit/lamb.py b/deepspeed/runtime/fp16/onebit/lamb.py index aeff08b9861b..696550ca41ba 100644 --- a/deepspeed/runtime/fp16/onebit/lamb.py +++ b/deepspeed/runtime/fp16/onebit/lamb.py @@ -46,9 +46,9 @@ class OnebitLamb(torch.optim.Optimizer): coefficient during compression stage (default: 0.5) factor_threshold (float, optional): threshold of how much the scaling factor can fluctuate between steps (default: 0.1) - .. _Large Batch Optimization for Deep Learning\: Training BERT in 76 minutes: + .. _Large Batch Optimization for Deep Learning\\: Training BERT in 76 minutes: https://arxiv.org/abs/1904.00962 - .. _Adam\: A Method for Stochastic Optimization: + .. _Adam\\: A Method for Stochastic Optimization: https://arxiv.org/abs/1412.6980 .. _On the Convergence of Adam and Beyond: https://openreview.net/forum?id=ryQu7f-RZ diff --git a/deepspeed/runtime/fp16/onebit/zoadam.py b/deepspeed/runtime/fp16/onebit/zoadam.py index 11ccb404a9ff..53dfde99ae38 100644 --- a/deepspeed/runtime/fp16/onebit/zoadam.py +++ b/deepspeed/runtime/fp16/onebit/zoadam.py @@ -3,13 +3,9 @@ ''' import types import torch -import importlib import numpy as np -import time from deepspeed import comm as dist -from deepspeed.utils.logging import logger - class ZeroOneAdam(torch.optim.Optimizer): """Implements the 0/1 Adam algorithm. Currently GPU-only. @@ -47,7 +43,7 @@ class ZeroOneAdam(torch.optim.Optimizer): cuda_aware (boolean, required): Set True if the underlying MPI implementation supports CUDA-Aware communication. (default: False) comm_backend_name (string, optional): Set to 'mpi' if needed. (default: 'nccl') - .. _Adam\: A Method for Stochastic Optimization: + .. _Adam\\: A Method for Stochastic Optimization: https://arxiv.org/abs/1412.6980 .. _On the Convergence of Adam and Beyond: https://openreview.net/forum?id=ryQu7f-RZ diff --git a/deepspeed/runtime/lr_schedules.py b/deepspeed/runtime/lr_schedules.py index b4dc749ae193..faf5e6fee910 100755 --- a/deepspeed/runtime/lr_schedules.py +++ b/deepspeed/runtime/lr_schedules.py @@ -10,9 +10,7 @@ import argparse from torch.optim import Optimizer -from typing import Union, List import math -from deepspeed.runtime.constants import * from deepspeed.utils import logger LR_SCHEDULE = 'lr_schedule' diff --git a/deepspeed/runtime/pipe/engine.py b/deepspeed/runtime/pipe/engine.py index 43c65b15b525..a47f1e45b241 100644 --- a/deepspeed/runtime/pipe/engine.py +++ b/deepspeed/runtime/pipe/engine.py @@ -1,28 +1,18 @@ # Copyright 2019 The Microsoft DeepSpeed Team -import time -import logging -import copy -import os - from types import MethodType -from numpy import prod - import torch -import torch.nn as nn -import torch.optim as optim from deepspeed import comm as dist from deepspeed.utils import logger -from deepspeed.utils.timer import SynchronizedWallClockTimer, ThroughputTimer +from deepspeed.utils.timer import ThroughputTimer -from deepspeed.inference.engine import InferenceEngine from ..engine import DeepSpeedEngine, MEMORY_OPT_ALLREDUCE_SIZE -from ..utils import PartitionedTensor, ensure_directory_exists +from ..utils import PartitionedTensor from ..dataloader import RepeatingLoader -from .module import PipelineModule, PipelineError, TiedLayerSpec +from .module import PipelineModule, PipelineError from . import p2p from . import schedule @@ -587,6 +577,11 @@ def set_dataiterator(self, iterator): self.data_iterator = iterator def set_batch_fn(self, fn): + """Execute a post-processing function on input data. + + Args: + fn (function): The function to run. + """ self.batch_fn = fn def is_gradient_accumulation_boundary(self): @@ -1373,11 +1368,3 @@ def _exec_schedule(self, pipe_schedule): # Equivalent to: self._exec_forward_pass(buffer_id=0) self._exec_instr = MethodType(self._INSTRUCTION_MAP[type(cmd)], self) self._exec_instr(**cmd.kwargs) - - def set_batch_fn(self, fn): - """Execute a post-processing function on input data. - - Args: - fn (function): The function to run. - """ - self.batch_fn = fn diff --git a/deepspeed/runtime/pipe/module.py b/deepspeed/runtime/pipe/module.py index f47806ee8673..ac35a9fa2cf8 100644 --- a/deepspeed/runtime/pipe/module.py +++ b/deepspeed/runtime/pipe/module.py @@ -1,10 +1,8 @@ import os import glob -import enum import re as regex -from collections import defaultdict from functools import partial import torch diff --git a/deepspeed/runtime/pipe/topology.py b/deepspeed/runtime/pipe/topology.py index 954e73592943..6c0cd96440fb 100644 --- a/deepspeed/runtime/pipe/topology.py +++ b/deepspeed/runtime/pipe/topology.py @@ -1,9 +1,6 @@ # Copyright 2019 The Microsoft DeepSpeed Team -from deepspeed.utils import logger - from deepspeed import comm as dist -import sys from collections import namedtuple from itertools import product as cartesian_product @@ -58,7 +55,7 @@ def get_rank(self, **coord_kwargs): raise ValueError('get_rank() does not support slices. Use filter_match())') key = self.ProcessCoord(**coord_kwargs) - assert key in self.mapping, f'key {kwargs} invalid' + assert key in self.mapping, f'key {coord_kwargs} invalid' return self.mapping[key] def get_axis_names(self): diff --git a/deepspeed/runtime/quantize.py b/deepspeed/runtime/quantize.py index d264ddf2a459..7d27fd210f34 100755 --- a/deepspeed/runtime/quantize.py +++ b/deepspeed/runtime/quantize.py @@ -1,7 +1,5 @@ -import pdb import torch import math -from deepspeed.utils import log_dist from deepspeed.utils import logger from deepspeed.ops.quantizer import ds_quantizer diff --git a/deepspeed/runtime/swap_tensor/optimizer_utils.py b/deepspeed/runtime/swap_tensor/optimizer_utils.py index f34ff3a457a8..1323dcb926e5 100644 --- a/deepspeed/runtime/swap_tensor/optimizer_utils.py +++ b/deepspeed/runtime/swap_tensor/optimizer_utils.py @@ -13,7 +13,7 @@ from deepspeed.runtime.zero.offload_constants import * from deepspeed.runtime.swap_tensor.constants import * from deepspeed.runtime.swap_tensor.utils import swap_in_tensors, swap_out_tensors, \ - MIN_AIO_BYTES, AIO_ALIGNED_BYTES, get_sized_buffers, get_sized_buffer + MIN_AIO_BYTES, AIO_ALIGNED_BYTES, get_sized_buffers from deepspeed.runtime.swap_tensor.utils import SwapBufferManager, SwapBufferPool diff --git a/deepspeed/runtime/swap_tensor/partitioned_optimizer_swapper.py b/deepspeed/runtime/swap_tensor/partitioned_optimizer_swapper.py index 124500de888b..c83a69544d56 100644 --- a/deepspeed/runtime/swap_tensor/partitioned_optimizer_swapper.py +++ b/deepspeed/runtime/swap_tensor/partitioned_optimizer_swapper.py @@ -5,7 +5,6 @@ Functionality of swapping optimizer tensors to/from (NVMe) storage devices. """ -import os import torch from deepspeed.utils.logging import logger @@ -14,7 +13,7 @@ from deepspeed.runtime.swap_tensor.constants import * from deepspeed.runtime.swap_tensor.utils import swap_in_tensors, swap_out_tensors, print_object, \ - MIN_AIO_BYTES, AIO_ALIGNED_BYTES, get_sized_buffers, get_sized_buffer + get_sized_buffers from deepspeed.runtime.swap_tensor.async_swapper import AsyncTensorSwapper from deepspeed.runtime.swap_tensor.optimizer_utils import OptimizerSwapper diff --git a/deepspeed/runtime/swap_tensor/partitioned_param_swapper.py b/deepspeed/runtime/swap_tensor/partitioned_param_swapper.py index c488b84692cd..efce4f96922c 100644 --- a/deepspeed/runtime/swap_tensor/partitioned_param_swapper.py +++ b/deepspeed/runtime/swap_tensor/partitioned_param_swapper.py @@ -11,7 +11,6 @@ import torch from deepspeed import comm as dist -from deepspeed.utils.logging import logger from deepspeed.ops.aio import AsyncIOBuilder from .constants import * from .utils import swap_in_tensors, swap_out_tensors, MIN_AIO_BYTES, AIO_ALIGNED_BYTES, print_object, SwapBufferPool diff --git a/deepspeed/runtime/swap_tensor/pipelined_optimizer_swapper.py b/deepspeed/runtime/swap_tensor/pipelined_optimizer_swapper.py index 598585078632..cbcfa15ca598 100644 --- a/deepspeed/runtime/swap_tensor/pipelined_optimizer_swapper.py +++ b/deepspeed/runtime/swap_tensor/pipelined_optimizer_swapper.py @@ -5,19 +5,14 @@ Functionality of swapping optimizer tensors to/from (NVMe) storage devices. """ -import os -import torch - -from deepspeed.utils.logging import logger from deepspeed.ops.aio import AsyncIOBuilder from deepspeed import comm as dist from deepspeed.runtime.zero.offload_constants import * from deepspeed.runtime.swap_tensor.constants import * -from deepspeed.runtime.swap_tensor.utils import swap_in_tensors, swap_out_tensors, print_object, \ - MIN_AIO_BYTES, AIO_ALIGNED_BYTES +from deepspeed.runtime.swap_tensor.utils import swap_in_tensors, swap_out_tensors, print_object from deepspeed.runtime.swap_tensor.async_swapper import AsyncTensorSwapper -from deepspeed.runtime.swap_tensor.optimizer_utils import SwapBufferManager, get_sized_buffer +from deepspeed.runtime.swap_tensor.utils import get_sized_buffer from deepspeed.runtime.swap_tensor.optimizer_utils import OptimizerSwapper diff --git a/deepspeed/runtime/swap_tensor/utils.py b/deepspeed/runtime/swap_tensor/utils.py index 9d544e3d9b11..3059e0d15798 100644 --- a/deepspeed/runtime/swap_tensor/utils.py +++ b/deepspeed/runtime/swap_tensor/utils.py @@ -5,15 +5,11 @@ Functionality of swapping tensors to/from (NVMe) storage devices. """ -import os import torch from deepspeed.utils.logging import logger from deepspeed import comm as dist -from deepspeed.runtime.swap_tensor.constants import AIO_BLOCK_SIZE, AIO_QUEUE_DEPTH, \ - AIO_THREAD_COUNT, AIO_SINGLE_SUBMIT, AIO_OVERLAP_EVENTS - MIN_AIO_BYTES = 1024**2 AIO_ALIGNED_BYTES = 1024 diff --git a/deepspeed/runtime/utils.py b/deepspeed/runtime/utils.py index 69660ac3c272..fa90486cf8d5 100755 --- a/deepspeed/runtime/utils.py +++ b/deepspeed/runtime/utils.py @@ -7,13 +7,13 @@ ''' from collections.abc import Iterable -from deepspeed.moe.utils import is_moe_param, split_params_into_shared_and_expert_params +from deepspeed.moe.utils import is_moe_param import os import psutil import gc -from math import ceil, sqrt +from math import sqrt from math import floor -from bisect import bisect_left, bisect_right +from bisect import bisect_left import torch from torch._six import inf diff --git a/deepspeed/runtime/weight_quantizer.py b/deepspeed/runtime/weight_quantizer.py index a6c241d19f58..8b7b1eb9d8ea 100644 --- a/deepspeed/runtime/weight_quantizer.py +++ b/deepspeed/runtime/weight_quantizer.py @@ -127,7 +127,7 @@ def quantize_fn(layer, policy_cls): layer_scales = [] for key in range(len(keys)): - if self.mlp_extra_grouping and is_mlp(keys[key]): + if self.mlp_extra_grouping and self.is_mlp(keys[key]): data_quantized, data_scale = self.quantize_data(keys[key], quantize_bits, groups * 2) elif policy_cls is HFBertLayerPolicy and self.is_qkv(keys[key]): data_quantized, data_scale = self.quantize_data(keys[key], quantize_bits, groups * 3) diff --git a/deepspeed/runtime/zero/offload_config.py b/deepspeed/runtime/zero/offload_config.py index c438a7638a76..9c8afda673ef 100644 --- a/deepspeed/runtime/zero/offload_config.py +++ b/deepspeed/runtime/zero/offload_config.py @@ -5,7 +5,6 @@ from deepspeed.runtime.config_utils import get_scalar_param from .offload_constants import * -from .utils import logger OFFLOAD_PARAM_KEY_DEFAULT_DICT = { OFFLOAD_PARAM_DEVICE: OFFLOAD_PARAM_DEVICE_DEFAULT, diff --git a/deepspeed/runtime/zero/partition_parameters.py b/deepspeed/runtime/zero/partition_parameters.py index de1c819fae58..c08423a64452 100755 --- a/deepspeed/runtime/zero/partition_parameters.py +++ b/deepspeed/runtime/zero/partition_parameters.py @@ -5,7 +5,6 @@ import math import os -import time import types from typing import Callable, Iterable from enum import Enum @@ -19,22 +18,19 @@ from torch.nn import Module from torch.nn import Parameter -from .linear import LinearModuleForZeroStage3, zero3_linear_wrap +from .linear import zero3_linear_wrap from .offload_constants import * import deepspeed from ..utils import get_only_unique_item, see_memory_usage from deepspeed.runtime.zero.utils import assert_ints_same_as_other_ranks -from deepspeed.utils import instrument_w_nvtx, logger +from deepspeed.utils import instrument_w_nvtx from deepspeed.comm.comm import init_distributed from deepspeed.utils.debug import (debug_param2name_id_shape, debug_param2name_id_shape_device, debug_module2name, - debug_param2name, debug_param2name_id, - debug_param2name_id_shape_status, - printflock, - log_rank_file) + debug_param2name_id_shape_status) from deepspeed.utils.logging import logger from ..swap_tensor.partitioned_param_swapper import AsyncPartitionedParameterSwapper, PartitionedParamStatus diff --git a/deepspeed/runtime/zero/partitioned_param_coordinator.py b/deepspeed/runtime/zero/partitioned_param_coordinator.py index 7baf12f9f4b7..323b0f8e2e69 100644 --- a/deepspeed/runtime/zero/partitioned_param_coordinator.py +++ b/deepspeed/runtime/zero/partitioned_param_coordinator.py @@ -4,16 +4,11 @@ """ from dataclasses import dataclass -import functools import collections -from collections import OrderedDict, UserDict -from typing import Deque, Dict, Iterable, Set, Tuple -import torch +from collections import UserDict +from typing import Deque, Set from torch.cuda import Event, Stream -from torch.nn import Module, Parameter -from deepspeed import comm as dist -from deepspeed.utils.logging import logger from deepspeed.runtime.zero.partition_parameters import * from deepspeed.runtime.zero.offload_constants import * from deepspeed.runtime.swap_tensor.partitioned_param_swapper import PartitionedParamStatus diff --git a/deepspeed/runtime/zero/stage3.py b/deepspeed/runtime/zero/stage3.py index fb284d120011..b3daefa3768d 100755 --- a/deepspeed/runtime/zero/stage3.py +++ b/deepspeed/runtime/zero/stage3.py @@ -4,35 +4,22 @@ """ import gc -from dataclasses import dataclass -import functools -import os import collections -from collections import OrderedDict, UserDict -import itertools -from typing import Deque, Dict, Iterable, Set, Tuple -import torch +from typing import Deque, Dict, Tuple from torch.cuda import Event, Stream -from torch.nn import Module, Parameter -from deepspeed import comm as dist -import math from torch._six import inf -from torch.nn import Module -from torch.nn.parameter import Parameter from deepspeed.runtime import ZeROOptimizer from deepspeed.utils import logger from deepspeed.runtime.fp16.loss_scaler import LossScaler, DynamicLossScaler from deepspeed.runtime.comm.coalesced_collectives import reduce_scatter_coalesced -from deepspeed.runtime.utils import get_global_norm, see_memory_usage, is_model_parallel_parameter +from deepspeed.runtime.utils import get_global_norm, is_model_parallel_parameter from deepspeed.runtime.zero.partition_parameters import * -from deepspeed.runtime.zero.partition_parameters import _init_external_params from deepspeed.runtime.zero.parameter_offload import DeepSpeedZeRoOffload from deepspeed.runtime.zero.constants import ZERO_OPTIMIZATION_WEIGHTS from deepspeed.ops.adam import DeepSpeedCPUAdam from deepspeed.ops.op_builder import UtilsBuilder from deepspeed.runtime.zero.offload_constants import * -from deepspeed.runtime.zero.partitioned_param_coordinator import PartitionedParameterCoordinator, iter_params from deepspeed.runtime.swap_tensor.partitioned_param_swapper import PartitionedParamStatus from deepspeed.runtime.swap_tensor.partitioned_optimizer_swapper import PartitionedOptimizerSwapper from deepspeed.runtime.swap_tensor.pipelined_optimizer_swapper import PipelinedOptimizerSwapper @@ -42,8 +29,6 @@ # with gradient partitioning and without pg_correctness_test = False -from deepspeed.utils.debug import debug_module2name_id, debug_param2name_id, debug_param2name_id_numel, debug_param2name_id_shape_device, debug_module2name_class, printflock, log_rank_file - def print_rank_0(message, debug=False, force=False): rank = dist.get_rank() @@ -2480,9 +2465,6 @@ def model_to_params(model): return total_params, largest_layer_params -import math - - def estimate_zero3_model_states_mem_needs_all_live(model, num_gpus_per_node=1, num_nodes=1, diff --git a/deepspeed/utils/comms_logging.py b/deepspeed/utils/comms_logging.py index 1c17868e09b1..34f8e2e2e1dc 100644 --- a/deepspeed/utils/comms_logging.py +++ b/deepspeed/utils/comms_logging.py @@ -1,12 +1,9 @@ -import logging -import sys -import os import math from deepspeed.utils import log_dist def get_caller_func(frame=3): - import inspect, sys + import sys return sys._getframe(frame).f_code.co_name @@ -43,7 +40,7 @@ def calc_bw_log(comm_op, size, duration): tput = (size / duration) busbw = tput else: - print_rank_0("wrong comm_op specified") + print_rank_0("wrong comm_op specified") # noqa: F821 exit(0) # convert to Gbps @@ -78,18 +75,18 @@ def configure(self, comms_config): # - Global profiling (profile all comms) # - Op-type profiling (e.g. profile all all_reduce comms) # - Op profiling (e.g. profile a specific all_reduce op) - def start_profiling_comms(): + def start_profiling_comms(self): self.prof_all = True - def stop_profiling_comms(): + def stop_profiling_comms(self): self.prof_all = True # E.g. start_profiling_op('all_reduce') - def start_profiling_op(op_name_list): - self.prof_ops = list(set(comms_logger.prof_ops) | set(op_name_list)) + def start_profiling_op(self, op_name_list): + self.prof_ops = list(set(self.prof_ops) | set(op_name_list)) - def stop_profiling_op(op_name_list): - self.prof_ops = [op for op in comms_logger.prof_ops if op not in op_name_list] + def stop_profiling_op(self, op_name_list): + self.prof_ops = [op for op in self.prof_ops if op not in op_name_list] # Add log entry def append(self, raw_name, record_name, latency, msg_size): diff --git a/deepspeed/utils/groups.py b/deepspeed/utils/groups.py index f45b6681c77b..c4d4d4d7cfe5 100644 --- a/deepspeed/utils/groups.py +++ b/deepspeed/utils/groups.py @@ -24,10 +24,9 @@ For inference and other new scenarios, the code will be either reused or added to this file. """ -import torch from deepspeed import comm as dist -from deepspeed.utils import logger, log_dist +from deepspeed.utils import log_dist from deepspeed.utils.exceptions import DeprecatedException # Expert parallel group that the current rank belongs to. diff --git a/deepspeed/utils/timer.py b/deepspeed/utils/timer.py index 8685f144bbb2..9ba150dd0d80 100755 --- a/deepspeed/utils/timer.py +++ b/deepspeed/utils/timer.py @@ -2,8 +2,6 @@ Copyright 2019 The Microsoft DeepSpeed Team """ -from numpy.core.numeric import count_nonzero -from deepspeed.elasticity.elasticity import compute_elastic_config import time import torch from numpy import mean diff --git a/deepspeed/utils/zero_to_fp32.py b/deepspeed/utils/zero_to_fp32.py index 8e8b0fd17dd1..e5249853c891 100755 --- a/deepspeed/utils/zero_to_fp32.py +++ b/deepspeed/utils/zero_to_fp32.py @@ -17,11 +17,9 @@ # while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with # DeepSpeed data structures it has to be available in the current python environment. -import deepspeed from deepspeed.utils import logger from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, - PARAM_SHAPES, SINGLE_PARTITION_OF_FP32_GROUPS, FP32_FLAT_GROUPS, ZERO_STAGE, diff --git a/docs/code-docs/source/conf.py b/docs/code-docs/source/conf.py index fab292764f9f..c43bd0dc554c 100644 --- a/docs/code-docs/source/conf.py +++ b/docs/code-docs/source/conf.py @@ -70,9 +70,6 @@ "conf_py_path": "/docs/code-docs/source/", } -# Mock imports so we don't have to install torch to build the docs. -from unittest.mock import MagicMock - sys.path.insert(0, os.path.abspath('../../../')) # Prepend module names to class descriptions? diff --git a/op_builder/builder.py b/op_builder/builder.py index 5850f4dd8425..384129a9a941 100644 --- a/op_builder/builder.py +++ b/op_builder/builder.py @@ -4,7 +4,6 @@ import os import sys import time -import json import importlib from pathlib import Path import subprocess @@ -193,11 +192,11 @@ def installed_rocm_version(): if rocm_ver_file.is_file(): with open(rocm_ver_file, 'r') as file: ROCM_VERSION_DEV_RAW = file.read() - elif "rocm" in roch.__version__: + elif "rocm" in torch.__version__: ROCM_VERSION_DEV_RAW = torch.__version__.split("rocm")[1] else: assert False, "Could not detect ROCm version" - assert ROCM_VERSION_DEV_RAW is not "", "Could not detect ROCm version" + assert ROCM_VERSION_DEV_RAW != "", "Could not detect ROCm version" ROCM_MAJOR = ROCM_VERSION_DEV_RAW.split('.')[0] ROCM_MINOR = ROCM_VERSION_DEV_RAW.split('.')[1] OpBuilder._rocm_version = (int(ROCM_MAJOR), int(ROCM_MINOR)) @@ -475,7 +474,7 @@ def jit_load(self, verbose=True): f"Unable to JIT load the {self.name} op due to it not being compatible due to hardware/software issue." ) try: - import ninja + import ninja # noqa: F401 except ImportError: raise RuntimeError( f"Unable to JIT load the {self.name} op due to ninja not being installed." diff --git a/op_builder/sparse_attn.py b/op_builder/sparse_attn.py index 00fc890eb983..414bc212e8bc 100644 --- a/op_builder/sparse_attn.py +++ b/op_builder/sparse_attn.py @@ -1,7 +1,6 @@ """ Copyright 2020 The Microsoft DeepSpeed Team """ -import warnings from .builder import OpBuilder try: diff --git a/scripts/check-torchdist.py b/scripts/check-torchdist.py index 695bee58fd48..dbbdc2cfa8ad 100755 --- a/scripts/check-torchdist.py +++ b/scripts/check-torchdist.py @@ -5,7 +5,6 @@ """ from __future__ import annotations -import os import subprocess import sys @@ -25,7 +24,7 @@ def err(s: str) -> None: "grep", "-Hn", "--no-index", - "torch\.distributed", + r"torch\.distributed", *sys.argv[1:]], capture_output=True, ) diff --git a/setup.py b/setup.py index 2e92a847a75d..24e5ec62b7dd 100755 --- a/setup.py +++ b/setup.py @@ -16,9 +16,7 @@ import os import sys -import shutil import subprocess -import warnings from setuptools import setup, find_packages from setuptools.command import egg_info import time diff --git a/tests/benchmarks/flatten_bench.py b/tests/benchmarks/flatten_bench.py index a337a1b525f3..d87971dc1a78 100755 --- a/tests/benchmarks/flatten_bench.py +++ b/tests/benchmarks/flatten_bench.py @@ -11,7 +11,7 @@ import gc import torch -from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors +from torch._utils import _flatten_dense_tensors from deepspeed.ops.op_builder import UtilsBuilder from apex_C import flatten as flatten_apex @@ -107,15 +107,15 @@ def timeme(): def line_profileme(): print("--------------- line_profiler -----------------") print("py") - profile(py)() + profile(py)() # noqa: F821 gc.collect() torch.cuda.empty_cache() print("cpp") - profile(cpp)() + profile(cpp)() # noqa: F821 gc.collect() torch.cuda.empty_cache() print("apex") - profile(apex)() + profile(apex)() # noqa: F821 gc.collect() torch.cuda.empty_cache() diff --git a/tests/benchmarks/unflatten_bench.py b/tests/benchmarks/unflatten_bench.py index 85baf751ad9c..23fb3f87566d 100755 --- a/tests/benchmarks/unflatten_bench.py +++ b/tests/benchmarks/unflatten_bench.py @@ -116,15 +116,15 @@ def timeme(): def line_profileme(): print("--------------- line_profier -----------------") print("py") - profile(py)() + profile(py)() # noqa: F821 gc.collect() torch.cuda.empty_cache() print("cpp") - profile(cpp)() + profile(cpp)() # noqa: F821 gc.collect() torch.cuda.empty_cache() print("apex") - profile(apex)() + profile(apex)() # noqa: F821 gc.collect() torch.cuda.empty_cache() diff --git a/tests/model/BingBertSquad/BingBertSquad_run_func_test.py b/tests/model/BingBertSquad/BingBertSquad_run_func_test.py index 90e6858e8bcb..828771cd324b 100755 --- a/tests/model/BingBertSquad/BingBertSquad_run_func_test.py +++ b/tests/model/BingBertSquad/BingBertSquad_run_func_test.py @@ -3,9 +3,7 @@ # Note: please copy webtext data to "Megatron-LM" folder, before running this script. import unittest -import subprocess import os -import time import re from .BingBertSquad_test_common import BaseTestCase @@ -16,7 +14,7 @@ def grep_loss_from_file(file_name): with open(file_name, 'r') as f: lines = f.readlines() line_filter = "bert_squad_progress: step=" - match_number = re.compile('loss=([-+]?[0-9]+\.?[0-9]*(?:[Ee][-+]?[0-9]+)?)') + match_number = re.compile(r'loss=([-+]?[0-9]+\.?[0-9]*(?:[Ee][-+]?[0-9]+)?)') for line in lines: if line_filter in line: diff --git a/tests/model/BingBertSquad/BingBertSquad_test_common.py b/tests/model/BingBertSquad/BingBertSquad_test_common.py index a9678bb6923f..b6069d76e69a 100755 --- a/tests/model/BingBertSquad/BingBertSquad_test_common.py +++ b/tests/model/BingBertSquad/BingBertSquad_test_common.py @@ -5,7 +5,6 @@ import subprocess import os import time -import re class BaseTestCase(unittest.TestCase): diff --git a/tests/model/BingBertSquad/test_e2e_squad.py b/tests/model/BingBertSquad/test_e2e_squad.py index 0140ebd87770..0854a8339e1b 100644 --- a/tests/model/BingBertSquad/test_e2e_squad.py +++ b/tests/model/BingBertSquad/test_e2e_squad.py @@ -1,11 +1,9 @@ import subprocess as sp -import datetime import os from math import isclose import sys import pytest import json -import argparse sys.path.append("../../../DeepSpeedExamples/BingBertSquad") import evaluate as eval diff --git a/tests/model/Megatron_GPT2/run_checkpoint_test.py b/tests/model/Megatron_GPT2/run_checkpoint_test.py index fe564d4fdb8a..628547ef2f14 100755 --- a/tests/model/Megatron_GPT2/run_checkpoint_test.py +++ b/tests/model/Megatron_GPT2/run_checkpoint_test.py @@ -5,7 +5,6 @@ import unittest import subprocess import os -import time import re from .test_common import BaseTestCase @@ -26,7 +25,7 @@ def grep_loss_from_file(file_name): with open(file_name, 'r') as f: lines = f.readlines() line_filter = "validation loss at the end of training for test data | LM loss:" - match_number = re.compile('LM loss: ([-+]?[0-9]+\.?[0-9]*(?:[Ee][-+]?[0-9]+)?)') + match_number = re.compile(r'LM loss: ([-+]?[0-9]+\.?[0-9]*(?:[Ee][-+]?[0-9]+)?)') for line in lines: if line_filter in line: diff --git a/tests/model/Megatron_GPT2/run_func_test.py b/tests/model/Megatron_GPT2/run_func_test.py index 463aa1f94f15..78a685e0f0e2 100755 --- a/tests/model/Megatron_GPT2/run_func_test.py +++ b/tests/model/Megatron_GPT2/run_func_test.py @@ -3,9 +3,7 @@ # Note: please copy webtext data to "Megatron-LM" folder, before running this script. import unittest -import subprocess import os -import time import re from .test_common import BaseTestCase @@ -22,7 +20,7 @@ def grep_loss_from_file(file_name): with open(file_name, 'r') as f: lines = f.readlines() line_filter = "validation loss at the end of training for test data | LM loss:" - match_number = re.compile('LM loss: ([-+]?[0-9]+\.?[0-9]*(?:[Ee][-+]?[0-9]+)?)') + match_number = re.compile(r'LM loss: ([-+]?[0-9]+\.?[0-9]*(?:[Ee][-+]?[0-9]+)?)') for line in lines: if line_filter in line: diff --git a/tests/model/Megatron_GPT2/run_perf_baseline.py b/tests/model/Megatron_GPT2/run_perf_baseline.py index f30e9cfe9bc1..0c7233d5dc8f 100755 --- a/tests/model/Megatron_GPT2/run_perf_baseline.py +++ b/tests/model/Megatron_GPT2/run_perf_baseline.py @@ -3,9 +3,6 @@ # Note: please copy webtext data to "Megatron-LM" folder, before running this script. import unittest -import subprocess -import os -import time import re from test_common import BaseTestCase @@ -103,7 +100,7 @@ def grep_latency_from_file(self, file_name): lines = f.readlines() line_filter = "elapsed time per iteration" match_number = re.compile( - 'elapsed time per iteration \(ms\): ([-+]?[0-9]+\.?[0-9]*(?:[Ee][-+]?[0-9]+)?)' + r'elapsed time per iteration \(ms\): ([-+]?[0-9]+\.?[0-9]*(?:[Ee][-+]?[0-9]+)?)' ) for line in lines: diff --git a/tests/model/Megatron_GPT2/run_perf_test.py b/tests/model/Megatron_GPT2/run_perf_test.py index 64b20f4866a4..623f945a4425 100755 --- a/tests/model/Megatron_GPT2/run_perf_test.py +++ b/tests/model/Megatron_GPT2/run_perf_test.py @@ -3,9 +3,6 @@ # Note: please copy webtext data to "Megatron-LM" folder, before running this script. import unittest -import subprocess -import os -import time import re from test_common import BaseTestCase @@ -107,7 +104,7 @@ def grep_latency_from_file(self, file_name): lines = f.readlines() line_filter = "elapsed time per iteration" match_number = re.compile( - 'elapsed time per iteration \(ms\): ([-+]?[0-9]+\.?[0-9]*(?:[Ee][-+]?[0-9]+)?)' + r'elapsed time per iteration \(ms\): ([-+]?[0-9]+\.?[0-9]*(?:[Ee][-+]?[0-9]+)?)' ) for line in lines: diff --git a/tests/model/Megatron_GPT2/test_common.py b/tests/model/Megatron_GPT2/test_common.py index 04b3e4a23a6c..6f9bec89eeb5 100755 --- a/tests/model/Megatron_GPT2/test_common.py +++ b/tests/model/Megatron_GPT2/test_common.py @@ -5,7 +5,6 @@ import subprocess import os import time -import re class BaseTestCase(unittest.TestCase): diff --git a/tests/model/run_sanity_check.py b/tests/model/run_sanity_check.py index 2f020b52db16..a226ccb8ca06 100755 --- a/tests/model/run_sanity_check.py +++ b/tests/model/run_sanity_check.py @@ -8,8 +8,6 @@ sys.path.append('../DeepSpeedExamples/Megatron_GPT2') sys.path.append('../DeepSpeedExamples/BingBertSquad') -import os - # Import the test cases here. import Megatron_GPT2 import BingBertSquad diff --git a/tests/onebit/test_mpi_backend.py b/tests/onebit/test_mpi_backend.py index 57dc7371c4f9..65cfb3ed96c5 100644 --- a/tests/onebit/test_mpi_backend.py +++ b/tests/onebit/test_mpi_backend.py @@ -1,5 +1,4 @@ from mpi4py import MPI -import time import torch import deepspeed.comm as dist import numpy as np diff --git a/tests/onebit/test_mpi_perf.py b/tests/onebit/test_mpi_perf.py index b782cbc5dc3e..1652e946985c 100644 --- a/tests/onebit/test_mpi_perf.py +++ b/tests/onebit/test_mpi_perf.py @@ -1,8 +1,5 @@ from mpi4py import MPI -import time import torch -import deepspeed.comm as dist -import numpy as np import deepspeed from deepspeed.runtime.comm.mpi import MpiBackend diff --git a/tests/onebit/test_nccl_backend.py b/tests/onebit/test_nccl_backend.py index 6a99c9fe2a9c..395b1053f917 100644 --- a/tests/onebit/test_nccl_backend.py +++ b/tests/onebit/test_nccl_backend.py @@ -1,4 +1,3 @@ -import time import torch import deepspeed.comm as dist import numpy as np diff --git a/tests/onebit/test_nccl_perf.py b/tests/onebit/test_nccl_perf.py index d4cfbccfd7da..86923ef90b82 100644 --- a/tests/onebit/test_nccl_perf.py +++ b/tests/onebit/test_nccl_perf.py @@ -1,4 +1,3 @@ -import time import torch import deepspeed.comm as dist import numpy as np diff --git a/tests/small_model_debugging/test.py b/tests/small_model_debugging/test.py index 25418f3c0f93..331a8ef35ca6 100644 --- a/tests/small_model_debugging/test.py +++ b/tests/small_model_debugging/test.py @@ -1,8 +1,6 @@ import torch from deepspeed.pt.deepspeed_linear import LinearModuleForZeroStage3 -from deepspeed.pt.deepspeed_utils import see_memory_usage from deepspeed.pt.log_utils import logger -import deepspeed def see_memory_usage(message): diff --git a/tests/unit/common.py b/tests/unit/common.py index 10037008aa90..7a60cbe8c81a 100644 --- a/tests/unit/common.py +++ b/tests/unit/common.py @@ -8,9 +8,6 @@ import deepspeed import pytest -from functools import wraps -import unittest -from pathlib import Path from pathlib import Path diff --git a/tests/unit/megatron_model.py b/tests/unit/megatron_model.py index fd2ef69b7259..6fc55393295f 100644 --- a/tests/unit/megatron_model.py +++ b/tests/unit/megatron_model.py @@ -1,4 +1,3 @@ -from pathlib import Path import torch import os import sys diff --git a/tests/unit/modeling.py b/tests/unit/modeling.py index e3b6b4d836f0..e8a38afc9538 100755 --- a/tests/unit/modeling.py +++ b/tests/unit/modeling.py @@ -28,7 +28,6 @@ import shutil import tarfile import tempfile -import sys from io import open import torch @@ -38,10 +37,8 @@ import deepspeed.comm as dist from torch.nn import Module -from torch.nn.parameter import Parameter import torch.nn.functional as F import torch.nn.init as init -import time #from numba import cuda @@ -187,8 +184,8 @@ def swish(x): class GPUTimer: def __init__(self): super().__init__() - self.start = cuda.event() - self.stop = cuda.event() + self.start = cuda.event() # noqa: F821 + self.stop = cuda.event() # noqa: F821 def record(self): self.start.record() @@ -216,9 +213,7 @@ def __init__(self, self.out_features = out_features self.fused_gelu = False self.fused_tanh = False - if isinstance(act, - str) or (sys.version_info[0] == 2 and isinstance(act, - unicode)): + if isinstance(act, str): if bias and act == 'gelu': self.fused_gelu = True elif bias and act == 'tanh': @@ -307,10 +302,7 @@ def __init__(self, initializer_range: The sttdev of the truncated_normal_initializer for initializing all weight matrices. """ - if isinstance(vocab_size_or_config_json_file, - str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, - unicode)): + if isinstance(vocab_size_or_config_json_file, str): with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: json_config = json.loads(reader.read()) for key, value in json_config.items(): @@ -644,8 +636,8 @@ def get_grads(self): def get_modules(self, big_node, input): for mdl in big_node.named_children(): - graph.append(mdl) - get_modules(self, mdl, input) + self.graph.append(mdl) + self.get_modules(self, mdl, input) def forward(self, hidden_states, @@ -864,22 +856,22 @@ def from_pretrained(cls, archive_file = PRETRAINED_MODEL_ARCHIVE_MAP[pretrained_model_name_or_path] else: archive_file = pretrained_model_name_or_path - if resolved_archive_file == archive_file: + if resolved_archive_file == archive_file: # noqa: F821 logger.info("loading archive file {}".format(archive_file)) else: logger.info("loading archive file {} from cache at {}".format( archive_file, - resolved_archive_file)) + resolved_archive_file)) # noqa: F821 tempdir = None - if os.path.isdir(resolved_archive_file) or from_tf: - serialization_dir = resolved_archive_file + if os.path.isdir(resolved_archive_file) or from_tf: # noqa: F821 + serialization_dir = resolved_archive_file # noqa: F821 else: # Extract archive to temp dir tempdir = tempfile.mkdtemp() logger.info("extracting archive file {} to temp dir {}".format( - resolved_archive_file, + resolved_archive_file, # noqa: F821 tempdir)) - with tarfile.open(resolved_archive_file, 'r:gz') as archive: + with tarfile.open(resolved_archive_file, 'r:gz') as archive: # noqa: F821 archive.extractall(tempdir) serialization_dir = tempdir # Load config diff --git a/tests/unit/modelingpreln.py b/tests/unit/modelingpreln.py index 34a933bc6b29..673a73ac91f4 100755 --- a/tests/unit/modelingpreln.py +++ b/tests/unit/modelingpreln.py @@ -28,7 +28,6 @@ import shutil import tarfile import tempfile -import sys from io import open import torch @@ -38,10 +37,8 @@ import deepspeed.comm as dist from torch.nn import Module -from torch.nn.parameter import Parameter import torch.nn.functional as F import torch.nn.init as init -import time #from numba import cuda @@ -187,8 +184,8 @@ def swish(x): class GPUTimer: def __init__(self): super().__init__() - self.start = cuda.event() - self.stop = cuda.event() + self.start = cuda.event() # noqa: F821 + self.stop = cuda.event() # noqa: F821 def record(self): self.start.record() @@ -216,9 +213,7 @@ def __init__(self, self.out_features = out_features self.fused_gelu = False self.fused_tanh = False - if isinstance(act, - str) or (sys.version_info[0] == 2 and isinstance(act, - unicode)): + if isinstance(act, str): if bias and act == 'gelu': self.fused_gelu = True elif bias and act == 'tanh': @@ -307,10 +302,7 @@ def __init__(self, initializer_range: The sttdev of the truncated_normal_initializer for initializing all weight matrices. """ - if isinstance(vocab_size_or_config_json_file, - str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, - unicode)): + if isinstance(vocab_size_or_config_json_file, str): with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: json_config = json.loads(reader.read()) for key, value in json_config.items(): @@ -738,8 +730,8 @@ def get_grads(self): def get_modules(self, big_node, input): for mdl in big_node.named_children(): - graph.append(mdl) - get_modules(self, mdl, input) + self.graph.append(mdl) + self.get_modules(self, mdl, input) def forward(self, hidden_states, @@ -959,22 +951,22 @@ def from_pretrained(cls, archive_file = PRETRAINED_MODEL_ARCHIVE_MAP[pretrained_model_name_or_path] else: archive_file = pretrained_model_name_or_path - if resolved_archive_file == archive_file: + if resolved_archive_file == archive_file: # noqa: F821 logger.info("loading archive file {}".format(archive_file)) else: logger.info("loading archive file {} from cache at {}".format( archive_file, - resolved_archive_file)) + resolved_archive_file)) # noqa: F821 tempdir = None - if os.path.isdir(resolved_archive_file) or from_tf: - serialization_dir = resolved_archive_file + if os.path.isdir(resolved_archive_file) or from_tf: # noqa: F821 + serialization_dir = resolved_archive_file # noqa: F821 else: # Extract archive to temp dir tempdir = tempfile.mkdtemp() logger.info("extracting archive file {} to temp dir {}".format( - resolved_archive_file, + resolved_archive_file, # noqa: F821 tempdir)) - with tarfile.open(resolved_archive_file, 'r:gz') as archive: + with tarfile.open(resolved_archive_file, 'r:gz') as archive: # noqa: F821 archive.extractall(tempdir) serialization_dir = tempdir # Load config diff --git a/tests/unit/multi_output_model.py b/tests/unit/multi_output_model.py index 7caf6f7de75f..240c1a4b7aa6 100755 --- a/tests/unit/multi_output_model.py +++ b/tests/unit/multi_output_model.py @@ -1,6 +1,3 @@ -import os -import json -import argparse import torch diff --git a/tests/unit/test_autocast.py b/tests/unit/test_autocast.py index 004cd8533869..7bffad14530d 100644 --- a/tests/unit/test_autocast.py +++ b/tests/unit/test_autocast.py @@ -1,12 +1,11 @@ import pytest import torch -import deepspeed from deepspeed.runtime.zero.linear import LinearModuleForZeroStage3 def _skip_autocast_test(): try: - from torch.cuda.amp import custom_fwd, custom_bwd + from torch.cuda.amp import custom_fwd, custom_bwd # noqa: F401 except (ImportError, AttributeError) as exp: return True diff --git a/tests/unit/test_autotuning.py b/tests/unit/test_autotuning.py index 2a7898b8af0a..681c3108b15b 100644 --- a/tests/unit/test_autotuning.py +++ b/tests/unit/test_autotuning.py @@ -1,6 +1,5 @@ import os import pytest -import torch from .simple_model import create_config_from_dict from deepspeed.launcher import runner as dsrun from deepspeed.autotuning.autotuner import Autotuner diff --git a/tests/unit/test_bf16.py b/tests/unit/test_bf16.py index aa2ab132394c..4930a74640de 100644 --- a/tests/unit/test_bf16.py +++ b/tests/unit/test_bf16.py @@ -1,4 +1,3 @@ -import math import torch import deepspeed import pytest diff --git a/tests/unit/test_checkpointing.py b/tests/unit/test_checkpointing.py index dd93e006081f..e55fc30fa56c 100755 --- a/tests/unit/test_checkpointing.py +++ b/tests/unit/test_checkpointing.py @@ -1,10 +1,5 @@ -import torch - -import deepspeed.comm as dist - import deepspeed from deepspeed.runtime.zero.stage_1_and_2 import DeepSpeedZeroOptimizer -from deepspeed.utils import groups from deepspeed.runtime.fp16.fused_optimizer import FP16_Optimizer from deepspeed.runtime.fp16.unfused_optimizer import FP16_UnfusedOptimizer from deepspeed.moe.utils import split_params_into_different_moe_groups_for_optimizer @@ -19,10 +14,7 @@ from .util import required_minimum_torch_version, required_torch_version import itertools -import argparse import pytest -import json -import os import numbers from .common import distributed_test from .simple_model import * @@ -1383,7 +1375,6 @@ def _test_load_immediate_save(args, model, tmpdir): @pytest.mark.parametrize('zero_stage', [0, 1, 2, 3]) def test_save_before_accum_grad_is_done(tmpdir, zero_stage): config_dict = { - "train_batch_size": 4, "optimizer": { "type": 'Adam' }, diff --git a/tests/unit/test_coalesced_collectives.py b/tests/unit/test_coalesced_collectives.py index a7e0ec35751b..9597a1e8536a 100644 --- a/tests/unit/test_coalesced_collectives.py +++ b/tests/unit/test_coalesced_collectives.py @@ -1,7 +1,5 @@ """unit tests for coalesced collectives""" -import pytest - import torch import deepspeed.comm as dist from deepspeed.runtime.comm.coalesced_collectives import reduce_scatter_coalesced diff --git a/tests/unit/test_compression.py b/tests/unit/test_compression.py index f00aafaca1ba..d8d21bb630c0 100755 --- a/tests/unit/test_compression.py +++ b/tests/unit/test_compression.py @@ -1,4 +1,3 @@ -from zlib import compressobj import torch import pytest import random diff --git a/tests/unit/test_configurable_parallel.py b/tests/unit/test_configurable_parallel.py index daa2cd1791b0..f9ff67f578e0 100755 --- a/tests/unit/test_configurable_parallel.py +++ b/tests/unit/test_configurable_parallel.py @@ -1,14 +1,11 @@ import torch import deepspeed import pytest -import os -import time import random import numpy as np import torch.multiprocessing as mp import deepspeed.comm as dist from .common import distributed_test -from .simple_model import args_from_dict, create_deepspeed_args from .megatron_model import get_gpt2_model, get_megatron_version from .megatron_model import MockGPT2ModelPipe as GPT2ModelPipe from deepspeed.utils import RepeatingLoader diff --git a/tests/unit/test_cpu_adam.py b/tests/unit/test_cpu_adam.py index 74356dffd8cf..7357c086d08d 100755 --- a/tests/unit/test_cpu_adam.py +++ b/tests/unit/test_cpu_adam.py @@ -1,9 +1,6 @@ -import argparse import torch -import time import numpy as np import pytest -import copy from cpuinfo import get_cpu_info import deepspeed diff --git a/tests/unit/test_cuda_backward.py b/tests/unit/test_cuda_backward.py index d7faee7c0502..42bf50c6ad4e 100755 --- a/tests/unit/test_cuda_backward.py +++ b/tests/unit/test_cuda_backward.py @@ -1,6 +1,6 @@ +import math import numpy as np import torch -import torch.nn.functional as F import pytest import random import copy @@ -121,7 +121,7 @@ def custom_forward(*inputs): num_layers = len(self.layer) chunk_length = math.ceil(math.sqrt(num_layers)) while l < num_layers: - hidden_states = checkpoint.checkpoint(custom(l, + hidden_states = checkpoint.checkpoint(custom(l, # noqa: F821 l + chunk_length), hidden_states, attention_mask * 1) diff --git a/tests/unit/test_cuda_forward.py b/tests/unit/test_cuda_forward.py index 2a5d2d13858e..9c2b7f7afaa2 100755 --- a/tests/unit/test_cuda_forward.py +++ b/tests/unit/test_cuda_forward.py @@ -1,19 +1,13 @@ -import argparse +import math import numpy as np import torch -import torch.nn.functional as F import pytest -import json import random -import time import copy from torch import nn from .modelingpreln import BertEncoder as BertEncoderPreln from .modeling import BertLayerNorm, BertConfig, BertEncoder as BertEncoderPostln from deepspeed import DeepSpeedTransformerLayer, DeepSpeedTransformerConfig -import deepspeed - -import sys def check_equal(first, second, atol=1e-2, verbose=False): @@ -74,7 +68,7 @@ def custom_forward(*inputs): num_layers = len(self.layer) chunk_length = math.ceil(math.sqrt(num_layers)) while l < num_layers: - hidden_states = checkpoint.checkpoint(custom(l, + hidden_states = checkpoint.checkpoint(custom(l, # noqa: F821 l + chunk_length), hidden_states, attention_mask * 1) diff --git a/tests/unit/test_curriculum_learning.py b/tests/unit/test_curriculum_learning.py index 22dde25fcd35..cb7af95b1edb 100644 --- a/tests/unit/test_curriculum_learning.py +++ b/tests/unit/test_curriculum_learning.py @@ -1,12 +1,4 @@ -import torch -import deepspeed.comm as dist import deepspeed -import argparse -import pytest -import json -import os -import numpy as np -import time from .common import distributed_test from .simple_model import Curriculum_SimpleModel, random_dataloader, args_from_dict diff --git a/tests/unit/test_dynamic_loss_scale.py b/tests/unit/test_dynamic_loss_scale.py index 65a679d94de7..3d9209fcc76a 100755 --- a/tests/unit/test_dynamic_loss_scale.py +++ b/tests/unit/test_dynamic_loss_scale.py @@ -1,9 +1,5 @@ import torch import deepspeed -import argparse -import pytest -import json -import os import numpy as np from .common import distributed_test from .simple_model import SimpleModel, args_from_dict diff --git a/tests/unit/test_elastic.py b/tests/unit/test_elastic.py index 353d6def37ba..9f5d1f0d06bd 100644 --- a/tests/unit/test_elastic.py +++ b/tests/unit/test_elastic.py @@ -2,7 +2,7 @@ import deepspeed from .common import distributed_test from deepspeed.git_version_info import version as ds_version -from .simple_model import SimpleModel, SimpleOptimizer, random_dataloader, args_from_dict +from .simple_model import SimpleModel, args_from_dict base_ds_config = { "elasticity": { diff --git a/tests/unit/test_flops_profiler.py b/tests/unit/test_flops_profiler.py index 173fa7eed09c..9a01f5c6a322 100644 --- a/tests/unit/test_flops_profiler.py +++ b/tests/unit/test_flops_profiler.py @@ -1,9 +1,8 @@ import torch import pytest import deepspeed -import deepspeed.runtime.utils as ds_utils -from deepspeed.profiling.flops_profiler import FlopsProfiler, get_model_profile -from .simple_model import SimpleModel, SimpleOptimizer, random_dataloader, args_from_dict +from deepspeed.profiling.flops_profiler import get_model_profile +from .simple_model import SimpleModel, random_dataloader, args_from_dict from .common import distributed_test TORCH_MAJOR = int(torch.__version__.split('.')[0]) diff --git a/tests/unit/test_fp16.py b/tests/unit/test_fp16.py index d8826e59e8e7..43d76994b38d 100755 --- a/tests/unit/test_fp16.py +++ b/tests/unit/test_fp16.py @@ -1,12 +1,7 @@ -import math -from deepspeed.utils import groups import torch import deepspeed.comm as dist import deepspeed -import argparse import pytest -import json -import os from deepspeed.ops.adam import FusedAdam from .common import distributed_test from deepspeed.ops.op_builder import CPUAdamBuilder @@ -14,7 +9,7 @@ from .util import required_torch_version try: - from apex import amp + from apex import amp # noqa: F401 _amp_available = True except ImportError: _amp_available = False diff --git a/tests/unit/test_get_optim_files.py b/tests/unit/test_get_optim_files.py index 68d046bfe99e..b0b5b242c200 100644 --- a/tests/unit/test_get_optim_files.py +++ b/tests/unit/test_get_optim_files.py @@ -1,6 +1,5 @@ import os import pytest -import deepspeed from deepspeed.utils.zero_to_fp32 import get_optim_files diff --git a/tests/unit/test_ignore_unused_parameters.py b/tests/unit/test_ignore_unused_parameters.py index eb26f46ca209..fd1f427d1220 100644 --- a/tests/unit/test_ignore_unused_parameters.py +++ b/tests/unit/test_ignore_unused_parameters.py @@ -1,8 +1,4 @@ -import torch import pytest -import json -import argparse -import os from .common import distributed_test from .simple_model import UnusedParametersModel, random_dataloader, args_from_dict from deepspeed.ops.op_builder import CPUAdamBuilder diff --git a/tests/unit/test_inference.py b/tests/unit/test_inference.py index 006fe6cc884f..c53c817abdf5 100644 --- a/tests/unit/test_inference.py +++ b/tests/unit/test_inference.py @@ -1,12 +1,10 @@ import os -import sys import time import torch import pytest import itertools import deepspeed from deepspeed.git_version_info import torch_info -from collections import defaultdict from .common import distributed_test from packaging import version as pkg_version from deepspeed.ops.op_builder import OpBuilder diff --git a/tests/unit/test_lr_schedulers.py b/tests/unit/test_lr_schedulers.py index 47bcfb1ef329..49da0111d985 100755 --- a/tests/unit/test_lr_schedulers.py +++ b/tests/unit/test_lr_schedulers.py @@ -1,11 +1,8 @@ import torch import deepspeed -import argparse import pytest -import json -import os from .common import distributed_test -from .simple_model import SimpleModel, SimpleOptimizer, random_dataloader, args_from_dict +from .simple_model import SimpleModel, random_dataloader, args_from_dict from deepspeed.runtime.lr_schedules import LR_RANGE_TEST, LR_RANGE_TEST_MIN_LR, LR_RANGE_TEST_STEP_RATE, LR_RANGE_TEST_STEP_SIZE, LR_RANGE_TEST_STAIRCASE from deepspeed.runtime.lr_schedules import WARMUP_LR, WARMUP_MIN_LR, WARMUP_MAX_LR, WARMUP_NUM_STEPS, WARMUP_TYPE, WARMUP_LOG_RATE, WARMUP_LINEAR_RATE from deepspeed.runtime.lr_schedules import ONE_CYCLE, CYCLE_MIN_LR, CYCLE_MAX_LR, CYCLE_FIRST_STEP_SIZE, DECAY_LR_RATE, DECAY_STEP_SIZE diff --git a/tests/unit/test_moe.py b/tests/unit/test_moe.py index 779bafbb758f..cb1a89b9a1eb 100644 --- a/tests/unit/test_moe.py +++ b/tests/unit/test_moe.py @@ -1,20 +1,12 @@ -import math -from deepspeed.utils import groups import torch -import deepspeed.comm as dist import deepspeed -import argparse import pytest -import json -import os -from deepspeed.ops.adam import FusedAdam from .common import distributed_test -from deepspeed.ops.op_builder import CPUAdamBuilder -from .simple_model import SimpleModel, SimplePRMoEModel, SimpleOptimizer, random_dataloader, args_from_dict, create_deepspeed_args, SimpleMoEModel, sequence_dataloader +from .simple_model import SimplePRMoEModel, args_from_dict, SimpleMoEModel, sequence_dataloader from .util import required_torch_version try: - from apex import amp + from apex import amp # noqa: F401 _amp_available = True except ImportError: _amp_available = False diff --git a/tests/unit/test_monitor.py b/tests/unit/test_monitor.py index 95f045d54dea..a417fb9f7d8c 100644 --- a/tests/unit/test_monitor.py +++ b/tests/unit/test_monitor.py @@ -2,7 +2,6 @@ from deepspeed.monitor.constants import * -from deepspeed.monitor.monitor import MonitorMaster from deepspeed.monitor.tensorboard import TensorBoardMonitor from deepspeed.monitor.wandb import WandbMonitor from deepspeed.monitor.csv_monitor import csvMonitor @@ -10,10 +9,9 @@ from .simple_model import * from .common import distributed_test from deepspeed.runtime.config import DeepSpeedConfig -from deepspeed.monitor.config import DeepSpeedMonitorConfig try: - import tensorboard + import tensorboard # noqa: F401 _tb_available = True except ImportError: _tb_available = False @@ -21,7 +19,7 @@ reason="tensorboard is not installed") try: - import wandb + import wandb # noqa: F401 _wandb_available = True except ImportError: _wandb_available = False diff --git a/tests/unit/test_multi_output_model.py b/tests/unit/test_multi_output_model.py index 478bdc8d383d..deef776c0815 100755 --- a/tests/unit/test_multi_output_model.py +++ b/tests/unit/test_multi_output_model.py @@ -1,10 +1,6 @@ import torch import deepspeed -import argparse -import pytest from pytest import approx -import json -import os from .common import distributed_test from .simple_model import args_from_dict from .multi_output_model import MultiOutputModel, multi_output_dataloader diff --git a/tests/unit/test_onebit.py b/tests/unit/test_onebit.py index b7806b0831c7..b6f1f8bd4e15 100644 --- a/tests/unit/test_onebit.py +++ b/tests/unit/test_onebit.py @@ -1,23 +1,19 @@ import torch import torch.nn as nn -import torch.nn.functional as F import deepspeed.comm as dist import deepspeed -import argparse import pytest import copy -import json import os import numpy as np -import time -from deepspeed.runtime.pipe.topology import PipeDataParallelTopology, PipeModelDataParallelTopology +from deepspeed.runtime.pipe.topology import PipeDataParallelTopology from deepspeed.ops.op_builder import OpBuilder PipeTopo = PipeDataParallelTopology -from deepspeed.runtime.pipe.module import PipelineModule, LayerSpec +from deepspeed.runtime.pipe.module import PipelineModule from .common import distributed_test -from .simple_model import SimpleModel, SimpleOptimizer, random_dataloader, args_from_dict, create_deepspeed_args +from .simple_model import SimpleModel, random_dataloader, args_from_dict from .test_pipe import AlexNetPipe, train_cifar TORCH_MAJOR = int(torch.__version__.split('.')[0]) diff --git a/tests/unit/test_pipe.py b/tests/unit/test_pipe.py index 832d06f9d3ce..de1bd3ff279c 100755 --- a/tests/unit/test_pipe.py +++ b/tests/unit/test_pipe.py @@ -1,4 +1,3 @@ -import os import copy import torch @@ -11,8 +10,7 @@ import deepspeed import deepspeed.runtime.utils as ds_utils - -from deepspeed.runtime.pipe.topology import PipeDataParallelTopology, PipeModelDataParallelTopology +from deepspeed.runtime.pipe.topology import PipeDataParallelTopology PipeTopo = PipeDataParallelTopology from deepspeed.runtime.pipe.module import PipelineModule, LayerSpec diff --git a/tests/unit/test_pipe_module.py b/tests/unit/test_pipe_module.py index e50c7d6231a5..1cba989b54e8 100644 --- a/tests/unit/test_pipe_module.py +++ b/tests/unit/test_pipe_module.py @@ -8,11 +8,11 @@ import deepspeed -from deepspeed.runtime.pipe.topology import PipeDataParallelTopology, PipeModelDataParallelTopology +from deepspeed.runtime.pipe.topology import PipeDataParallelTopology PipeTopo = PipeDataParallelTopology -from deepspeed.pipe import PipelineModule, LayerSpec +from deepspeed.pipe import PipelineModule from deepspeed.utils import RepeatingLoader from .common import distributed_test diff --git a/tests/unit/test_pld.py b/tests/unit/test_pld.py index 5d275d16379c..0953b648dce4 100755 --- a/tests/unit/test_pld.py +++ b/tests/unit/test_pld.py @@ -4,7 +4,7 @@ from deepspeed.runtime.progressive_layer_drop import ProgressiveLayerDrop from .common import distributed_test -from .simple_model import SimpleModel, PLD_SimpleModel, SimpleOptimizer, random_dataloader, args_from_dict +from .simple_model import SimpleModel, PLD_SimpleModel, random_dataloader, args_from_dict @pytest.mark.parametrize('theta', [0, 0.1, 0.9, 1.0]) diff --git a/tests/unit/test_reshape_checkpoint.py b/tests/unit/test_reshape_checkpoint.py index 317f3bb1661f..ed83baf43178 100644 --- a/tests/unit/test_reshape_checkpoint.py +++ b/tests/unit/test_reshape_checkpoint.py @@ -1,6 +1,3 @@ -import pytest -import deepspeed - from deepspeed.checkpoint import model_3d_desc diff --git a/tests/unit/test_runtime_utils.py b/tests/unit/test_runtime_utils.py index 2012233cdf63..8e1697105000 100644 --- a/tests/unit/test_runtime_utils.py +++ b/tests/unit/test_runtime_utils.py @@ -1,11 +1,9 @@ -from deepspeed.moe.utils import is_moe_param, split_params_grads_into_shared_and_expert_params, split_params_into_shared_and_expert_params import torch from torch._utils import _flatten_dense_tensors import deepspeed.comm as dist import pytest import deepspeed.runtime.utils as ds_utils -from deepspeed.utils.logging import log_dist import deepspeed.utils.groups as groups from .common import distributed_test diff --git a/tests/unit/test_sparse_attention.py b/tests/unit/test_sparse_attention.py index 531524e45421..f3079982c964 100755 --- a/tests/unit/test_sparse_attention.py +++ b/tests/unit/test_sparse_attention.py @@ -16,7 +16,7 @@ def test_sparse_attention_module_availability(): return True try: - from deepspeed.ops import sparse_attention + from deepspeed.ops import sparse_attention # noqa: F401 except ImportError: print("Sparse Attention Module is not installed!") return False @@ -26,7 +26,7 @@ def test_sparse_attention_module_availability(): def test_matmul_module_availability(): return True try: - from deepspeed.ops.sparse_attention.matmul import MatMul + from deepspeed.ops.sparse_attention.matmul import MatMul # noqa: F401 except ImportError: print("Sparse MatMul Module is not installed!") return False @@ -36,7 +36,7 @@ def test_matmul_module_availability(): def test_softmax_module_availability(): return True try: - from deepspeed.ops.sparse_attention.softmax import Softmax + from deepspeed.ops.sparse_attention.softmax import Softmax # noqa: F401 except ImportError: print("Sparse Softmax Module is not installed!") return False @@ -46,7 +46,7 @@ def test_softmax_module_availability(): def test_sparsityconfig_module_availability(): return True try: - from deepspeed.ops.sparse_attention import SparsityConfig + from deepspeed.ops.sparse_attention import SparsityConfig # noqa: F401 except ImportError: print("SparsityConfig Module is not installed!") return False @@ -56,7 +56,7 @@ def test_sparsityconfig_module_availability(): def test_densesparsityconfig_module_availability(): return True try: - from deepspeed.ops.sparse_attention import DenseSparsityConfig + from deepspeed.ops.sparse_attention import DenseSparsityConfig # noqa: F401 except ImportError: print("DenseSparsityConfig Module is not installed!") return False @@ -66,7 +66,7 @@ def test_densesparsityconfig_module_availability(): def test_fixedsparsityconfig_module_availability(): return True try: - from deepspeed.ops.sparse_attention import FixedSparsityConfig + from deepspeed.ops.sparse_attention import FixedSparsityConfig # noqa: F401 except ImportError: print("FixedSparsityConfig Module is not installed!") return False @@ -76,7 +76,7 @@ def test_fixedsparsityconfig_module_availability(): def test_variablesparsityconfig_module_availability(): return True try: - from deepspeed.ops.sparse_attention import VariableSparsityConfig + from deepspeed.ops.sparse_attention import VariableSparsityConfig # noqa: F401 except ImportError: print("VariableSparsityConfig Module is not installed!") return False @@ -86,7 +86,7 @@ def test_variablesparsityconfig_module_availability(): def test_bigbirdsparsityconfig_module_availability(): return True try: - from deepspeed.ops.sparse_attention import BigBirdSparsityConfig + from deepspeed.ops.sparse_attention import BigBirdSparsityConfig # noqa: F401 except ImportError: print("BigBirdSparsityConfig Module is not installed!") return False @@ -96,7 +96,7 @@ def test_bigbirdsparsityconfig_module_availability(): def test_bslongformersparsityconfig_module_availability(): return True try: - from deepspeed.ops.sparse_attention import BSLongformerSparsityConfig + from deepspeed.ops.sparse_attention import BSLongformerSparsityConfig # noqa: F401 except ImportError: print("BSLongformerSparsityConfig Module is not installed!") return False @@ -106,7 +106,7 @@ def test_bslongformersparsityconfig_module_availability(): def test_sparseselfattention_module_availability(): return True try: - from deepspeed.ops.sparse_attention import SparseSelfAttention + from deepspeed.ops.sparse_attention import SparseSelfAttention # noqa: F401 except ImportError: print("SparseSelfAttention Module is not installed!") return False @@ -116,7 +116,7 @@ def test_sparseselfattention_module_availability(): def test_bertsparseselfattention_module_availability(): return True try: - from deepspeed.ops.sparse_attention import BertSparseSelfAttention + from deepspeed.ops.sparse_attention import BertSparseSelfAttention # noqa: F401 except ImportError: print("BertSparseSelfAttention Module is not installed!") return False @@ -126,7 +126,7 @@ def test_bertsparseselfattention_module_availability(): def test_sparseattentionutils_availability(): return True try: - from deepspeed.ops.sparse_attention import SparseAttentionUtils + from deepspeed.ops.sparse_attention import SparseAttentionUtils # noqa: F401 except ImportError: print("SparseAttentionUtils Module is not installed!") return False @@ -136,7 +136,7 @@ def test_sparseattentionutils_availability(): def test_cpp_utils_availability(): return True try: - from deepspeed.ops.sparse_attention import cpp_utils + from deepspeed.ops.sparse_attention import cpp_utils # noqa: F401 except ImportError: print("Sparse Attention cpp_utils Module is not installed!") return False diff --git a/tests/unit/test_sparse_grads.py b/tests/unit/test_sparse_grads.py index b146946f30a8..5be8ec3968fb 100644 --- a/tests/unit/test_sparse_grads.py +++ b/tests/unit/test_sparse_grads.py @@ -1,7 +1,5 @@ import torch -import deepspeed.comm as dist import deepspeed -import pytest from .common import distributed_test import deepspeed.utils.groups as groups diff --git a/tests/unit/test_zero.py b/tests/unit/test_zero.py index 453eaaadb0f7..b580fc4eaaa5 100755 --- a/tests/unit/test_zero.py +++ b/tests/unit/test_zero.py @@ -10,7 +10,7 @@ from torch.nn.parameter import Parameter from .common import distributed_test -from .simple_model import SimpleModel, random_dataloader, args_from_dict +from .simple_model import SimpleModel, random_dataloader import deepspeed from deepspeed.runtime.engine import DeepSpeedEngine diff --git a/tests/unit/test_zero_tiled.py b/tests/unit/test_zero_tiled.py index 474ad02e7c27..e76734308ac9 100644 --- a/tests/unit/test_zero_tiled.py +++ b/tests/unit/test_zero_tiled.py @@ -1,7 +1,6 @@ import copy import torch -import deepspeed from deepspeed.runtime.zero.tiling import TiledLinear, TiledLinearReturnBias import pytest