Add flake8 to pre-commit checks

.pre-commit-config.yaml

@@ -54,3 +54,9 @@ repos:
             --check-filenames,
             --check-hidden
         ]
+
+-   repo: https://github.com/pycqa/flake8
+    rev: 4.0.1
+    hooks:
+    -   id: flake8
+        args: ['--ignore=E,F403,F405,F541,F841,W', '--select=E9,F,W6', '--per-file-ignores=__init__.py:F401']

benchmarks/communication/all_gather.py

@@ -1,12 +1,7 @@
-import torch
 from benchmarks.communication.utils import *
 from benchmarks.communication.constants import *
 
 import time
-import argparse
-import os
-
-import math
 
 
 # Run all_gather and print metrics

benchmarks/communication/all_reduce.py

@@ -1,11 +1,7 @@
-import torch
 from benchmarks.communication.utils import *
 from benchmarks.communication.constants import *
 
 import time
-import argparse
-import os
-import math
 
 
 def timed_all_reduce(input, args):

benchmarks/communication/all_to_all.py

@@ -1,11 +1,7 @@
-import torch
 from benchmarks.communication.utils import *
 from benchmarks.communication.constants import *
 
 import time
-import argparse
-import os
-import math
 
 
 def timed_all_to_all(input, output, args):

benchmarks/communication/broadcast.py

@@ -3,9 +3,6 @@
 from benchmarks.communication.constants import *
 
 import time
-import argparse
-import os
-import math
 
 
 def timed_broadcast(input, args):

benchmarks/communication/constants.py

@@ -1,5 +1,3 @@
-import torch
-
 DEFAULT_WARMUPS = 5
 DEFAULT_TRIALS = 50
 DEFAULT_TYPE = 'float'

benchmarks/communication/pt2pt.py

@@ -1,11 +1,7 @@
-import torch
 from benchmarks.communication.utils import *
 from benchmarks.communication.constants import *
 
 import time
-import argparse
-import os
-import math
 
 
 def timed_pt2pt(input, args):

benchmarks/communication/run_all.py

@@ -1,4 +1,3 @@
-import torch
 from benchmarks.communication.utils import *
 from benchmarks.communication.all_reduce import run_all_reduce
 from benchmarks.communication.all_gather import run_all_gather
@@ -7,10 +6,6 @@
 from benchmarks.communication.broadcast import run_broadcast
 from benchmarks.communication.constants import *
 
-import time
-import argparse
-import os
-
 
 # For importing
 def main(args, rank):

bin/ds_bench

@@ -3,7 +3,6 @@
 from benchmarks.communication.run_all import main
 from benchmarks.communication.constants import *
 from benchmarks.communication.utils import *
-import argparse
 import os
 import sys
 

csrc/aio/py_test/aio_bench_perf_sweep.py

@@ -277,7 +277,6 @@ def script_path():
 
 
 def async_io_setup():
-    import deepspeed
     from deepspeed.ops.aio import AsyncIOBuilder
     return AsyncIOBuilder().is_compatible()
 

csrc/aio/py_test/parse_aio_stats.py

@@ -7,7 +7,6 @@
 
 import os
 import argparse
-import re
 
 READ_SPEED = 'read_speed'
 WRITE_SPEED = 'write_speed'

csrc/aio/py_test/test_ds_aio.py

@@ -6,11 +6,7 @@
 """
 
 import os
-import torch
 import argparse
-import time
-import sys
-from multiprocessing import Pool
 import multiprocessing as mp
 from ds_aio_basic import aio_basic_multiprocessing
 from ds_aio_handle import aio_handle_multiprocessing

csrc/aio/py_test/test_ds_aio_utils.py

@@ -5,8 +5,6 @@
 Functionality of swapping optimizer tensors to/from (NVMe) storage devices.
 """
 
-import os
-
 BYTES_PER_GB = 1024**3
 LOG_TIDS = [0]
 

csrc/aio/py_test/validate_async_io.py

@@ -4,6 +4,5 @@
 
 Functionality of swapping optimizer tensors to/from (NVMe) storage devices.
 """
-import deepspeed
 from deepspeed.ops.aio import AsyncIOBuilder
 assert AsyncIOBuilder().is_compatible()

deepspeed/autotuning/autotuner.py

@@ -1,10 +1,5 @@
-import copy
-import json
-import os
-from random import sample
 import shutil
 import subprocess
-import hjson
 import torch
 import time
 import datetime
@@ -13,10 +8,9 @@
 from ..runtime.config_utils import dict_raise_error_on_duplicate_keys
 from ..runtime.constants import *
 from ..runtime.zero.constants import *
-from ..utils import logger
 from .config import DeepSpeedAutotuningConfig
 from .constants import *
-from .scheduler import ResourceManager, run_experiment
+from .scheduler import ResourceManager
 from .tuner import GridSearchTuner, RandomTuner, ModelBasedTuner
 from .utils import *
 

deepspeed/autotuning/scheduler.py

@@ -1,17 +1,11 @@
 import copy
-from re import I
 
 from numpy import BUFSIZE
-from deepspeed.env_report import SUCCESS
-from enum import Flag
 import json
-import os
 import subprocess
 import sys
 import threading
 import time
-from pathlib import Path
-from typing import List
 
 import hjson
 from tqdm import tqdm
@@ -27,8 +21,6 @@
 
 from deepspeed import comm as dist
 
-from datetime import datetime
-
 TIMEOUT = 5
 
 

deepspeed/autotuning/tuner/base_tuner.py

@@ -1,12 +1,9 @@
-import atexit
 import sys
 
 from deepspeed.autotuning.constants import *
 from deepspeed.autotuning.utils import write_experiments
 from deepspeed.utils import logger
 
-import json
-
 
 class BaseTuner:
     def __init__(self, exps, resource_manager, metric):

deepspeed/autotuning/tuner/cost_model.py

@@ -1,5 +1,3 @@
-import numpy as np
-
 from .utils import *
 
 try:

deepspeed/autotuning/tuner/index_based_tuner.py

@@ -1,7 +1,5 @@
 import random
 
-from deepspeed.utils import logger
-
 from .base_tuner import BaseTuner
 
 

deepspeed/autotuning/tuner/model_based_tuner.py

@@ -1,8 +1,6 @@
 import hjson
-import numpy as np
-from deepspeed.utils import logger
 
-from ..constants import AUTOTUNING, AUTOTUNING_METRIC_PATH, AUTOTUNING_METRIC_DEFAULT
+from ..constants import AUTOTUNING, AUTOTUNING_METRIC_PATH
 from .base_tuner import BaseTuner
 from .cost_model import XGBoostCostModel
 from .utils import *

deepspeed/autotuning/utils.py

@@ -3,8 +3,6 @@
 import os
 import json
 from deepspeed.runtime.constants import GRADIENT_ACCUMULATION_STEPS, TRAIN_MICRO_BATCH_SIZE_PER_GPU
-import hjson
-import sys
 import itertools
 import copy
 
@@ -35,23 +33,11 @@ def was_interruptted(filename):
     return False
 
 
-def was_interruptted(filename):
-    if not os.path.exists(filename):
-        return "stderr.log does not exist"
-    with open(filename) as f:
-        for line in f:
-            s = "KeyboardInterrupt"
-            idx = line.find(s)
-            if idx != -1:
-                return True
-    return False
-
-
 def find_replace_str(value, replace_dict):
     if not isinstance(value, str):
         return str(value)
 
-    matches = re.findall("\$[A-Za-z0-9_]+", value)
+    matches = re.findall(r"\$[A-Za-z0-9_]+", value)
     for var in matches:
         var_key = var.replace("$", "").lower()
         if var_key == "nvme_path":

deepspeed/comm/comm.py

@@ -28,7 +28,6 @@
 from enum import Enum
 import torch
 import os
-import time
 
 from ..constants import TORCH_DISTRIBUTED_DEFAULT_PORT, default_pg_timeout
 from .constants import *
@@ -48,7 +47,6 @@ class ReduceOp(Enum):
 
 from deepspeed.utils.comms_logging import CommsLogger
 from deepspeed.utils import timer, get_caller_func
-from deepspeed.comm.backend import Backend
 from deepspeed.comm.torch import TorchBackend
 
 from deepspeed import utils

deepspeed/comm/config.py

@@ -3,9 +3,7 @@
 Licensed under the MIT license.
 """
 
-from typing import Optional
-from deepspeed.runtime.config_utils import get_scalar_param
-from pydantic import BaseModel, validator, ValidationError, create_model
+from pydantic import BaseModel
 from .constants import *
 
 

deepspeed/comm/torch.py

@@ -1,12 +1,7 @@
 '''
 Copyright 2021 The Microsoft DeepSpeed Team
 '''
-import os
-import torch
-
 from deepspeed import utils
-from ..constants import TORCH_DISTRIBUTED_DEFAULT_PORT, default_pg_timeout
-from datetime import timedelta
 
 from .utils import *
 from .backend import *

deepspeed/comm/utils.py

@@ -1,5 +1,4 @@
 import os
-import enum
 import torch
 import inspect
 from deepspeed.utils import get_caller_func
@@ -145,7 +144,7 @@ def get_msg_size_from_args(func, *args, **kwargs):
         # Sum of tensor sizes for list colls such as torch's all_to_all
         # NOTE: msg_size for list colls will not be the actual size transmitted by a given MPI/NCCL call within the coll op. Instead, it's the total amount of data transmitted.
         if type(tensor_arg) is list:
-            return sum(x.element_size() * x.nelement() for x in func_args['tensor_list'])
+            return sum(x.element_size() * x.nelement() for x in tensor_arg)
         else:
             return tensor_arg.element_size() * tensor_arg.nelement()
 

deepspeed/compression/basic_layer.py

@@ -1,4 +1,3 @@
-import copy
 import torch
 import math
 from torch import nn

deepspeed/compression/compress.py

@@ -1,4 +1,3 @@
-import torch
 import re
 from .helper import compression_preparation, fix_compression, recursive_getattr, is_module_compressible
 from .config import get_compression_config
@@ -7,8 +6,6 @@
 import os
 import json
 
-from deepspeed.utils import logger
-
 
 def check_deepspeed_config(config):
     if isinstance(config, dict):

deepspeed/elasticity/elasticity.py

@@ -2,7 +2,6 @@
 Copyright 2020 The Microsoft DeepSpeed Team
 """
 import os
-import re
 import json
 import numpy as np
 
@@ -11,8 +10,7 @@
 from .config import ElasticityConfig, ElasticityConfigError, ElasticityError, \
     ElasticityIncompatibleWorldSize
 from .constants import ELASTICITY, ENABLED, ENABLED_DEFAULT, LATEST_ELASTICITY_VERSION, \
-    MINIMUM_DEEPSPEED_VERSION, IGNORE_NON_ELASTIC_BATCH_INFO, \
-    IGNORE_NON_ELASTIC_BATCH_INFO_DEFAULT, DEEPSPEED_ELASTICITY_CONFIG
+    MINIMUM_DEEPSPEED_VERSION, DEEPSPEED_ELASTICITY_CONFIG
 from ..git_version_info import version as __version__
 from ..utils import logger
 

deepspeed/env_report.py

@@ -4,7 +4,6 @@
 import argparse
 from .ops.op_builder import ALL_OPS
 from .git_version_info import installed_ops, torch_info
-from .ops import __compatible_ops__ as compatible_ops
 
 GREEN = '\033[92m'
 RED = '\033[91m'
@@ -54,7 +53,7 @@ def op_report(verbose=True):
 
 def ninja_installed():
     try:
-        import ninja
+        import ninja  # noqa: F401
     except ImportError:
         return False
     return True

deepspeed/git_version_info.py

@@ -1,6 +1,6 @@
 try:
     #  This is populated by setup.py
-    from .git_version_info_installed import *
+    from .git_version_info_installed import *  # noqa: F401
 except ModuleNotFoundError:
     import os
     if os.path.isfile('version.txt'):

deepspeed/inference/engine.py

@@ -5,7 +5,6 @@
 import os
 
 from deepspeed import comm as dist
-from deepspeed.utils import groups
 from deepspeed.utils.logging import log_dist
 
 from torch.nn.modules import Module
@@ -18,7 +17,6 @@
 from ..comm.comm import init_distributed
 from ..pipe import PipelineModule
 from ..moe.utils import has_moe_layers
-from ..moe.layer import MoE
 from ..runtime.zero import GatheredParameters
 from ..module_inject import LinearAllreduce, LinearLayer, Normalize, ReplaceWithTensorSlicing