add lint and format check (Oneflow-Inc#59)

Author: L1aoXingyu

.flake8

@@ -0,0 +1,16 @@
+# This is an example .flake8 config, used when developing *Black* itself.
+# Keep in sync with setup.cfg which is used for source packages.
+
+[flake8]
+ignore = W503, E203, E221, C901, C408, E741, C407, B017
+max-line-length = 100
+max-complexity = 18
+select = B,C,E,F,W,T4,B9
+exclude = build
+per-file-ignores =
+  **/__init__.py:F401,F403,E402
+  **/configs/**.py:F401,E402
+  configs/**.py:F401,E402
+  **/tests/config/**.py:F401,E402
+  tests/config/**.py:F401,E402
+  tests/**.py: E402

.github/workflows/py.yml

@@ -19,34 +19,30 @@ jobs:
     if: github.event.pull_request.draft == false && contains(github.event.pull_request.requested_reviewers.*.login, 'oneflow-ci-bot')
     steps:
       - uses: actions/checkout@v2
+
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v2
         with:
-          ref: ${{ github.event.pull_request.head.ref }}
-          repository: ${{github.event.pull_request.head.repo.full_name}}
-      - name: Checkout Oneflow-Inc/oneflow
-        uses: actions/checkout@v2
-        with:
-          repository: Oneflow-Inc/oneflow
-          path: ${{ env.ONEFLOW_SRC }}
-      - name: Check Python format
-        id: py_check
-        run: |
-          python3 -m pip install black==19.10b0
-          python3 ${{ env.ONEFLOW_SRC }}/ci/check/run_py_format.py --source_dir $PWD
-      - name: Run Python Format
-        id: py_fmt
-        if: ${{ failure() }}
+          python-version: 3.6
+
+      - name: Install dependencies
+        # flake8-bugbear flake8-comprehensions are useful but not available internally
         run: |
-          python3 -m pip install black==19.10b0
-          python3 ${{ env.ONEFLOW_SRC }}/ci/check/run_py_format.py --source_dir $PWD --fix
-      - name: Git push
-        id: git_push
-        if: ${{ failure() }}
+          python -m pip install --upgrade pip
+          python -m pip install flake8==3.8.1 isort==5.10.1 autoflake==1.4
+          python -m pip install black==21.4b2
+          flake8 --version
+
+      - name: Lint
+        id: py_check
         run: |
-          git config --global user.email "[email protected]"
-          git config --global user.name "oneflow-ci-bot"
-          git add -u
-          git commit -m "auto format by CI"
-          git push
+          echo "Running autoflake"
+          autoflake --ignore-init-module-imports --remove-all-unused-imports -r .
+          echo "Running isort"
+          isort --diff --check .
+          echo "Running flake8"
+          flake8 .
+
       - name: Please request CI again
         if: ${{ failure() }}
         run: |

Makefile

@@ -0,0 +1,10 @@
+format:
+	autoflake -i --ignore-init-module-imports --remove-all-unused-imports -r .
+	black .
+	isort .
+	flake8 .
+
+lint:
+	autoflake --ignore-init-module-imports --remove-all-unused-imports -r .
+	isort --diff --check .
+	flake8 .

configs/bert_large_pretrain.py

@@ -1,8 +1,6 @@
 from libai.config import LazyCall
 from .common.models.bert import pretrain_model as model
 from .common.train import train
-from .common.optim import optim, scheduler
-from .common.data.nlp_data import data
 
 from libai.models import BertForPretrainingGraph
 
@@ -21,13 +19,13 @@
 graph = dict(
     # options for graph or eager mode
     enabled=True,
-    debug=-1, # debug mode for graph
+    debug=-1,  # debug mode for graph
     train_graph=LazyCall(BertForPretrainingGraph)(
         fp16=train.amp.enabled,
         is_train=True,
     ),
     eval_graph=LazyCall(BertForPretrainingGraph)(
-        fp16=train.amp.enabled, 
+        fp16=train.amp.enabled,
         is_train=False
     ),
 )

configs/common/data/cv_data.py

@@ -1,12 +1,6 @@
 from omegaconf import OmegaConf
-from libai.config.instantiate import instantiate
 
-from libai.data.datasets import (
-    ImageNetDataset,
-    CIFAR10Dataset,
-    CIFAR100Dataset,
-    MNISTDataset,
-)
+from libai.data.datasets import ImageNetDataset
 from libai.data.build import build_image_train_loader, build_image_test_loader
 from libai.config import LazyCall
 

configs/common/data/nlp_data.py

@@ -5,16 +5,23 @@
 
 tokenizer = dict(
     tokenizer_name="BertTokenizer",
-    tokenizer_cfg=dict(vocab_file="bert-vocab.txt", do_lower_case=True,),
+    tokenizer_cfg=dict(
+        vocab_file="bert-vocab.txt",
+        do_lower_case=True,
+    ),
     append_eod=False,
 )
 
 dataloader = OmegaConf.create()
 
 dataloader.train = LazyCall(build_nlp_train_val_test_loader)(
     dataset=[
-        LazyCall(DemoNlpDataset)(data_root="train1",),
-        LazyCall(DemoNlpDataset)(data_root="train2",),
+        LazyCall(DemoNlpDataset)(
+            data_root="train1",
+        ),
+        LazyCall(DemoNlpDataset)(
+            data_root="train2",
+        ),
     ],
     splits=[[949.0, 50.0, 1.0], [900.0, 99.0, 1.0]],
     weights=[0.5, 0.5],
@@ -23,9 +30,13 @@
 
 dataloader.test = [
     LazyCall(build_nlp_test_loader)(
-        dataset=LazyCall(DemoNlpDataset)(data_root="test1",)
+        dataset=LazyCall(DemoNlpDataset)(
+            data_root="test1",
+        )
     ),
     LazyCall(build_nlp_test_loader)(
-        dataset=LazyCall(DemoNlpDataset)(data_root="test2",)
+        dataset=LazyCall(DemoNlpDataset)(
+            data_root="test2",
+        )
     ),
 ]

configs/common/data/transform.py

@@ -1,16 +1,12 @@
 from flowvision import transforms
 from flowvision.transforms import InterpolationMode
-from flowvision.transforms.functional import str_to_interp_mode, str_to_pil_interp
+from flowvision.transforms.functional import str_to_interp_mode
 
 from flowvision.data.constants import (
     IMAGENET_DEFAULT_MEAN,
     IMAGENET_DEFAULT_STD,
 )
-from flowvision.data.auto_augment import (
-    rand_augment_transform,
-    augment_and_mix_transform,
-    auto_augment_transform,
-)
+from flowvision.data.auto_augment import rand_augment_transform
 from flowvision.data.random_erasing import RandomErasing
 
 from libai.config import LazyCall
@@ -37,22 +33,33 @@
         ),
         LazyCall(transforms.ToTensor)(),
         LazyCall(transforms.Normalize)(
-            mean=IMAGENET_DEFAULT_MEAN, std=IMAGENET_DEFAULT_STD,
+            mean=IMAGENET_DEFAULT_MEAN,
+            std=IMAGENET_DEFAULT_STD,
         ),
         LazyCall(RandomErasing)(
-            probability=0.25, mode="pixel", max_count=1, num_splits=0, device="cpu",
+            probability=0.25,
+            mode="pixel",
+            max_count=1,
+            num_splits=0,
+            device="cpu",
         ),
     ]
 )
 
 
 default_test_transform = LazyCall(transforms.Compose)(
     transforms=[
-        LazyCall(transforms.Resize)(size=256, interpolation=InterpolationMode.BICUBIC,),
-        LazyCall(transforms.CenterCrop)(size=224,),
+        LazyCall(transforms.Resize)(
+            size=256,
+            interpolation=InterpolationMode.BICUBIC,
+        ),
+        LazyCall(transforms.CenterCrop)(
+            size=224,
+        ),
         LazyCall(transforms.ToTensor)(),
         LazyCall(transforms.Normalize)(
-            mean=IMAGENET_DEFAULT_MEAN, std=IMAGENET_DEFAULT_STD,
+            mean=IMAGENET_DEFAULT_MEAN,
+            std=IMAGENET_DEFAULT_STD,
         ),
     ]
 )

configs/common/optim.py

@@ -6,7 +6,8 @@
 
 optim = LazyCall(flow.optim.AdamW)(
     parameters=LazyCall(get_default_optimizer_params)(
-        # parameters.model is meant to be set to the model object, before instantiating the optimizer.
+        # parameters.model is meant to be set to the model object,
+        # before instantiating the optimizer.
         clip_grad_max_norm=1.0,
         clip_grad_norm_type=2.0,
         weight_decay_norm=0.0,

configs/common/tokenizer.py

@@ -1,5 +1,8 @@
 tokenizer = dict(
     tokenizer_name="BertTokenizer",
-    tokenizer_cfg=dict(vocab_file="bert-vocab.txt", do_lower_case=True,),
+    tokenizer_cfg=dict(
+        vocab_file="bert-vocab.txt",
+        do_lower_case=True,
+    ),
     append_eod=False,
 )

libai/config/__init__.py

@@ -13,10 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .lazy import LazyCall, LazyConfig
-from .instantiate import instantiate
 from .arguments import default_argument_parser
 from .config import configurable, try_get_key
+from .instantiate import instantiate
+from .lazy import LazyCall, LazyConfig
 
 __all__ = [
     "LazyCall",

libai/config/arguments.py

@@ -14,8 +14,8 @@
 # limitations under the License.
 
 
-import sys
 import argparse
+import sys
 
 
 def default_argument_parser(epilog=None):
@@ -35,18 +35,22 @@ def default_argument_parser(epilog=None):
 
 Run on single machine:
     $ python3 -m oneflow.distributed.launch \
-    --nproc_per_node 8 --nnodes 1 --node_rank 0 --master_addr 127.0.0.1 {sys.argv[0]} --config-file cfg.yaml
+    --nproc_per_node 8 --nnodes 1 --node_rank 0 --master_addr 127.0.0.1 {sys.argv[0]} \
+    --config-file cfg.yaml
 
 Change some config options:
     $ python3 -m oneflow.distributed.launch \
-    --nproc_per_node 8 --nnodes 1 --node_rank 0 --master_addr 127.0.0.1 {sys.argv[0]} --config-file cfg.yaml train.load_weight=/path/to/weight.pth optim.lr=0.001
+    --nproc_per_node 8 --nnodes 1 --node_rank 0 --master_addr 127.0.0.1 {sys.argv[0]} \
+    --config-file cfg.yaml train.load_weight=/path/to/weight.pth optim.lr=0.001
 
 Run on multiple machines:
     (machine0)$ python3 -m oneflow.distributed.launch \
-    --nproc_per_node 8 --nnodes 2 --node_rank 0 --master_addr <URL> {sys.argv[0]} --config-file cfg.yaml
-    
+    --nproc_per_node 8 --nnodes 2 --node_rank 0 --master_addr <URL> {sys.argv[0]} \
+    --config-file cfg.yaml
+
     $ python3 -m oneflow.distributed.launch \
-    --nproc_per_node 8 --nnodes 2 --node_rank 1 --master_addr <URL> {sys.argv[0]} --config-file cfg.yaml 
+    --nproc_per_node 8 --nnodes 2 --node_rank 1 --master_addr <URL> {sys.argv[0]} \
+    --config-file cfg.yaml
 
 """,
         formatter_class=argparse.RawDescriptionHelpFormatter,

libai/config/config.py

@@ -13,10 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from omegaconf import OmegaConf
 import functools
 import inspect
 
+from omegaconf import OmegaConf
+
 
 def configurable(init_func=None, *, from_config=None):
     """

libai/config/lazy.py

@@ -13,19 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import pydoc
 import ast
 import builtins
 import importlib
 import inspect
 import logging
 import os
+import pydoc
 import uuid
 from collections import abc
 from contextlib import contextmanager
 from copy import deepcopy
 from dataclasses import is_dataclass
-from typing import List, Tuple, Union, Any
+from typing import Any, List, Tuple, Union
+
 import cloudpickle
 import yaml
 from omegaconf import DictConfig, ListConfig, OmegaConf

libai/data/__init__.py

@@ -13,5 +13,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from .build import build_image_test_loader, build_image_train_loader
 from .structures import DistTensorData, Instance
-from .build import build_image_train_loader, build_image_test_loader

libai/data/build.py

@@ -18,8 +18,9 @@
 from oneflow.utils.data.dataset import ConcatDataset
 
 from libai.utils import distributed as dist
+
 from .structures import Instance
-from .temp_file import CyclicSampler, SingleRoundSampler, BlendableDataset, split_ds
+from .temp_file import CyclicSampler, SingleRoundSampler, split_ds
 
 
 def build_nlp_train_val_test_loader(
@@ -35,8 +36,8 @@ def build_nlp_train_val_test_loader(
     collate_fn=None,
     blendable_dataset=ConcatDataset,
 ):
-    """ 
-    Build nlp train_val_test dataloder
+    """
+    Build nlp train_val_test dataloader
     """
     # TODO: add input type
 
@@ -115,9 +116,14 @@ def build_nlp_train_val_test_loader(
 
 
 def build_nlp_test_loader(
-    dataset, test_batch_size, sampler=None, num_workers=4, seed=0, collate_fn=None,
+    dataset,
+    test_batch_size,
+    sampler=None,
+    num_workers=4,
+    seed=0,
+    collate_fn=None,
 ):
-    """ 
+    """
     Build nlp test dataloder
     """
     # TODO: add input type

libai/data/data_utils/__init__.py

@@ -13,13 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from .blendable_dataset import BlendableDataset
+from .data_utils import get_prefixes_and_weights
 from .indexed_dataset import (
-    get_indexed_dataset,
-    IndexedDataset,
     IndexedCachedDataset,
+    IndexedDataset,
     MMapIndexedDataset,
+    get_indexed_dataset,
 )
-from .blendable_dataset import BlendableDataset
-from .data_utils import get_prefixes_and_weights
-from .reindexed_dataset import SentenceIndexedDataset, BlockIndexedDataset
-from .split_dataset import split_ds, SplitDataset
+from .reindexed_dataset import BlockIndexedDataset, SentenceIndexedDataset
+from .split_dataset import SplitDataset, split_ds