Lint for Caffe converter (#6264)

* pylint fix

* update

* update

* fix

Makefile

@@ -282,8 +282,7 @@ cpplint:
 	--exclude_path src/operator/contrib/ctc_include
 
 pylint:
-# ideally we want to check all, such as: python tools example tests
-	pylint python/mxnet --rcfile=$(ROOTDIR)/tests/ci_build/pylintrc
+	pylint python/mxnet tools/caffe_converter/*.py --rcfile=$(ROOTDIR)/tests/ci_build/pylintrc
 
 doc: docs
 

tools/caffe_converter/caffe_parser.py

@@ -1,3 +1,5 @@
+"""Parse caffe's protobuf
+"""
 import re
 try:
     import caffe

tools/caffe_converter/convert_caffe_modelzoo.py

@@ -1,12 +1,13 @@
+"""Convert Caffe's modelzoo
+"""
 import os
-import requests
 import argparse
-import logging
 from convert_model import convert_model
 from convert_mean import convert_mean
 import mxnet as mx
 
 _mx_caffe_model = 'http://data.mxnet.io/models/imagenet/test/caffe/'
+
 """Dictionary for model meta information
 
 For each model, it requires three attributes:
@@ -21,6 +22,7 @@
   - top-5-acc : top 5 accuracy for testing
 """
 model_meta_info = {
+    # pylint: disable=line-too-long
     'bvlc_alexnet' : {
         'prototxt' : 'https://raw.githubusercontent.com/BVLC/caffe/master/models/bvlc_googlenet/deploy.prototxt',
         'caffemodel' : 'http://dl.caffe.berkeleyvision.org/bvlc_googlenet.caffemodel',
@@ -31,23 +33,23 @@
     'bvlc_googlenet' : {
         'prototxt' : 'https://raw.githubusercontent.com/BVLC/caffe/master/models/bvlc_googlenet/deploy.prototxt',
         'caffemodel' : 'http://dl.caffe.berkeleyvision.org/bvlc_googlenet.caffemodel',
-        'mean' : (123,117,104),
+        'mean' : (123, 117, 104),
         'top-1-acc' : 0.687,
         'top-5-acc' : 0.889
     },
     'vgg-16' : {
         'prototxt' : 'https://gist.githubusercontent.com/ksimonyan/211839e770f7b538e2d8/raw/c3ba00e272d9f48594acef1f67e5fd12aff7a806/VGG_ILSVRC_16_layers_deploy.prototxt',
         # 'caffemodel' : 'http://www.robots.ox.ac.uk/~vgg/software/very_deep/caffe/VGG_ILSVRC_16_layers.caffemodel',
         'caffemodel' : 'http://data.mxnet.io/models/imagenet/test/caffe/VGG_ILSVRC_16_layers.caffemodel',
-        'mean': (123.68,116.779,103.939),
+        'mean': (123.68, 116.779, 103.939),
         'top-1-acc' : 0.734,
         'top-5-acc' : 0.914
     },
     'vgg-19' : {
         'prototxt' : 'https://gist.githubusercontent.com/ksimonyan/3785162f95cd2d5fee77/raw/bb2b4fe0a9bb0669211cf3d0bc949dfdda173e9e/VGG_ILSVRC_19_layers_deploy.prototxt',
         # 'caffemodel' : 'http://www.robots.ox.ac.uk/~vgg/software/very_deep/caffe/VGG_ILSVRC_19_layers.caffemodel',
         'caffemodel' : 'http://data.mxnet.io/models/imagenet/test/caffe/VGG_ILSVRC_19_layers.caffemodel',
-        'mean' : (123.68,116.779,103.939),
+        'mean' : (123.68, 116.779, 103.939),
         'top-1-acc' : 0.731,
         'top-5-acc' : 0.913
     },
@@ -110,5 +112,5 @@ def convert_caffe_model(model_name, meta_info, dst_dir='./model'):
     parser.add_argument('model_name', help='can be '+', '.join(model_meta_info.keys()))
     args = parser.parse_args()
     assert args.model_name in model_meta_info, 'Unknown model ' + args.model_name
-    model_name, _ = convert_caffe_model(args.model_name, model_meta_info[args.model_name])
-    print('Model is saved into '+model_name)
+    fname, _ = convert_caffe_model(args.model_name, model_meta_info[args.model_name])
+    print('Model is saved into ' + fname)

tools/caffe_converter/convert_mean.py

@@ -1,6 +1,8 @@
+"""Convert caffe mean
+"""
+import argparse
 import mxnet as mx
 import numpy as np
-import argparse
 import caffe_parser
 
 def convert_mean(binaryproto_fname, output=None):
@@ -27,7 +29,7 @@ def convert_mean(binaryproto_fname, output=None):
         mean_blob.channels, mean_blob.height, mean_blob.width
     )
     # swap channels from Caffe BGR to RGB
-    img_mean_np[[0,2],:,:] = img_mean_np[[2,0],:,:]
+    img_mean_np[[0, 2], :, :] = img_mean_np[[2, 0], :, :]
     nd = mx.nd.array(img_mean_np)
     if output is not None:
         mx.nd.save(output, {"mean_image": nd})

tools/caffe_converter/convert_model.py

@@ -1,9 +1,11 @@
+"""Convert caffe model
+"""
 from __future__ import print_function
-import mxnet as mx
-import numpy as np
 import argparse
 import sys
 import caffe_parser
+import mxnet as mx
+import numpy as np
 from convert_symbol import convert_symbol
 
 def convert_model(prototxt_fname, caffemodel_fname, output_prefix=None):
@@ -32,7 +34,7 @@ def convert_model(prototxt_fname, caffemodel_fname, output_prefix=None):
          Input dimension
     """
     sym, input_dim = convert_symbol(prototxt_fname)
-    arg_shapes, output_shapes, aux_shapes = sym.infer_shape(data=tuple(input_dim))
+    arg_shapes, _, aux_shapes = sym.infer_shape(data=tuple(input_dim))
     arg_names = sym.list_arguments()
     aux_names = sym.list_auxiliary_states()
     arg_shape_dic = dict(zip(arg_names, arg_shapes))
@@ -46,8 +48,8 @@ def convert_model(prototxt_fname, caffemodel_fname, output_prefix=None):
     layers_proto = caffe_parser.get_layers(caffe_parser.read_prototxt(prototxt_fname))
 
     for layer_name, layer_type, layer_blobs in layer_iter:
-        if layer_type == 'Convolution' or layer_type == 'InnerProduct' or layer_type == 4 or layer_type == 14 \
-                or layer_type == 'PReLU':
+        if layer_type == 'Convolution' or layer_type == 'InnerProduct' \
+           or layer_type == 4 or layer_type == 14 or layer_type == 'PReLU':
             if layer_type == 'PReLU':
                 assert (len(layer_blobs) == 1)
                 wmat = layer_blobs[0].data
@@ -60,7 +62,8 @@ def convert_model(prototxt_fname, caffemodel_fname, output_prefix=None):
                 if len(layer_blobs[0].shape.dim) > 0:
                     wmat_dim = layer_blobs[0].shape.dim
                 else:
-                    wmat_dim = [layer_blobs[0].num, layer_blobs[0].channels, layer_blobs[0].height, layer_blobs[0].width]
+                    wmat_dim = [layer_blobs[0].num, layer_blobs[0].channels,
+                                layer_blobs[0].height, layer_blobs[0].width]
             else:
                 wmat_dim = list(layer_blobs[0].shape)
             wmat = np.array(layer_blobs[0].data).reshape(wmat_dim)
@@ -72,7 +75,8 @@ def convert_model(prototxt_fname, caffemodel_fname, output_prefix=None):
                     wmat[:, [0, 2], :, :] = wmat[:, [2, 0], :, :]
 
             assert(wmat.flags['C_CONTIGUOUS'] is True)
-            sys.stdout.write('converting layer {0}, wmat shape = {1}'.format(layer_name, wmat.shape))
+            sys.stdout.write('converting layer {0}, wmat shape = {1}'.format(
+                layer_name, wmat.shape))
             if len(layer_blobs) == 2:
                 bias = np.array(layer_blobs[1].data)
                 bias = bias.reshape((bias.shape[0], 1))
@@ -116,14 +120,15 @@ def convert_model(prototxt_fname, caffemodel_fname, output_prefix=None):
 
             assert gamma.flags['C_CONTIGUOUS'] is True
             assert beta.flags['C_CONTIGUOUS'] is True
-            print ('converting scale layer, beta shape = {}, gamma shape = {}'.format(beta.shape, gamma.shape))
+            print('converting scale layer, beta shape = {}, gamma shape = {}'.format(
+                beta.shape, gamma.shape))
         elif layer_type == 'BatchNorm':
             bn_name = layer_name
             mean = layer_blobs[0].data
             var = layer_blobs[1].data
             rescale_factor = layer_blobs[2].data
             if rescale_factor != 0:
-            	rescale_factor = 1 / rescale_factor
+                rescale_factor = 1 / rescale_factor
             mean_name = '{}_moving_mean'.format(bn_name)
             var_name = '{}_moving_var'.format(bn_name)
             mean = mean.reshape(aux_shape_dic[mean_name])
@@ -132,21 +137,22 @@ def convert_model(prototxt_fname, caffemodel_fname, output_prefix=None):
             aux_params[var_name] = mx.nd.zeros(var.shape)
             # Get the original epsilon
             for idx, layer in enumerate(layers_proto):
-            	if layer.name == bn_name:
-            		bn_index = idx
+                if layer.name == bn_name:
+                    bn_index = idx
             eps_caffe = layers_proto[bn_index].batch_norm_param.eps
             # Compensate for the epsilon shift performed in convert_symbol
-            eps_symbol = float( sym.attr_dict()[bn_name + '_moving_mean']['eps'] )
+            eps_symbol = float(sym.attr_dict()[bn_name + '_moving_mean']['eps'])
             eps_correction = eps_caffe - eps_symbol
             # Fill parameters
             aux_params[mean_name][:] = mean * rescale_factor
             aux_params[var_name][:] = var * rescale_factor + eps_correction
             assert var.flags['C_CONTIGUOUS'] is True
             assert mean.flags['C_CONTIGUOUS'] is True
-            print ('converting batchnorm layer, mean shape = {}, var shape = {}'.format(mean.shape, var.shape))
+            print('converting batchnorm layer, mean shape = {}, var shape = {}'.format(
+                mean.shape, var.shape))
         else:
             assert len(layer_blobs) == 0
-            print ('\tskipping layer {} of type {}'.format(layer_name, layer_type))
+            print('\tskipping layer {} of type {}'.format(layer_name, layer_type))
 
     if output_prefix is not None:
         model = mx.mod.Module(symbol=sym, label_names=['prob_label', ])
@@ -157,8 +163,8 @@ def convert_model(prototxt_fname, caffemodel_fname, output_prefix=None):
     return sym, arg_params, aux_params, input_dim
 
 def main():
-    parser = argparse.ArgumentParser(description='Caffe prototxt to mxnet model parameter converter.\
-                    Note that only basic functions are implemented. You are welcomed to contribute to this file.')
+    parser = argparse.ArgumentParser(
+        description='Caffe prototxt to mxnet model parameter converter.')
     parser.add_argument('prototxt', help='The prototxt filename')
     parser.add_argument('caffemodel', help='The binary caffemodel filename')
     parser.add_argument('save_model_name', help='The name of the output model prefix')