diff --git a/tensorflow_app/views/import_graphdef.py b/tensorflow_app/views/import_graphdef.py index 984d495ae..26a3a0720 100644 --- a/tensorflow_app/views/import_graphdef.py +++ b/tensorflow_app/views/import_graphdef.py @@ -1,128 +1,54 @@ -import numpy as np import tensorflow as tf from google.protobuf import text_format from tensorflow.core.framework import graph_pb2 from django.views.decorators.csrf import csrf_exempt from django.http import JsonResponse -import math -import re import urllib2 from urlparse import urlparse -# map from operation name(tensorflow) to layer name(caffe) -op_layer_map = {'Placeholder': 'Input', 'Conv2D': 'Convolution', 'Conv3D': 'Convolution', - 'MaxPool': 'Pooling', 'MaxPool3D': 'Pooling', 'AvgPool3D': 'Pooling', - 'DepthwiseConv2dNative': 'DepthwiseConv', 'MatMul': 'InnerProduct', - 'Prod': 'InnerProduct', 'LRN': 'LRN', 'Concat': 'Concat', - 'AvgPool': 'Pooling', 'Reshape': 'Flatten', 'FusedBatchNorm': 'BatchNorm', - 'Conv2DBackpropInput': 'Deconvolution'} - -activation_map = {'Sigmoid': 'Sigmoid', 'Softplus': 'Softplus', 'Softsign': 'Softsign', - 'Elu': 'ELU', 'LeakyRelu': 'ReLU', 'Softmax': 'Softmax', - 'Relu': 'ReLU', 'Tanh': 'TanH', 'SELU': 'SELU'} - -name_map = {'flatten': 'Flatten', 'dropout': 'Dropout', 'lrn': 'LRN', 'concatenate': 'Concat', - 'batch': 'BatchNorm', 'add': 'Eltwise', 'mul': 'Eltwise'} - -initializer_map = {'random_uniform': 'RandomUniform', 'random_normal': 'RandomNormal', - 'Const': 'Constant', 'zeros': 'Zeros', 'ones': 'Ones', - 'eye': 'Identity', 'truncated_normal': 'TruncatedNormal'} - - -def get_layer_name(node_name): - i = node_name.find('/') - if i == -1: - name = str(node_name) - elif str(node_name[:i]) in ['Repeat', 'Stack']: - name = str(node_name.split('/')[1]) - else: - name = str(node_name[:i]) - return name - - -def get_layer_type(node_name): - i = node_name.find('_') - if i == -1: - name = str(node_name) - else: - name = str(node_name[:i]) - return name - - -def get_padding(node, layer, session, input_layer_name, input_layer_dim): - layer_name = get_layer_name(node.name) - input_shape = None - output_shape = None - for input_tensor in node.inputs: - if get_layer_name(input_tensor.op.name) != layer_name: - input_shape = input_tensor.get_shape() - try: - # check to see if dimensions are present in shape - temp = [int(i) for i in input_shape[1:]] - except: - input_shape = (session.run(input_tensor, - feed_dict={session.graph.get_tensor_by_name(input_layer_name + ':0'): - np.zeros(input_layer_dim)})).shape - for output_tensor in node.outputs: - output_shape = output_tensor.get_shape() - try: - # check to see if dimensions are present in shape - temp = [int(i) for i in output_shape[1:]] # noqa - except: - output_shape = (session.run(output_tensor, - feed_dict={session.graph.get_tensor_by_name(input_layer_name + ':0'): - np.zeros(input_layer_dim)})).shape - - ''' - Use this link: https://www.tensorflow.org/versions/r0.10/api_docs/python/nn.html - pad_along_height = ((out_height - 1) * strides[1] + - filter_height - in_height) - pad_along_width = ((out_width - 1) * strides[2] + - filter_width - in_width) - pad_top = pad_along_height / 2 - pad_left = pad_along_width / 2 - ''' - - if node.type in ["Conv3D", "MaxPool3D", "AvgPool3D"]: - pad_d = ((int(output_shape[1]) - 1) * layer['params']['stride_d'] + - layer['params']['kernel_d'] - int(input_shape[1])) / float(2) - pad_h = ((int(output_shape[2]) - 1) * layer['params']['stride_h'] + - layer['params']['kernel_h'] - int(input_shape[2])) / float(2) - pad_w = ((int(output_shape[3]) - 1) * layer['params']['stride_w'] + - layer['params']['kernel_w'] - int(input_shape[3])) / float(2) - - if node.type == "Conv3D": - pad_d = math.ceil(pad_d) - pad_h = math.ceil(pad_h) - pad_w = math.ceil(pad_w) - elif node.type in ["MaxPool3D", "AvgPool3D"]: - pad_d = math.floor(pad_d) - pad_h = math.floor(pad_h) - pad_w = math.floor(pad_w) - - return int(pad_d), int(pad_h), int(pad_w) - elif node.type == "Conv2DBackpropInput": - # if deconvolution layer padding calculation logic changes - if ('padding' in node.node_def.attr): - pad_h = ((int(input_shape[1]) - 1) * layer['params']['stride_h'] + - layer['params']['kernel_h'] - int(output_shape[1])) / float(2) - pad_w = ((int(input_shape[2]) - 1) * layer['params']['stride_w'] + - layer['params']['kernel_w'] - int(output_shape[2])) / float(2) - return int(math.floor(pad_h)), int(math.floor(pad_w)) - else: - pad_h = ((int(output_shape[1]) - 1) * layer['params']['stride_h'] + - layer['params']['kernel_h'] - int(input_shape[1])) / float(2) - pad_w = ((int(output_shape[2]) - 1) * layer['params']['stride_w'] + - layer['params']['kernel_w'] - int(input_shape[2])) / float(2) - # check this logic (see caffe-tensorflow/kaffe/shapes.py) - if node.type == "Conv2D": - pad_h = math.ceil(pad_h) - pad_w = math.ceil(pad_w) - elif node.type in ["MaxPool", "AvgPool"]: - pad_h = math.floor(pad_h) - pad_w = math.floor(pad_w) - - return int(pad_h), int(pad_w) +from layers_import import import_placeholder, import_conv2d, import_conv3d, import_deconvolution, \ + import_depthwise_convolution, import_pooling2d, import_pooling3d, \ + import_inner_product, import_batchnorm, import_eltwise, import_activation, \ + import_dropout, import_flatten, import_concat, import_lrn + +from layers_import import get_layer_name, get_layer_type, jsonLayer, activation_layers + +layer_map = { + 'Placeholder': import_placeholder, + 'Conv2D': import_conv2d, + 'Conv3D': import_conv3d, + 'MaxPool': import_pooling2d, + 'MaxPool3D': import_pooling3d, + 'AvgPool3D': import_pooling3d, + 'DepthwiseConv2dNative': import_depthwise_convolution, + 'FusedBatchNorm': import_batchnorm, + 'Conv2DBackpropInput': import_deconvolution, + 'LRN': import_lrn, + 'MatMul': import_inner_product, + 'Prod': import_inner_product, + 'Concat': import_concat, + 'AvgPool': import_pooling2d, + 'Reshape': import_flatten +} + +name_map = { + 'flatten': import_flatten, + 'dropout': import_dropout, + 'lrn': import_lrn, + 'concatenate': import_concat, + 'batch': import_batchnorm, + 'BatchNorm': import_batchnorm, + 'add': import_eltwise, + 'mul': import_eltwise +} + + +def get_all_ops_in_layer(layer_name, all_ops): + ops_from_same_layer = [] + for op in all_ops: + if get_layer_name(op.name) == layer_name: + ops_from_same_layer.append(op) + return ops_from_same_layer @csrf_exempt @@ -134,6 +60,7 @@ def import_graph_def(request): try: f = request.FILES['file'] config = f.read() + f.close() except Exception: return JsonResponse({'result': 'error', 'error': 'No GraphDef model file found'}) elif 'config' in request.POST: @@ -152,10 +79,6 @@ def import_graph_def(request): tf.reset_default_graph() graph_def = graph_pb2.GraphDef() - d = {} - order = [] - input_layer_name = '' - input_layer_dim = [] try: text_format.Merge(config, graph_def) @@ -165,385 +88,87 @@ def import_graph_def(request): tf.import_graph_def(graph_def, name='') graph = tf.get_default_graph() session = tf.Session(graph=graph) + all_ops = graph.get_operations() - for node in graph.get_operations(): - name = get_layer_name(node.name) - if node.type == 'NoOp': - # if init ops is found initialize graph_def - init_op = session.graph.get_operation_by_name(node.name) - session.run(init_op) - continue - if name not in d: - d[name] = {'type': [], 'input': [], 'output': [], 'params': {}} - order.append(name) - if node.type in op_layer_map: - if (node.type == "FusedBatchNorm"): - d[name]['type'] = [] - d[name]['type'].append(op_layer_map[node.type]) - elif node.type in activation_map: - d[name]['type'].append(activation_map[node.type]) - else: # For cases where the ops are composed of only basic ops - layer_type = get_layer_type(node.name) - if layer_type in name_map: - if name_map[layer_type] not in d[name]['type']: - d[name]['type'].append(name_map[layer_type]) + net = {} + processed_layers = [] + layers_with_inplace_relu = {} - for input_tensor in node.inputs: - input_layer_name = get_layer_name(input_tensor.op.name) - if input_layer_name != name: - d[name]['input'].append(input_layer_name) - if name not in d[input_layer_name]['output']: - d[input_layer_name]['output'].append(name) + for node in all_ops: - # seperating relu layers - # This logic is only needed for inplace operations, it might be possible to do this - # in a better way - temp_d = {} - for layer_name in d: - if 'ReLU' in d[layer_name]['type'] and get_layer_type(layer_name) != 'activation': - relu_layer_name = layer_name + '_relu' - temp_d[relu_layer_name] = {'type': ['ReLU'], 'input': [layer_name], - 'output': d[layer_name]['output'], 'params': {}} - for output_layer_name in d[layer_name]['output']: - for n, i in enumerate(d[output_layer_name]['input']): - if i == layer_name: - d[output_layer_name]['input'][n] = relu_layer_name - d[layer_name]['output'] = [relu_layer_name] - for key in temp_d: - d[key] = temp_d[key] + layer_name = get_layer_name(node.name) + layer_type = get_layer_type(node.name) - # setting params - for node in graph.get_operations(): - if node.type == 'NoOp': + if layer_name in processed_layers: continue - name = get_layer_name(node.name) - layer = d[name] - - if layer['type'][0] == 'Input': - input_dim = [int(dim.size) for dim in node.get_attr('shape').dim] - # Swapping channel value to convert NCHW/NCDHW format - if input_dim[0] == -1: - input_dim[0] = 1 - # preserving input shape to calculate deconv output shape - input_layer_name = node.name - input_layer_dim = input_dim[:] - temp = input_dim[1] - input_dim[1] = input_dim[len(input_dim) - 1] - input_dim[len(input_dim) - 1] = temp - - layer['params']['dim'] = str(input_dim)[1:-1] - - elif layer['type'][0] == 'Convolution': - # searching for weights and kernel ops to extract kernel_h, - # kernl_w, num_outputs of conv layer also considering repeat & stack layer - # as prefixes - if str(node.name) in [name + '/weights', name + '/kernel', - 'Repeat/' + name + '/weights', 'Repeat/' + name + '/kernel', - 'Stack/' + name + '/weights', 'Stack/' + name + '/kernel']: - # since conv takes weights as input, this node will be processed first - # acquired parameters are then required in get_padding function - if len(node.get_attr('shape').dim) == 4: - layer['params']['kernel_h'] = int( - node.get_attr('shape').dim[0].size) - layer['params']['kernel_w'] = int( - node.get_attr('shape').dim[1].size) - layer['params']['num_output'] = int( - node.get_attr('shape').dim[3].size) - elif len(node.get_attr('shape').dim) == 5: - layer['params']['kernel_d'] = int( - node.get_attr('shape').dim[0].size) - layer['params']['kernel_h'] = int( - node.get_attr('shape').dim[1].size) - layer['params']['kernel_w'] = int( - node.get_attr('shape').dim[2].size) - layer['params']['num_output'] = int( - node.get_attr('shape').dim[4].size) - if node.type == 'Conv2D': - layer['params']['stride_h'] = int( - node.get_attr('strides')[1]) - layer['params']['stride_w'] = int( - node.get_attr('strides')[2]) - layer['params']['layer_type'] = '2D' - try: - layer['params']['pad_h'], layer['params']['pad_w'] = \ - get_padding(node, layer, session, input_layer_name, input_layer_dim) - except TypeError: - return JsonResponse({'result': 'error', 'error': - 'Missing shape info in GraphDef'}) - elif node.type == 'Conv3D': - layer['params']['stride_d'] = int( - node.get_attr('strides')[1]) - layer['params']['stride_h'] = int( - node.get_attr('strides')[2]) - layer['params']['stride_w'] = int( - node.get_attr('strides')[3]) - layer['params']['layer_type'] = '3D' - try: - layer['params']['pad_h'], layer['params']['pad_w'],\ - layer['params']['pad_d'] = \ - get_padding(node, layer, session, input_layer_name, input_layer_dim) - except TypeError: - return JsonResponse({'result': 'error', 'error': - 'Missing shape info in GraphDef'}) - - elif layer['type'][0] == 'Deconvolution': - if str(node.name) == name + '/weights' or str(node.name) == name + '/kernel': - # since conv takes weights as input, this node will be processed first - # acquired parameters are then required in get_padding function - layer['params']['kernel_h'] = int( - node.get_attr('shape').dim[0].size) - layer['params']['kernel_w'] = int( - node.get_attr('shape').dim[1].size) - layer['params']['num_output'] = int( - node.get_attr('shape').dim[3].size) - # extracting weight initializer - if re.match('.*/kernel/Initializer.*', str(node.name)): - w_filler = str(node.name).split('/')[3] - layer['params']['weight_filler'] = initializer_map[w_filler] - # extracting bias initializer - if re.match('.*/bias/Initializer.*', str(node.name)): - b_filler = str(node.name).split('/')[3] - layer['params']['bias_filler'] = initializer_map[b_filler] - # extracting stride height & width - if str(node.type) == 'Conv2DBackpropInput': - layer['params']['stride_h'] = int( - node.get_attr('strides')[1]) - layer['params']['stride_w'] = int( - node.get_attr('strides')[2]) - layer['params']['layer_type'] = '2D' - layer['params']['layer_type'] = node.get_attr('padding') - try: - layer['params']['pad_h'], layer['params']['pad_w'] = \ - get_padding(node, layer, session, input_layer_name, input_layer_dim) - except TypeError: - return JsonResponse({'result': 'error', 'error': - 'Missing shape info in GraphDef'}) - elif layer['type'][0] == 'DepthwiseConv': - if '3D' in node.type: - pass - else: - # searching for depthwise_weights and depthwise_kernel ops to extract kernel_h, - # kernl_w, num_outputs of deconv layer also considering repeat & stack layer - # as prefixes - if str(node.name) in [name + '/depthwise_weights', name + '/depthwise_kernel', - 'Repeat/' + name + '/depthwise_weights', - 'Repeat/' + name + '/depthwise_kernel', - 'Stack/' + name + '/depthwise_weights', - 'Stack/' + name + '/depthwise_kernel']: - layer['params']['kernel_h'] = int( - node.get_attr('shape').dim[0].size) - layer['params']['kernel_w'] = int( - node.get_attr('shape').dim[1].size) - layer['params']['num_output'] = int( - node.get_attr('shape').dim[2].size) - layer['params']['depth_multiplier'] = int( - node.get_attr('shape').dim[3].size) - # extract pointwise_initializer - if name + '/pointwise_weights/Initializer' in str(node.name): - if len(str(node.name).split('/')) >= 4: - pointwise_initializer = str(node.name).split('/')[3] - if pointwise_initializer in initializer_map: - layer['params']['pointwise_initializer'] = \ - initializer_map[pointwise_initializer] - # extract depthwise_initializer - if name + '/depthwise_weights/Initializer' in str(node.name): - if len(str(node.name).split('/')) >= 4: - depthwise_initializer = str(node.name).split('/')[3] - if depthwise_initializer in initializer_map: - layer['params']['depthwise_initializer'] = \ - initializer_map[depthwise_initializer] - # extract padding of layer - if 'padding' in node.node_def.attr: - layer['params']['padding'] = str(node.get_attr('padding')) - # extract strides of a layer - if 'strides' in node.node_def.attr and str(node.type) == "DepthwiseConv2dNative": - layer['params']['stride_h'] = int( - node.get_attr('strides')[1]) - layer['params']['stride_w'] = int( - node.get_attr('strides')[2]) - try: - layer['params']['pad_h'], layer['params']['pad_w'] = \ - get_padding(node, layer, session, input_layer_name, input_layer_dim) - pass - except TypeError: - return JsonResponse({'result': 'error', 'error': - 'Missing shape info in GraphDef'}) - - elif layer['type'][0] == 'Pooling': - layer['params']['padding'] = str(node.get_attr('padding')) - if '3D' in node.type: - # checking type of pooling layer - if node.type == 'MaxPool3D': - layer['params']['pool'] = 'MAX' - elif node.type == 'AvgPool3D': - layer['params']['pool'] = 'AVE' - - layer['params']['kernel_d'] = int( - node.get_attr('ksize')[1]) - layer['params']['kernel_h'] = int( - node.get_attr('ksize')[2]) - layer['params']['kernel_w'] = int( - node.get_attr('ksize')[3]) - layer['params']['stride_d'] = int( - node.get_attr('strides')[1]) - layer['params']['stride_h'] = int( - node.get_attr('strides')[2]) - layer['params']['stride_w'] = int( - node.get_attr('strides')[3]) - layer['params']['layer_type'] = '3D' - try: - layer['params']['pad_d'], layer['params']['pad_h'], \ - layer['params']['pad_w'] = \ - get_padding(node, layer, session, input_layer_name, input_layer_dim) - except TypeError: - return JsonResponse({'result': 'error', 'error': - 'Missing shape info in GraphDef'}) - - else: - # checking type of pooling layer - if node.type == 'MaxPool': - layer['params']['pool'] = 'MAX' - elif node.type == 'AvgPool': - layer['params']['pool'] = 'AVE' - - layer['params']['kernel_h'] = int( - node.get_attr('ksize')[1]) - layer['params']['kernel_w'] = int( - node.get_attr('ksize')[2]) - layer['params']['stride_h'] = int( - node.get_attr('strides')[1]) - layer['params']['stride_w'] = int( - node.get_attr('strides')[2]) - layer['params']['layer_type'] = '2D' - try: - layer['params']['pad_h'], layer['params']['pad_w'] = \ - get_padding(node, layer, session, input_layer_name, input_layer_dim) - except TypeError: - return JsonResponse({'result': 'error', 'error': - 'Missing shape info in GraphDef'}) - - elif layer['type'][0] == 'InnerProduct': - # searching for weights and kernel ops to extract num_outputs - # of IneerProduct layer also considering repeat & stack layer - # as prefixes - if str(node.name) in [name + '/weights', name + '/kernel', - 'Repeat/' + name + '/weights', 'Repeat/' + name + '/kernel', - 'Stack/' + name + '/weights', 'Stack/' + name + '/kernel']: - layer['params']['num_output'] = int( - node.get_attr('shape').dim[1].size) - - elif layer['type'][0] == 'BatchNorm': - if re.match('.*\/batchnorm[_]?[0-9]?\/add.*', str(node.name)): - try: - layer['params']['eps'] = node.get_attr( - 'value').float_val[0] - except: - pass - - if (node.type == 'FusedBatchNorm'): - layer['params']['eps'] = float(node.get_attr( - 'epsilon')) - # searching for moving_mean/Initializer ops to extract moving - # mean initializer of batchnorm layer - if name + '/moving_mean/Initializer' in str(node.name): - layer['params']['moving_mean_initializer'] = \ - initializer_map[str(node.name).split('/')[3]] - # searching for AssignMovingAvg/decay ops to extract moving - # average fraction of batchnorm layer also considering repeat & stack layer - # as prefixes - if str(node.name) in [name + '/AssignMovingAvg/decay', - 'Repeat/' + name + '/AssignMovingAvg/decay', - 'Stack/' + name + '/AssignMovingAvg/decay']: - layer['params']['moving_average_fraction'] = node.get_attr( - 'value').float_val[0] - - elif layer['type'][0] == 'Eltwise': - if str(node.name).split('_')[0] == 'add': - layer['params']['layer_type'] = 'Sum' - if str(node.name).split('_')[0] == 'mul': - layer['params']['layer_type'] = 'Product' - if str(node.name).split('_')[0] == 'dot': - layer['params']['layer_type'] = 'Dot' - - elif layer['type'][0] == 'ReLU': - # if layer is a LeakyReLU layer - if 'alpha' in node.node_def.attr: - layer['params']['negative_slope'] = node.get_attr('alpha') - - elif layer['type'][0] == 'ELU': - # default value as tf.nn.elu layer computes exp(feature)-1 if < 0 - layer['params']['alpha'] = 1 - - elif layer['type'][0] == 'Softplus': - pass - - elif layer['type'][0] == 'Softsign': - pass - - elif layer['type'][0] == 'SELU': - pass - - elif layer['type'][0] == 'TanH': - pass - - elif layer['type'][0] == 'Concat': - if 'axis' in node.node_def.attr: - layer['params']['axis'] = node.get_attr('axis') - - elif layer['type'][0] == 'LRN': - if ('alpha' in node.node_def.attr): - layer['params']['alpha'] = node.get_attr('alpha') - if ('beta' in node.node_def.attr): - layer['params']['beta'] = node.get_attr('beta') - if ('local_size' in node.node_def.attr): - layer['params']['local_size'] = node.get_attr('depth_radius') - if ('bias' in node.node_def.attr): - layer['params']['k'] = node.get_attr('bias') - - elif layer['type'][0] == 'Softmax': - pass - - elif layer['type'][0] == 'Flatten': - pass - - elif layer['type'][0] == 'Dropout': - if ('rate' in node.node_def.attr): - layer['params']['rate'] = node.get_attr('rate') - if ('seed' in node.node_def.attr): - layer['params']['seed'] = node.get_attr('seed') - if ('training' in node.node_def.attr): - layer['params']['trainable'] = node.get_attr('training') - net = {} - batch_norms = [] - for key in d.keys(): - if d[key]['type'][0] == 'BatchNorm' and len(d[key]['input']) > 0 and len(d[key]['output']) > 0: - batch_norms.append(key) - - temp_d_batch = {} - for layer_name in batch_norms: - scale_layer_name = layer_name + '_scale' - temp_d_batch[scale_layer_name] = {'type': ['Scale'], 'input': [layer_name], - 'output': d[layer_name]['output'], 'params': {}} - for output_layer_name in d[layer_name]['output']: - for n, i in enumerate(d[output_layer_name]['input']): - if i == layer_name: - d[output_layer_name]['input'][n] = scale_layer_name - d[layer_name]['output'] = [scale_layer_name] - for key in temp_d_batch: - d[key] = temp_d_batch[key] + if node.type == 'NoOp': + init_op = session.graph.get_operation_by_name(node.name) + session.run(init_op) + continue - for key in d.keys(): - net[key] = { - 'info': { - 'type': d[key]['type'][0], - 'phase': None - }, - 'connection': { - 'input': d[key]['input'], - 'output': d[key]['output'] - }, - 'params': d[key]['params'] - } + all_ops_in_layer = get_all_ops_in_layer(layer_name, all_ops) + for op in all_ops_in_layer: + if op.type == 'FusedBatchNorm': + net[layer_name] = import_batchnorm(all_ops_in_layer) + processed_layers.append(layer_name) + + if node.type in layer_map: + for i, op in enumerate(all_ops_in_layer): + # if the layer has an inplace relu operation, separate the relu op + # this prevents net[layer_name] from being overwritten by an inplace + # relu layer when the layer might actually contain another important + # layer like a dense layer for example + if op.type == 'Relu': + del(all_ops_in_layer[i]) + relu_layer = jsonLayer('ReLU', {}, [layer_name]) + relu_layer_name = layer_name + '_relu' + net[relu_layer_name] = relu_layer + layers_with_inplace_relu[layer_name] = relu_layer_name + json_layer = layer_map[node.type](all_ops_in_layer) + net[layer_name] = json_layer + processed_layers.append(layer_name) + + elif node.type in activation_layers: + json_layer = import_activation(all_ops_in_layer) + net[layer_name] = json_layer + processed_layers.append(layer_name) + + elif layer_type in name_map: + json_layer = name_map[layer_type](all_ops_in_layer) + net[layer_name] = json_layer + processed_layers.append(layer_name) + + # connect layers with the previous layer's inplace relu ops, if any + for layer_name in net: + for i, input_layer in enumerate(net[layer_name]['connection']['input']): + if (input_layer in layers_with_inplace_relu.keys()) and \ + layers_with_inplace_relu[input_layer] != layer_name: + net[layer_name]['connection']['input'][i] = layers_with_inplace_relu[input_layer] + + # fill in outputs of every layer in net using inputs of consumer layers + outputs = {} + for layer_name in net.keys(): + for input_layer_name in net[layer_name]['connection']['input']: + if input_layer_name not in outputs: + outputs[input_layer_name] = [] + if layer_name not in outputs[input_layer_name]: + outputs[input_layer_name].append(layer_name) + for layer in outputs: + net[layer]['connection']['output'] = outputs[layer] + + # add a scale layer next to batch normalization layers + scale_layers = {} + for layer_name in net: + if net[layer_name]['info']['type'] == 'BatchNorm': + batch_norm_outputs = net[layer_name]['connection']['output'][:] + scale_layer_name = layer_name + '_scale' + scale_layer = jsonLayer('Scale', {}, [layer_name], batch_norm_outputs) + net[layer_name]['connection']['output'] = [scale_layer_name] + scale_layers[scale_layer_name] = scale_layer + for scale_layer_name in scale_layers: + net[scale_layer_name] = scale_layers[scale_layer_name] + + session.close() return JsonResponse({'result': 'success', 'net': net, 'net_name': ''}) diff --git a/tensorflow_app/views/layers_import.py b/tensorflow_app/views/layers_import.py new file mode 100644 index 000000000..b284cdf75 --- /dev/null +++ b/tensorflow_app/views/layers_import.py @@ -0,0 +1,473 @@ +import math +import re + + +initializer_map = {'random_uniform': 'RandomUniform', 'random_normal': 'RandomNormal', + 'Const': 'Constant', 'zeros': 'Zeros', 'ones': 'Ones', + 'eye': 'Identity', 'truncated_normal': 'TruncatedNormal'} + +activation_layers = [ + 'Sigmoid', + 'Softplus', + 'Softsign', + 'Elu', + 'LeakyRelu', + 'Softmax', + 'Relu', + 'Tanh', + 'SELU' +] + + +def get_layer_name(node_name): + i = node_name.find('/') + if i == -1: + name = str(node_name) + elif str(node_name[:i]) in ['Repeat', 'Stack']: + name = str(node_name.split('/')[1]) + else: + name = str(node_name[:i]) + return name + + +def get_layer_type(node_name): + return node_name.split('_')[0] + + +def get_padding(node, kernel_shape, strides): + if node.type in ["Conv3D", "MaxPool3D", "AvgPool3D"]: + input_tensor = node.inputs[0] + output_tensor = node.outputs[0] + input_shape = [1 if i.value is None else int(i) for i in input_tensor.shape] + output_shape = [1 if i.value is None else int(i) for i in output_tensor.shape] + + kernel_d = kernel_shape[0] + kernel_h = kernel_shape[1] + kernel_w = kernel_shape[2] + stride_d = strides[1] + stride_h = strides[2] + stride_w = strides[3] + + pad_d = ((int(output_shape[1]) - 1) * stride_d + + kernel_d - int(input_shape[1])) / float(2) + pad_h = ((int(output_shape[2]) - 1) * stride_h + + kernel_h - int(input_shape[2])) / float(2) + pad_w = ((int(output_shape[3]) - 1) * stride_w + + kernel_w - int(input_shape[3])) / float(2) + + if node.type == "Conv3D": + pad_d = math.ceil(pad_d) + pad_h = math.ceil(pad_h) + pad_w = math.ceil(pad_w) + elif node.type in ["MaxPool3D", "AvgPool3D"]: + pad_d = math.floor(pad_d) + pad_h = math.floor(pad_h) + pad_w = math.floor(pad_w) + + return int(pad_d), int(pad_h), int(pad_w) + + elif node.type == "Conv2DBackpropInput": + input_tensor = node.inputs[2] + output_tensor = node.outputs[0] + input_shape = [1 if i.value is None else int(i) for i in input_tensor.shape] + output_shape = [1 if i.value is None else int(i) for i in output_tensor.shape] + + # if deconvolution layer padding calculation logic changes + if ('padding' in node.node_def.attr): + kernel_h = kernel_shape[0] + kernel_w = kernel_shape[1] + stride_h = strides[1] + stride_w = strides[2] + pad_h = ((int(input_shape[1]) - 1) * stride_h + + kernel_h - int(output_shape[1])) / float(2) + pad_w = ((int(input_shape[2]) - 1) * stride_w + + kernel_w - int(output_shape[2])) / float(2) + + return int(math.floor(pad_h)), int(math.floor(pad_w)) + + else: + input_tensor = node.inputs[0] + output_tensor = node.outputs[0] + input_shape = [1 if i.value is None else int(i) for i in input_tensor.shape] + output_shape = [1 if i.value is None else int(i) for i in output_tensor.shape] + kernel_h = kernel_shape[0] + kernel_w = kernel_shape[1] + stride_h = strides[1] + stride_w = strides[2] + + pad_h = ((int(output_shape[1]) - 1) * stride_h + + kernel_h - int(input_shape[1])) / float(2) + pad_w = ((int(output_shape[2]) - 1) * stride_w + + kernel_w - int(input_shape[2])) / float(2) + + # check this logic (see caffe-tensorflow/caffe/shapes.py) + if node.type == "Conv2D": + pad_h = math.ceil(pad_h) + pad_w = math.ceil(pad_w) + elif node.type in ["MaxPool", "AvgPool"]: + pad_h = math.floor(pad_h) + pad_w = math.floor(pad_w) + + return int(pad_h), int(pad_w) + + +def get_initializer_type(layer_ops): + """Returns a dict mapping variables (weight, bias etc) to initializer types. + The returned dict maybe empty if no initializers are found. + """ + weight_name_patterns = [r'.*/weight/*', r'.*/kernel/*'] + bias_name_patterns = [r'.*/bias/*'] + pointwise_weight_name_patterns = [r'.*/pointwise_weights/*'] + depthwise_weight_name_patterns = [r'.*/depthwise_weights/*'] + + initializers = {} + for op in layer_ops: + # extracting weights initializer + for weight_name_pattern in weight_name_patterns: + if re.match(weight_name_pattern, str(op.name)) and op.type in initializer_map.keys(): + initializers['weight'] = initializer_map[op.type] + # extracting bias initializer + for bias_name_pattern in bias_name_patterns: + if re.match(bias_name_pattern, str(op.name)) and op.type in initializer_map.keys(): + initializers['bias'] = initializer_map[op.type] + # extracting pointwise wei + for pointwise_weight_name_pattern in pointwise_weight_name_patterns: + if re.match(pointwise_weight_name_pattern, str(op.name)) and op.type in initializer_map.keys(): + initializers['pointwise_weight'] = initializer_map[op.type] + for depthwise_weight_name_pattern in depthwise_weight_name_patterns: + if re.match(depthwise_weight_name_pattern, str(op.name)) and op.type in initializer_map.keys(): + initializers['depthwise_weight'] = initializer_map[op.type] + + return initializers + + +def get_input_layers(layer_ops): + ''' + return the name of the layers directly preceeding the layer of layer_ops. + layer_ops is a list of all ops of the layer we want the inputs of. + ''' + input_layer_names = [] + name = get_layer_name(layer_ops[0].name) + for node in layer_ops: + for input_tensor in node.inputs: + input_layer_name = get_layer_name(input_tensor.op.name) + if input_layer_name != name: + input_layer_names.append(input_layer_name) + return input_layer_names + + +def import_activation(layer_ops): + layer_type = '' + layer_params = {} + + activation_op = next((x for x in layer_ops if x.type in activation_layers), None) + + if activation_op.type == 'Relu': + layer_type = 'ReLU' + + elif activation_op.type == 'LeakyRelu': + if 'alpha' in activation_op.node_def.attr: + layer_params['negative_slope'] = activation_op.get_attr('alpha') + layer_type = 'ReLU' + + elif activation_op.type == 'Elu': + layer_params['alpha'] = 1 + layer_type = 'ELU' + + elif activation_op.type == 'Tanh': + layer_type = 'TanH' + + else: + # rest of the activations have the same name in TF and Fabrik + layer_type = activation_op.type + + return jsonLayer(layer_type, layer_params, get_input_layers(layer_ops), []) + + +def import_placeholder(layer_ops): + placeholder_op = layer_ops[0] + layer_params = {} + layer_dim = [int(dim.size) for dim in placeholder_op.get_attr('shape').dim] + + # make batch size 1 if it is -1 + if layer_dim[0] == 0: + layer_dim[0] = 1 + + # change tensor format from tensorflow default (NHWC/NDHWC) + # to (NCHW/NCDHW) + temp = layer_dim[1] + layer_dim[1] = layer_dim[-1] + layer_dim[-1] = temp + layer_params['dim'] = str(layer_dim)[1:-1] + + return jsonLayer('Input', layer_params, get_input_layers(layer_ops), []) + + +def import_conv2d(layer_ops): + conv2d_op = next((x for x in layer_ops if x.type == 'Conv2D'), None) + layer_params = {} + layer_params['layer_type'] = '2D' + + strides = [int(i) for i in conv2d_op.get_attr('strides')] + kernel_shape = [int(i) for i in conv2d_op.inputs[1].shape] + layer_params['stride_h'] = strides[1] + layer_params['stride_w'] = strides[2] + layer_params['kernel_h'] = kernel_shape[0] + layer_params['kernel_w'] = kernel_shape[1] + layer_params['num_output'] = kernel_shape[3] + layer_params['pad_h'], layer_params['pad_w'] = get_padding(conv2d_op, kernel_shape, strides) + + initializers = get_initializer_type(layer_ops) + try: + layer_params['weight_filler'] = initializers['kernel'] + layer_params['bias_filler'] = initializers['bias'] + except KeyError: + # no initializers found, continue + pass + + return jsonLayer('Convolution', layer_params, get_input_layers(layer_ops), []) + + +def import_conv3d(layer_ops): + conv3d_op = next((x for x in layer_ops if x.type == 'Conv3D'), None) + layer_params = {} + layer_params['layer_type'] = '3D' + + kernel_shape = [int(i) for i in conv3d_op.inputs[1].shape] + layer_params['kernel_d'] = kernel_shape[0] + layer_params['kernel_h'] = kernel_shape[1] + layer_params['kernel_w'] = kernel_shape[2] + layer_params['num_output'] = kernel_shape[4] + + strides = [int(i) for i in conv3d_op.get_attr('strides')] + layer_params['stride_d'] = strides[1] + layer_params['stride_h'] = strides[2] + layer_params['stride_w'] = strides[3] + + pad_d, pad_h, pad_w = get_padding(conv3d_op, kernel_shape, strides) + layer_params['pad_d'] = pad_d + layer_params['pad_h'] = pad_h + layer_params['pad_w'] = pad_w + + initializers = get_initializer_type(layer_ops) + try: + layer_params['weight_filler'] = initializers['kernel'] + layer_params['bias_filler'] = initializers['bias'] + except KeyError: + # no initializers found, continue + pass + + return jsonLayer('Convolution', layer_params, get_input_layers(layer_ops), []) + + +def import_deconvolution(layer_ops): + deconv_op = next((x for x in layer_ops if x.type == 'Conv2DBackpropInput'), None) + layer_params = {} + layer_params['layer_type'] = '2D' + + kernel_shape = [int(i) for i in deconv_op.inputs[1].shape] + strides = [int(i) for i in deconv_op.get_attr('strides')] + layer_params['padding'] = deconv_op.get_attr('padding') + layer_params['kernel_h'] = kernel_shape[0] + layer_params['kernel_w'] = kernel_shape[1] + layer_params['num_output'] = kernel_shape[3] + layer_params['pad_h'], layer_params['pad_w'] = get_padding(deconv_op, kernel_shape, strides) + + initializers = get_initializer_type(layer_ops) + try: + layer_params['weight_filler'] = initializers['kernel'] + layer_params['bias_filler'] = initializers['bias'] + except KeyError: + # no initializers found, continue + pass + + return jsonLayer('Deconvolution', layer_params, get_input_layers(layer_ops), []) + + +def import_depthwise_convolution(layer_ops): + depthwise_conv_op = next((x for x in layer_ops if x.type == 'DepthwiseConv2dNative'), None) + layer_params = {} + if '3D' in depthwise_conv_op.type: + raise ValueError('3D depthwise convolution cannot be imported.') + + kernel_shape = [int(i) for i in depthwise_conv_op.inputs[1].shape] + layer_params['kernel_h'] = kernel_shape[0] + layer_params['kernel_w'] = kernel_shape[1] + layer_params['num_output'] = kernel_shape[2] + layer_params['depth_multiplier'] = kernel_shape[3] + + if 'padding' in depthwise_conv_op.node_def.attr: + layer_params['padding'] = str(depthwise_conv_op.get_attr('padding')) + strides = [int(i) for i in depthwise_conv_op.get_attr('strides')] + layer_params['stride_h'] = strides[1] + layer_params['stride_w'] = strides[2] + layer_params['pad_h'], layer_params['pad_w'] = get_padding(depthwise_conv_op, kernel_shape, strides) + + initializers = get_initializer_type(layer_ops) + try: + layer_params['pointwise_weight'] = initializers['pointwise_initializer'] + layer_params['depthwise_weight'] = initializers['depthwise_initializer'] + except KeyError: + # no initializers found, continue + pass + + return jsonLayer('DepthwiseConv', layer_params, get_input_layers(layer_ops), []) + + +def import_pooling2d(layer_ops): + pooling2d_op = next((x for x in layer_ops if x.type in ['MaxPool', 'AvgPool'])) + layer_params = {} + layer_params['layer_type'] = '2D' + + # checking type of pooling layer + if pooling2d_op.type == 'MaxPool': + layer_params['pool'] = 'MAX' + elif pooling2d_op.type == 'AvgPool': + layer_params['pool'] = 'AVE' + + kernel_shape = [int(i) for i in pooling2d_op.get_attr('ksize')] + strides = [int(i) for i in pooling2d_op.get_attr('strides')] + layer_params['kernel_h'] = kernel_shape[1] + layer_params['kernel_w'] = kernel_shape[2] + layer_params['stride_h'] = strides[1] + layer_params['stride_w'] = strides[2] + layer_params['padding'] = str(pooling2d_op.get_attr('padding')) + layer_params['pad_h'], layer_params['pad_w'] = get_padding(pooling2d_op, kernel_shape, strides) + + return jsonLayer('Pooling', layer_params, get_input_layers(layer_ops), []) + + +def import_pooling3d(layer_ops): + pooling3d_op = next((x for x in layer_ops if x.type in ['MaxPool3D', 'AvgPool3D'])) + layer_params = {} + layer_params['layer_type'] = '3D' + layer_params['padding'] = str(pooling3d_op.get_attr('padding')) + + # checking type of pooling layer + if pooling3d_op.type == 'MaxPool': + layer_params['pool'] = 'MAX' + elif pooling3d_op.type == 'AvgPool': + layer_params['pool'] = 'AVE' + + kernel_shape = [int(i) for i in pooling3d_op.get_attr('ksize')] + strides = [int(i) for i in pooling3d_op.get_attr('strides')] + layer_params['kernel_d'] = kernel_shape[1] + layer_params['kernel_h'] = kernel_shape[2] + layer_params['kernel_w'] = kernel_shape[3] + layer_params['stride_d'] = strides[1] + layer_params['stride_h'] = strides[2] + layer_params['stride_w'] = strides[3] + + pad_d, pad_h, pad_w = get_padding(pooling3d_op, kernel_shape, strides) + layer_params['pad_d'] = pad_d + layer_params['pad_h'] = pad_h + layer_params['pad_w'] = pad_w + + return jsonLayer('Pooling', layer_params, get_input_layers(layer_ops), []) + + +def import_inner_product(layer_ops): + inner_product_op = next((x for x in layer_ops if x.type in ['Prod', 'MatMul'])) + layer_params = {} + if inner_product_op.type == 'MatMul': + layer_params['num_output'] = int(inner_product_op.inputs[1].shape[1]) + + return jsonLayer('InnerProduct', layer_params, get_input_layers(layer_ops), []) + + +def import_batchnorm(layer_ops): + layer_params = {} + name = get_layer_name(layer_ops[0].name) + + for node in layer_ops: + if re.match('.*\/batchnorm[_]?[0-9]?\/add.*', str(node.name)): + try: + layer_params['eps'] = node.get_attr('value').float_val[0] + except: + pass + if (node.type == 'FusedBatchNorm'): + layer_params['eps'] = float(node.get_attr('epsilon')) + # searching for moving_mean/Initializer ops to extract moving + # mean initializer of batchnorm layer + if name + '/moving_mean/Initializer' in str(node.name): + layer_params['moving_mean_initializer'] = \ + initializer_map[str(node.name).split('/')[3]] + # searching for AssignMovingAvg/decay ops to extract moving + # average fraction of batchnorm layer also considering repeat & stack layer + # as prefixes + if str(node.name) in [name + '/AssignMovingAvg/decay', + 'Repeat/' + name + '/AssignMovingAvg/decay', + 'Stack/' + name + '/AssignMovingAvg/decay']: + layer_params['moving_average_fraction'] = node.get_attr( + 'value').float_val[0] + + return jsonLayer('BatchNorm', layer_params, get_input_layers(layer_ops), []) + + +def import_eltwise(layer_ops): + eltwise_op = next((x for x in layer_ops if x.type in ['add', 'mul', 'dot'])) + layer_params = {} + if eltwise_op.type == 'add': + layer_params['layer_type'] = 'Sum' + if eltwise_op.type == 'mul': + layer_params['layer_type'] = 'Product' + if eltwise_op.type == 'dot': + layer_params['layer_type'] = 'Dot' + + return jsonLayer('Eltwise', layer_params, get_input_layers(layer_ops), []) + + +def import_dropout(layer_ops): + layer_params = {} + for node in layer_ops: + if ('rate' in node.node_def.attr): + layer_params['rate'] = node.get_attr('rate') + if ('seed' in node.node_def.attr): + layer_params['seed'] = node.get_attr('seed') + if ('training' in node.node_def.attr): + layer_params['trainable'] = node.get_attr('training') + + return jsonLayer('Dropout', layer_params, get_input_layers(layer_ops), []) + + +def import_flatten(layer_ops): + return jsonLayer('Flatten', [], get_input_layers(layer_ops), []) + + +def import_concat(layer_ops): + layer_params = {} + for node in layer_ops: + if 'axis' in node.node_def.attr: + layer_params['axis'] = node.get_attr('axis') + + return jsonLayer('Concat', layer_params, get_input_layers(layer_ops), []) + + +def import_lrn(layer_ops): + layer_params = {} + for node in layer_ops: + if ('alpha' in node.node_def.attr): + layer_params['alpha'] = node.get_attr('alpha') + if ('beta' in node.node_def.attr): + layer_params['beta'] = node.get_attr('beta') + if ('local_size' in node.node_def.attr): + layer_params['local_size'] = node.get_attr('depth_radius') + if ('bias' in node.node_def.attr): + layer_params['k'] = node.get_attr('bias') + + return jsonLayer('LRN', layer_params, get_input_layers(layer_ops), []) + + +def jsonLayer(layer_type, layer_params={}, inputs=[], outputs=[]): + layer = { + 'info': { + 'type': layer_type, + 'phase': None + }, + 'connection': { + 'input': inputs, + 'output': outputs + }, + 'params': layer_params + } + return layer diff --git a/tutorials/adding_new_layers_tensorflow.md b/tutorials/adding_new_layers_tensorflow.md index 1520b5258..c032f174f 100644 --- a/tutorials/adding_new_layers_tensorflow.md +++ b/tutorials/adding_new_layers_tensorflow.md @@ -5,34 +5,48 @@ Follow the guide [here](adding_new_layers.md) to add your new layer to Fabrik's #### Importing a layer +- Open [layers_import.py](https://github.com/Cloud-CV/Fabrik/blob/master/tensorflow_app/views/layers_import.py). + + - Add a function to import your layer. + - Create a function name import_ that takes one parameter, layer_ops, that is a list of all ops in the layer being imported. + - Get layer parameters from the operations in layer_ops and build a dictionary mapping parameter names to values. + - Get a list of input layers to the layer being processed using `get_input_layers(layer_ops)`. + - Create and return a json layer for new layer, calling `jsonLayer` with the new layer_type, parameters, input layers and output_layers(optional). + + ``` + def import_dropout(layer_ops): + layer_params = {} + for node in layer_ops: + if ('rate' in node.node_def.attr): + layer_params['rate'] = node.get_attr('rate') + if ('seed' in node.node_def.attr): + layer_params['seed'] = node.get_attr('seed') + if ('training' in node.node_def.attr): + layer_params['trainable'] = node.get_attr('training') + return jsonLayer('Dropout', layer_params, get_input_layers(layer_ops), []) + ``` + - Open [import_graphdef.py](https://github.com/Cloud-CV/Fabrik/blob/master/tensorflow_app/views/import_graphdef.py) - - Add your layer to one of these four dictionaries that map Tensorflow ops to Caffe layers: - - `op_layer_map` : if the op has can be mapped to a Caffe layer directly - - `activation_map` : if the op is a simple activation - - `name_map` : if the op type can be inferred from the name of the op - - `initializer_map` : if the op is an initializer + - From `layers_import`, import the layer import function you just defined. + + `from layers_import import_dropout` + - Add your layer import function to either `layer_map` or `name_map`: + - `layer_map` if the layer type can be determined directly by the type of nodes in the layer's ops. + ` 'name_map` if the laye type can only be determined from the name of the layer. + ```diff - name_map = {'flatten': 'Flatten', - 'lrn': 'LRN', - ... - + 'dropout': 'Dropout' + name_map = { + 'flatten': import_flatten, + 'lrn': import_lrn, + ... + + 'dropout': import_dropout, + ... + 'concatenate': import_concat } ``` - - Inside the loop `for node in graph.get_operations()`, write code to get any layer parameters needed and build the layer. - - ``` - elif layer['type'][0] == 'Dropout': - if ('rate' in node.node_def.attr): - layer['params']['rate'] = node.get_attr('rate') - if ('seed' in node.node_def.attr): - layer['params']['seed'] = node.get_attr('seed') - if ('training' in node.node_def.attr): - layer['params']['trainable'] = node.get_attr('training') - ``` - #### Exporting a layer Fabrik exports Tensorflow models using Keras. [See this guide](adding_new_layers_keras.md) for exporting a layer for Keras.