Skip to content

Enable QDQ-adapted _QuantizedFusedBatchNorm+LeakyRelu fusion #160

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ChendaLi-Intel merged 2 commits into from
Nov 25, 2022
Merged

Enable QDQ-adapted _QuantizedFusedBatchNorm+LeakyRelu fusion #160

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
78b122e
Enable QDQ-adapted _QuantizedFusedBatchNorm+LeakyRelu fusion
ChendaLi-Intel Nov 24, 2022
24dbaf1
Update spr-base binary for test
chensuyue Nov 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .azure-pipelines/scripts/ut/env_setup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ echo "mxnet version is $mxnet_version"
if [[ "${tensorflow_version}" == *"-official" ]]; then
pip install tensorflow==${tensorflow_version%-official}
elif [[ "${tensorflow_version}" == "spr-base" ]]; then
pip install /tf_dataset/tf_binary/tensorflow*.whl
pip install /tf_dataset/tf_binary/221125/tensorflow*.whl
if [[ $? -ne 0 ]]; then
exit 1
fi
Expand Down
1 change: 1 addition & 0 deletions neural_compressor/adaptor/tensorflow.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,7 @@
'Dequantize + DepthwiseConv2dNative + Add + Relu6 + QuantizeV2',
'Dequantize + DepthwiseConv2dNative + BiasAdd + QuantizeV2',
'Dequantize + FusedBatchNormV3 + Relu + QuantizeV2',
'Dequantize + FusedBatchNormV3 + LeakyRelu + QuantizeV2',
'Dequantize + _MklFusedInstanceNorm + Relu + QuantizeV2',
'Dequantize + _MklFusedInstanceNorm + LeakyRelu + QuantizeV2',
'Dequantize + Conv2DBackpropInput + BiasAdd + QuantizeV2',
Expand Down
108 changes: 105 additions & 3 deletions neural_compressor/adaptor/tf_utils/quantize_graph/qdq/fuse_qdq_bn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,9 @@ def __init__(self, **kwargs):
reverse=True)
if self.new_api:
self.fusion_mapping = {
'FusedBatchNormV3': self.apply_newly_bn_relu_fusion,
'FusedBatchNormV3Relu': self.apply_newly_bn_relu_fusion,
'FusedBatchNormV3': self.apply_newly_bn_relu_fusion
'FusedBatchNormV3LeakyRelu': self.apply_newly_bn_leakyrelu_fusion
}
else:
self.fusion_mapping = {}
Expand Down Expand Up @@ -75,8 +76,7 @@ def apply_newly_bn_relu_fusion(self, match_node_name):
[output_min_node_name] + [output_max_node_name] + control_inputs
output_min_node = helper.create_constant_node(output_min_node_name, -1., dtypes.float32)
output_max_node = helper.create_constant_node(output_max_node_name, 1., dtypes.float32)
quantized_bn_node = helper.create_node(node_op, quantized_node_name,
quantized_node_input_names)
quantized_bn_node = helper.create_node(node_op, quantized_node_name, quantized_node_input_names)
if relu_node_name is not None:
helper.set_attr_string(quantized_bn_node, "activation_mode", b'Relu')
if self.node_name_mapping[offset_name].node.op == "Const":
Expand Down Expand Up @@ -141,6 +141,108 @@ def apply_newly_bn_relu_fusion(self, match_node_name):
new_node.CopyFrom(node)
self.add_output_graph_node(new_node)

def apply_newly_bn_leakyrelu_fusion(self, match_node_name):
matched_node = self.node_name_mapping[match_node_name[0]]
skip_node_name = match_node_name[1:]
control_inputs, normal_inputs = self._get_node_input(
matched_node.node.name)
scale_name = normal_inputs[1]
offset_name = normal_inputs[2]
mean_name = normal_inputs[3]
variance_name = normal_inputs[4]

all_input_names = self._add_eightbit_prologue_nodes(matched_node.node.name)
all_input_names = [
all_input_names[0],
scale_name,
offset_name,
mean_name,
variance_name,
all_input_names[1],
all_input_names[2]
]

for _, node in enumerate(self.input_graph.node):
if node.name in skip_node_name:
self.logger.debug("skip node {}".format(node.name))
elif node.name == match_node_name[0]:
self.logger.debug("Matched node {} with input {}.".format(node.name, node.input))
leakyrelu_node_name = match_node_name[1]
node_op = '_QuantizedFusedBatchNorm'
quantized_node_name = node.name + "_eightbit_quantized_bn"
output_min_node_name = quantized_node_name + "_input7_output_min"
output_max_node_name = quantized_node_name + "_input8_output_max"
quantized_node_input_names = all_input_names + \
[output_min_node_name] + [output_max_node_name] + control_inputs
output_min_node = helper.create_constant_node(output_min_node_name, -1., dtypes.float32)
output_max_node = helper.create_constant_node(output_max_node_name, 1., dtypes.float32)
quantized_bn_node = helper.create_node(node_op, quantized_node_name, quantized_node_input_names)

helper.set_attr_string(quantized_bn_node, "activation_mode", b'LeakyRelu')
helper.copy_attr(quantized_bn_node, "alpha", \
self.node_name_mapping[leakyrelu_node_name].node.attr["alpha"])
if self.node_name_mapping[offset_name].node.op == "Const":
helper.set_attr_bool(quantized_bn_node, "is_offset_const", True)
else:
helper.set_attr_bool(quantized_bn_node, "is_offset_const", False)
if self.node_name_mapping[mean_name].node.op == "Const":
helper.set_attr_bool(quantized_bn_node, "is_mean_const", True)
else:
helper.set_attr_bool(quantized_bn_node, "is_mean_const", False)
helper.set_attr_dtype(quantized_bn_node, "T", dtypes.qint8)
helper.set_attr_dtype(quantized_bn_node, "U", dtypes.float32)
helper.set_attr_dtype(quantized_bn_node, "Tout", dtypes.qint8)

"""
# 0. x
# 1. scale
# 2. offset
# 3. mean
# 4. variance
# 5. x_min
# 6. x_max
# 7. {output_min}
# 8. {output_max}
"""
helper.set_attr_type_list(quantized_bn_node, 'input_types', [
dtypes.qint8.as_datatype_enum,
dtypes.float32.as_datatype_enum,
dtypes.float32.as_datatype_enum,
dtypes.float32.as_datatype_enum,
dtypes.float32.as_datatype_enum,
dtypes.float32.as_datatype_enum,
dtypes.float32.as_datatype_enum,
dtypes.float32.as_datatype_enum,
dtypes.float32.as_datatype_enum,
])


"""
# 0. output
# 1. output_min
# 2. output_max
"""
helper.set_attr_type_list(quantized_bn_node, 'out_types', [
dtypes.qint8.as_datatype_enum,
dtypes.float32.as_datatype_enum,
dtypes.float32.as_datatype_enum,
])
self.add_output_graph_node(output_min_node)
self.add_output_graph_node(output_max_node)
self.add_output_graph_node(quantized_bn_node)
self._intel_cpu_add_dequantize_result_node(
quantized_output_name = quantized_node_name,
original_node_name = match_node_name[-1],
dtype = dtypes.qint8,
min_tensor_index = 1,
performance_only=self.performance_only
)

else:
new_node = node_def_pb2.NodeDef()
new_node.CopyFrom(node)
self.add_output_graph_node(new_node)

def get_longest_fuse(self):
self._get_op_list()
real_patterns = [pattern[1 :-1] for pattern in self.sorted_patterns]
Expand Down
108 changes: 105 additions & 3 deletions neural_compressor/adaptor/tf_utils/quantize_graph/quantize_graph_bn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,9 @@ def __init__(self, **kwargs):
reverse=True)
if self.new_api:
self.fusion_mapping = {
'FusedBatchNormV3': self.apply_newly_bn_relu_fusion,
'FusedBatchNormV3Relu': self.apply_newly_bn_relu_fusion,
'FusedBatchNormV3': self.apply_newly_bn_relu_fusion
'FusedBatchNormV3LeakyRelu': self.apply_newly_bn_leakyrelu_fusion
}
else:
self.fusion_mapping = {}
Expand Down Expand Up @@ -75,8 +76,7 @@ def apply_newly_bn_relu_fusion(self, match_node_name):
[output_min_node_name] + [output_max_node_name] + control_inputs
output_min_node = helper.create_constant_node(output_min_node_name, -1., dtypes.float32)
output_max_node = helper.create_constant_node(output_max_node_name, 1., dtypes.float32)
quantized_bn_node = helper.create_node(node_op, quantized_node_name,
quantized_node_input_names)
quantized_bn_node = helper.create_node(node_op, quantized_node_name, quantized_node_input_names)
if relu_node_name is not None:
helper.set_attr_string(quantized_bn_node, "activation_mode", b'Relu')
if self.node_name_mapping[offset_name].node.op == "Const":
Expand Down Expand Up @@ -140,6 +140,108 @@ def apply_newly_bn_relu_fusion(self, match_node_name):
new_node.CopyFrom(node)
self.add_output_graph_node(new_node)

def apply_newly_bn_leakyrelu_fusion(self, match_node_name):
matched_node = self.node_name_mapping[match_node_name[0]]
skip_node_name = match_node_name[1:]
control_inputs, normal_inputs = self._get_node_input(
matched_node.node.name)
scale_name = normal_inputs[1]
offset_name = normal_inputs[2]
mean_name = normal_inputs[3]
variance_name = normal_inputs[4]

all_input_names = self._add_eightbit_prologue_nodes(matched_node.node.name)
all_input_names = [
all_input_names[0],
scale_name,
offset_name,
mean_name,
variance_name,
all_input_names[1],
all_input_names[2]
]

for _, node in enumerate(self.input_graph.node):
if node.name in skip_node_name:
self.logger.debug("skip node {}".format(node.name))
elif node.name == match_node_name[0]:
self.logger.debug("Matched node {} with input {}.".format(node.name, node.input))
leakyrelu_node_name = match_node_name[1]
node_op = '_QuantizedFusedBatchNorm'
quantized_node_name = node.name + "_eightbit_quantized_bn"
output_min_node_name = quantized_node_name + "_input7_output_min"
output_max_node_name = quantized_node_name + "_input8_output_max"
quantized_node_input_names = all_input_names + \
[output_min_node_name] + [output_max_node_name] + control_inputs
output_min_node = helper.create_constant_node(output_min_node_name, -1., dtypes.float32)
output_max_node = helper.create_constant_node(output_max_node_name, 1., dtypes.float32)
quantized_bn_node = helper.create_node(node_op, quantized_node_name, quantized_node_input_names)

helper.set_attr_string(quantized_bn_node, "activation_mode", b'LeakyRelu')
helper.copy_attr(quantized_bn_node, "alpha", \
self.node_name_mapping[leakyrelu_node_name].node.attr["alpha"])
if self.node_name_mapping[offset_name].node.op == "Const":
helper.set_attr_bool(quantized_bn_node, "is_offset_const", True)
else:
helper.set_attr_bool(quantized_bn_node, "is_offset_const", False)
if self.node_name_mapping[mean_name].node.op == "Const":
helper.set_attr_bool(quantized_bn_node, "is_mean_const", True)
else:
helper.set_attr_bool(quantized_bn_node, "is_mean_const", False)
helper.set_attr_dtype(quantized_bn_node, "T", dtypes.qint8)
helper.set_attr_dtype(quantized_bn_node, "U", dtypes.float32)
helper.set_attr_dtype(quantized_bn_node, "Tout", dtypes.qint8)

"""
# 0. x
# 1. scale
# 2. offset
# 3. mean
# 4. variance
# 5. x_min
# 6. x_max
# 7. {output_min}
# 8. {output_max}
"""
helper.set_attr_type_list(quantized_bn_node, 'input_types', [
dtypes.qint8.as_datatype_enum,
dtypes.float32.as_datatype_enum,
dtypes.float32.as_datatype_enum,
dtypes.float32.as_datatype_enum,
dtypes.float32.as_datatype_enum,
dtypes.float32.as_datatype_enum,
dtypes.float32.as_datatype_enum,
dtypes.float32.as_datatype_enum,
dtypes.float32.as_datatype_enum,
])


"""
# 0. output
# 1. output_min
# 2. output_max
"""
helper.set_attr_type_list(quantized_bn_node, 'out_types', [
dtypes.qint8.as_datatype_enum,
dtypes.float32.as_datatype_enum,
dtypes.float32.as_datatype_enum,
])
self.add_output_graph_node(output_min_node)
self.add_output_graph_node(output_max_node)
self.add_output_graph_node(quantized_bn_node)
self._intel_cpu_add_dequantize_result_node(
quantized_output_name = quantized_node_name,
original_node_name = match_node_name[-1],
dtype = dtypes.qint8,
min_tensor_index = 1,
performance_only=self.performance_only
)

else:
new_node = node_def_pb2.NodeDef()
new_node.CopyFrom(node)
self.add_output_graph_node(new_node)

def get_longest_fuse(self):
self._get_op_list()
matched_rule, matched_node_name = self._is_match(self.sorted_patterns)
Expand Down
Loading