From d928ef494650737a53accf70ab88b37515011350 Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Thu, 4 Apr 2019 10:28:37 +0800 Subject: [PATCH 01/23] add quantized sum --- .../mkldnn/mkldnn_quantized_sum.cc | 215 ++++++++++++++++++ src/operator/quantization/quantized_sum.cc | 130 +++++++++++ ...erty.h => mkldnn_post_quantize_property.h} | 63 +++-- .../mkldnn/mkldnn_subgraph_property.cc | 4 +- .../tensor/elemwise_binary_op_basic.cc | 3 + .../python/quantization/test_quantization.py | 67 ++++++ 6 files changed, 456 insertions(+), 26 deletions(-) create mode 100644 src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc create mode 100644 src/operator/quantization/quantized_sum.cc rename src/operator/subgraph/mkldnn/{mkldnn_conv_post_quantize_property.h => mkldnn_post_quantize_property.h} (68%) diff --git a/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc b/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc new file mode 100644 index 000000000000..4df40163c618 --- /dev/null +++ b/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2019 by Contributors + * \file mkldnn_quantized_sum.cc + * \brief + */ + +#if MXNET_USE_MKLDNN == 1 +#include "./mkldnn_ops-inl.h" +#include "./mkldnn_base-inl.h" +#include "../quantization_utils.h" + +namespace mxnet { +namespace op { + +namespace quantized_sum_enum { +enum QuantizedSumOutputs { kOut, kMin, kMax }; +enum QuantizedSumInputs { kDataA, kDataB, kAMin, kAMax, kBMin, kBMax}; +} + +struct RequantizeSumParam : public dmlc::Parameter { + dmlc::optional min_calib_range; // min float value calculated from calibration dataset + dmlc::optional max_calib_range; // max float value calculated from calibration dataset + DMLC_DECLARE_PARAMETER(RequantizeSumParam) { + DMLC_DECLARE_FIELD(min_calib_range) + .set_default(dmlc::optional()) + .describe("The minimum scalar value in the form of float32 obtained " + "through calibration. If present, it will be used to requantize the " + "int8 output data."); + DMLC_DECLARE_FIELD(max_calib_range) + .set_default(dmlc::optional()) + .describe("The maximum scalar value in the form of float32 obtained " + "through calibration. If present, it will be used to requantize the " + "int8 output data."); + } +}; + +DMLC_REGISTER_PARAMETER(RequantizeSumParam); + +static float GetScale(const NDArray& data, float min, float max) { + auto data_range = (data.dtype() == mshadow::kInt8) ? kInt8Range : kUint8Range; + return data_range / MaxAbs(min, max); +} + +static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpContext& ctx, + const std::vector& in_data, + const std::vector& req, + const std::vector& out_data) { + const RequantizeSumParam& params = nnvm::get(attrs.parsed); + // A, B, A_min, A_max, B_min, B_max + CHECK_EQ(in_data.size(), static_cast(6)); + // C, C_min, C_max + CHECK_EQ(out_data.size(), 3U); + + // Collect data min,max,absmax + float dataA_min = in_data[quantized_sum_enum::kAMin].data().dptr()[0]; + float dataB_min = in_data[quantized_sum_enum::kBMin].data().dptr()[0]; + float dataA_max = in_data[quantized_sum_enum::kAMax].data().dptr()[0]; + float dataB_max = in_data[quantized_sum_enum::kBMax].data().dptr()[0]; + float dataA_absmax = MaxAbs(dataA_min, dataA_max); + float dataB_absmax = MaxAbs(dataB_min, dataB_max); + + auto dataA_mem = in_data[quantized_sum_enum::kDataA].GetMKLDNNData(); + auto dataB_mem = in_data[quantized_sum_enum::kDataB].GetMKLDNNData(); + bool dataA_int8 = (in_data[quantized_sum_enum::kDataA].dtype() == mshadow::kInt8) ? true : false; + + float A_scale = GetScale(in_data[quantized_sum_enum::kDataA], dataA_min, dataA_max); + float B_scale = GetScale(in_data[quantized_sum_enum::kDataB], dataB_min, dataB_max); + + // rescaled_mem is for reorder mkldnn memory + std::shared_ptr rescaled_mem; + size_t output_data_range = kInt8Range; + auto output_data_type = mkldnn::memory::s8; + // dataA && dataB are uint8 + if (in_data[quantized_sum_enum::kDataA].dtype() == in_data[quantized_sum_enum::kDataB].dtype() + && dataA_int8 == false) { + output_data_range = kUint8Range; + output_data_type = mkldnn::memory::u8; + } + float output_min = 0; + float output_max = 0; + float out_data_scale = 0; + if (params.max_calib_range.has_value() && params.min_calib_range.has_value()) { + output_min = params.min_calib_range.value(); + output_max = params.max_calib_range.value(); + out_data_scale = output_data_range/MaxAbs(output_min, output_max); + } else { + output_max = dataA_absmax + dataB_absmax; + output_min = 0 - output_max; + } + std::vector scales; + if (in_data[quantized_sum_enum::kDataA].dtype() != in_data[quantized_sum_enum::kDataB].dtype()) { + auto s8_pd = (dataA_int8 == true) + ? dataA_mem->get_primitive_desc() + : dataB_mem->get_primitive_desc(); + rescaled_mem = std::make_shared(s8_pd); + float u8_reorder_scale = 0; + if (params.max_calib_range.has_value() && params.min_calib_range.has_value()) { + if (dataA_int8 == true) { + u8_reorder_scale = out_data_scale/B_scale; + scales.push_back(out_data_scale/A_scale); + scales.push_back(1); + } else { + u8_reorder_scale = out_data_scale/A_scale; + scales.push_back(1); + scales.push_back(out_data_scale/B_scale); + } + } else { + // x*dataA_absmax/dataA_range = y*(dataA_absmax+dataB_absmax)/output_range + if (dataA_int8 == true) { + u8_reorder_scale = kInt8Range*dataB_absmax/(kUint8Range*(dataA_absmax + dataB_absmax)); + scales.push_back(dataA_absmax/(dataA_absmax + dataB_absmax)); + scales.push_back(1); + } else { + u8_reorder_scale = kInt8Range*dataA_absmax/(kUint8Range*(dataA_absmax + dataB_absmax)); + scales.push_back(1); + scales.push_back(dataB_absmax/(dataA_absmax + dataB_absmax)); + } + } + std::vector reorder_scale = {u8_reorder_scale}; + primitive_attr reorder_attr; + reorder_attr.set_int_output_round_mode(round_mode::round_nearest); + reorder_attr.set_output_scales(0, reorder_scale); + auto u8_mem = (dataA_int8 == true) ? dataB_mem : dataA_mem; + const auto reorder_pd = mkldnn::reorder::primitive_desc(u8_mem->get_primitive_desc(), + s8_pd, + reorder_attr); + MKLDNNStream::Get()->RegisterPrim(mkldnn::reorder(reorder_pd, *u8_mem, *rescaled_mem)); + + if (dataA_int8 == true) { + dataB_mem = rescaled_mem.get(); + } else { + dataA_mem = rescaled_mem.get(); + } + } else { + // same data type + if (params.max_calib_range.has_value() && params.min_calib_range.has_value()) { + scales.push_back(out_data_scale/A_scale); + scales.push_back(out_data_scale/B_scale); + } else { + scales.push_back(dataA_absmax/(dataA_absmax + dataB_absmax)); + scales.push_back(1 - scales[0]); + } + } + + std::vector in_prims; + std::vector in_pds; + in_prims.push_back(*dataA_mem); + in_prims.push_back(*dataB_mem); + in_pds.push_back(dataA_mem->get_primitive_desc()); + in_pds.push_back(dataB_mem->get_primitive_desc()); + size_t i_ndim = in_data[quantized_sum_enum::kDataA].shape().ndim(); + mkldnn::memory::dims i_dims = mkldnn::memory::dims(i_ndim); + for (size_t i = 0; i < i_ndim; i++) { + i_dims[i] = static_cast(in_data[quantized_sum_enum::kDataA].shape()[i]); + } + mkldnn::memory::format i_fmt = static_cast( + in_pds[quantized_sum_enum::kDataA].desc().data.format); + auto output_desc = memory::desc(i_dims, output_data_type, i_fmt); + mkldnn::sum::primitive_desc pdesc(output_desc, scales, in_pds); + auto mem = CreateMKLDNNMem(out_data[quantized_sum_enum::kOut], + pdesc.dst_primitive_desc(), + req[0], + &in_data[0]); + MKLDNNStream *stream = MKLDNNStream::Get(); + stream->RegisterPrim(mkldnn::sum(pdesc, in_prims, *mem.second)); + CommitOutput(out_data[quantized_sum_enum::kOut], mem); + stream->Submit(); + + out_data[quantized_sum_enum::kMin].data().dptr()[0] = output_min; + out_data[quantized_sum_enum::kMax].data().dptr()[0] = output_max; +} + +inline static bool SumStorageType(const nnvm::NodeAttrs& attrs, const int dev_mask, + DispatchMode* dispatch_mode, std::vector* in_attrs, + std::vector* out_attrs) { + // A, B, A_min, A_max, B_min, B_max + CHECK_EQ(in_attrs->size(), 6U); + // C, C_min, C_max + CHECK_EQ(out_attrs->size(), 3U); + + return MKLDNNStorageType(attrs, dev_mask, true, dispatch_mode, in_attrs, out_attrs); +} + +NNVM_REGISTER_OP(_contrib_quantized_sum) +.set_attr("FInferStorageType", SumStorageType) +.set_attr("FComputeEx", MKLDNNQuantizedSumForward) +.set_attr("FResourceRequest", [](const NodeAttrs& n) { + return std::vector{ResourceRequest::kTempSpace}; +}) +.set_attr("TIsMKLDNN", true) +.set_attr_parser(ParamParser) +.add_arguments(RequantizeSumParam::__FIELDS__()); +} // namespace op +} // namespace mxnet + +#endif // MXNET_USE_MKLDNN == 1 diff --git a/src/operator/quantization/quantized_sum.cc b/src/operator/quantization/quantized_sum.cc new file mode 100644 index 000000000000..5af8d28a94a8 --- /dev/null +++ b/src/operator/quantization/quantized_sum.cc @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2019 by Contributors + * \file quantized_sum.cc + * \brief +*/ +#include "../tensor/elemwise_unary_op.h" + +namespace mxnet { +namespace op { + +static bool SumShape(const nnvm::NodeAttrs& attrs, mxnet::ShapeVector* in_shape, + mxnet::ShapeVector* out_shape) { + // A, B, A_min, A_max, B_min, B_max + CHECK_EQ(in_shape->size(), 6U); + // C, C_min, C_max + CHECK_EQ(out_shape->size(), 3U); + CHECK_EQ((*in_shape)[0], (*in_shape)[1]); + + + SHAPE_ASSIGN_CHECK(*in_shape, 2, TShape{1}); + SHAPE_ASSIGN_CHECK(*in_shape, 3, TShape{1}); + SHAPE_ASSIGN_CHECK(*in_shape, 4, TShape{1}); + SHAPE_ASSIGN_CHECK(*in_shape, 5, TShape{1}); + + SHAPE_ASSIGN_CHECK(*out_shape, 0, (*in_shape)[0]); + SHAPE_ASSIGN_CHECK(*out_shape, 1, TShape{1}); + SHAPE_ASSIGN_CHECK(*out_shape, 2, TShape{1}); + return true; +} + +static bool SumType(const nnvm::NodeAttrs& attrs, std::vector* in_type, + std::vector* out_type) { + // A, B, A_min, A_max, B_min, B_max + CHECK_EQ(in_type->size(), 6U); + // C, C_min, C_max + CHECK_EQ(out_type->size(), 3U); + + // A, B + const int elem_add_num = 2; + for (int i = 0; i < elem_add_num; ++i) { + if (in_type->at(i) == mshadow::kInt8) { + TYPE_ASSIGN_CHECK(*in_type, i, mshadow::kInt8); + } else { + TYPE_ASSIGN_CHECK(*in_type, i, mshadow::kUint8); + } + } + // C + int dtype = (in_type->at(0) == in_type->at(1)) ? in_type->at(0) : mshadow::kInt8; + TYPE_ASSIGN_CHECK(*out_type, 0, dtype); + // C_min + TYPE_ASSIGN_CHECK(*out_type, 1, mshadow::kFloat32); + // C_max + TYPE_ASSIGN_CHECK(*out_type, 2, mshadow::kFloat32); + + return true; +} + +NNVM_REGISTER_OP(_contrib_quantized_sum) +.describe(R"code(Adds arguments element-wise. + +The storage type of ``elemwise_add`` output depends on storage types of inputs + + - elemwise_add(row_sparse, row_sparse) = row_sparse + - elemwise_add(csr, csr) = csr + - elemwise_add(default, csr) = default + - elemwise_add(csr, default) = default + - elemwise_add(default, rsp) = default + - elemwise_add(rsp, default) = default + - otherwise, ``elemwise_add`` generates output with default storage + +)code") +.set_num_inputs([](const NodeAttrs& attrs) { +// A, B, A_min, A_max, B_min, B_max + return 6; +}) +// C, C_min, C_max +.set_num_outputs(3) +.set_attr("FListInputNames", [](const NodeAttrs& attrs) { + return std::vector{"lhs", "rhs", "lhs_min", "lhs_max", "rhs_min", "rhs_max"}; \ +}) +.set_attr("FListOutputNames", [](const NodeAttrs& attrs) { + return std::vector{"output", "min_output", "max_output"}; +}) +// TODO(Xinyu): a temp solution to enable GluonCV INT8 flow, +// will be reverted after the improvement of CachedOP is done. +.set_attr("FGradient", MakeZeroGradNodes) +.set_attr("FInferType", SumType) +.set_attr("FInferShape", SumShape) +.set_attr("FNeedRequantize", [](const NodeAttrs& attrs) { return true; }) +.add_argument("lhs", "NDArray-or-Symbol", "first input") +.add_argument("rhs", "NDArray-or-Symbol", "4th input") +.add_argument("lhs_min", "NDArray-or-Symbol", "second input") +.add_argument("lhs_max", "NDArray-or-Symbol", "third input") +.add_argument("rhs_min", "NDArray-or-Symbol", "5th input") +.add_argument("rhs_max", "NDArray-or-Symbol", "6th input"); + + +NNVM_REGISTER_OP(elemwise_add) +.set_attr("FQuantizedOp", [](const NodeAttrs& attrs) { + nnvm::NodePtr node = nnvm::Node::Create(); + node->attrs.op = Op::Get("_contrib_quantized_sum"); + node->attrs.name = "quantized_" + attrs.name; + node->attrs.dict = attrs.dict; + if (node->op()->attr_parser != nullptr) { + node->op()->attr_parser(&(node->attrs)); + } + return node; +}); + +} // namespace op +} // namespace mxnet diff --git a/src/operator/subgraph/mkldnn/mkldnn_conv_post_quantize_property.h b/src/operator/subgraph/mkldnn/mkldnn_post_quantize_property.h similarity index 68% rename from src/operator/subgraph/mkldnn/mkldnn_conv_post_quantize_property.h rename to src/operator/subgraph/mkldnn/mkldnn_post_quantize_property.h index f9033f48d413..4aede396661c 100644 --- a/src/operator/subgraph/mkldnn/mkldnn_conv_post_quantize_property.h +++ b/src/operator/subgraph/mkldnn/mkldnn_post_quantize_property.h @@ -16,12 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -#ifndef MXNET_OPERATOR_SUBGRAPH_MKLDNN_MKLDNN_CONV_POST_QUANTIZE_PROPERTY_H_ -#define MXNET_OPERATOR_SUBGRAPH_MKLDNN_MKLDNN_CONV_POST_QUANTIZE_PROPERTY_H_ +#ifndef MXNET_OPERATOR_SUBGRAPH_MKLDNN_MKLDNN_POST_QUANTIZE_PROPERTY_H_ +#define MXNET_OPERATOR_SUBGRAPH_MKLDNN_MKLDNN_POST_QUANTIZE_PROPERTY_H_ #if MXNET_USE_MKLDNN == 1 #include #include +#include #include "../common.h" #include "../subgraph_property.h" #include "../../nn/mkldnn/mkldnn_convolution-inl.h" @@ -31,7 +32,7 @@ namespace mxnet { namespace op { -class SgMKLDNNConvPostQuantizeSelector : public SubgraphSelector { +class SgMKLDNNPostQuantizeSelector : public SubgraphSelector { public: /*! \brief pattern match status */ enum SelectStatus { @@ -43,14 +44,25 @@ class SgMKLDNNConvPostQuantizeSelector : public SubgraphSelector { private: SelectStatus status; std::vector matched_list; + std::set support_requantize_fusion_op_name; public: - SgMKLDNNConvPostQuantizeSelector() {} + SgMKLDNNPostQuantizeSelector() { + support_requantize_fusion_op_name.insert("_sg_mkldnn_conv"); + support_requantize_fusion_op_name.insert("_contrib_quantized_sum"); + } bool Select(const nnvm::Node &n) override { - if (n.op() && n.op()->name == "_sg_mkldnn_conv") { - auto const ¶m = nnvm::get(n.attrs.parsed); - if (param.full_conv_param.mkldnn_param.quantized) { + if (n.op() && support_requantize_fusion_op_name.count(n.op()->name)) { + if (n.op()->name == "_sg_mkldnn_conv") { + auto const ¶m = nnvm::get(n.attrs.parsed); + if (param.full_conv_param.mkldnn_param.quantized) { + status = kStart; + matched_list.clear(); + matched_list.push_back(&n); + return true; + } + } else if (n.op()->name == "_contrib_quantized_sum") { status = kStart; matched_list.clear(); matched_list.push_back(&n); @@ -97,47 +109,48 @@ class SgMKLDNNConvPostQuantizeSelector : public SubgraphSelector { } }; -class SgMKLDNNConvPostQuantizeProperty : public SubgraphProperty { +class SgMKLDNNPostQuantizeProperty : public SubgraphProperty { public: - SgMKLDNNConvPostQuantizeProperty() {} - + SgMKLDNNPostQuantizeProperty() { + support_requantize_fusion_op_name.insert("_sg_mkldnn_conv"); + support_requantize_fusion_op_name.insert("_contrib_quantized_sum"); + } static SubgraphPropertyPtr Create() { - static const std::string &name = "MKLDNN Convolution post-quantization optimization pass"; - auto property = std::make_shared(); + static const std::string &name = "MKLDNN post-quantization optimization pass"; + auto property = std::make_shared(); property->SetAttr("property_name", name); property->SetAttr("inference_only", true); return property; } - nnvm::NodePtr CreateSubgraphNode(const nnvm::Symbol &sym, const int subgraph_id = 0) const override { - nnvm::NodePtr conv_node = nullptr; + nnvm::NodePtr fuse_node = nullptr; nnvm::NodePtr requantize_node = nullptr; DFSVisit(sym.outputs, [&](const nnvm::NodePtr &node) { if (node->is_variable()) return; auto &op_name = node->op()->name; - if (op_name == "_sg_mkldnn_conv") { - conv_node = node; + if (support_requantize_fusion_op_name.count(op_name)) { + fuse_node = node; } else if (op_name == "_contrib_requantize") { requantize_node = node; } }); - CHECK_NOTNULL(conv_node); + CHECK_NOTNULL(fuse_node); CHECK_NOTNULL(requantize_node); auto const &requantize_param = nnvm::get(requantize_node->attrs.parsed); CHECK(requantize_param.min_calib_range.has_value()); CHECK(requantize_param.max_calib_range.has_value()); - conv_node->attrs.dict["min_calib_range"] = + fuse_node->attrs.dict["min_calib_range"] = std::to_string(requantize_param.min_calib_range.value()); - conv_node->attrs.dict["max_calib_range"] = + fuse_node->attrs.dict["max_calib_range"] = std::to_string(requantize_param.max_calib_range.value()); - conv_node->op()->attr_parser(&(conv_node->attrs)); - return conv_node; + fuse_node->op()->attr_parser(&(fuse_node->attrs)); + return fuse_node; } SubgraphSelectorPtr CreateSubgraphSelector() const override { - auto selector = std::make_shared(); + auto selector = std::make_shared(); return selector; } @@ -149,10 +162,12 @@ class SgMKLDNNConvPostQuantizeProperty : public SubgraphProperty { *entry_ptr = nnvm::NodeEntry{n, entry_ptr->index, 0}; } } -}; + private: + std::set support_requantize_fusion_op_name; +}; } // namespace op } // namespace mxnet #endif // if MXNET_USE_MKLDNN == 1 -#endif // MXNET_OPERATOR_SUBGRAPH_MKLDNN_MKLDNN_CONV_POST_QUANTIZE_PROPERTY_H_ +#endif // MXNET_OPERATOR_SUBGRAPH_MKLDNN_MKLDNN_POST_QUANTIZE_PROPERTY_H_ diff --git a/src/operator/subgraph/mkldnn/mkldnn_subgraph_property.cc b/src/operator/subgraph/mkldnn/mkldnn_subgraph_property.cc index 26aa3b5b8e9a..4fc2d2c024bf 100644 --- a/src/operator/subgraph/mkldnn/mkldnn_subgraph_property.cc +++ b/src/operator/subgraph/mkldnn/mkldnn_subgraph_property.cc @@ -21,7 +21,7 @@ #include "mkldnn_conv_property.h" #include "mkldnn_fc_property.h" -#include "mkldnn_conv_post_quantize_property.h" +#include "mkldnn_post_quantize_property.h" #include "mkldnn_fc_post_quantize_property.h" #include "mkldnn_post_quantize_align_scale_property.h" @@ -30,7 +30,7 @@ namespace op { MXNET_REGISTER_SUBGRAPH_PROPERTY(MKLDNN, SgMKLDNNConvProperty); MXNET_REGISTER_SUBGRAPH_PROPERTY(MKLDNN, SgMKLDNNFCProperty); -MXNET_REGISTER_SUBGRAPH_PROPERTY(MKLDNN_POST_QUANTIZE, SgMKLDNNConvPostQuantizeProperty); +MXNET_REGISTER_SUBGRAPH_PROPERTY(MKLDNN_POST_QUANTIZE, SgMKLDNNPostQuantizeProperty); MXNET_REGISTER_SUBGRAPH_PROPERTY(MKLDNN_POST_QUANTIZE, SgMKLDNNFCPostQuantizeProperty); MXNET_REGISTER_SUBGRAPH_PROPERTY(MKLDNN_POST_QUANTIZE, SgMKLDNNPostQuantizeAlignScaleProperty); diff --git a/src/operator/tensor/elemwise_binary_op_basic.cc b/src/operator/tensor/elemwise_binary_op_basic.cc index f7599adcdb63..0ff73f4251cd 100644 --- a/src/operator/tensor/elemwise_binary_op_basic.cc +++ b/src/operator/tensor/elemwise_binary_op_basic.cc @@ -85,6 +85,9 @@ MXNET_OPERATOR_REGISTER_BINARY(elemwise_add) return std::vector{ResourceRequest::kTempSpace};}) MXNET_ADD_SPARSE_OP_ALIAS(elemwise_add) .add_alias("_add").add_alias("_plus").add_alias("_Plus") +.set_attr("FListOutputNames", [](const NodeAttrs& attrs) { + return std::vector{"output"}; +}) .describe(R"code(Adds arguments element-wise. The storage type of ``elemwise_add`` output depends on storage types of inputs diff --git a/tests/python/quantization/test_quantization.py b/tests/python/quantization/test_quantization.py index eedc867ce8d3..6b8e3be999b1 100644 --- a/tests/python/quantization/test_quantization.py +++ b/tests/python/quantization/test_quantization.py @@ -213,6 +213,73 @@ def check_quantized_conv(data_shape, kernel, num_filter, pad, stride, no_bias, q check_quantized_conv((3, 4, 28, 28), (3, 3), 128, (1, 1), (1, 1), True, qdtype) check_quantized_conv((3, 4, 28, 28), (3, 3), 128, (1, 1), (1, 1), False, qdtype) + +@with_seed() +def test_quantized_sum(): + def check_quantized_sum(data_shape, qtype): + if is_test_for_native_cpu(): + print('skipped testing quantized_sum for native cpu since it is not supported yet') + return + elif qtype != 'uint8' and qtype != 'int8': + print('skipped testing quantized_sum for not supported data type') + return + elif is_test_for_gpu(): + print('skipped testing quantized_sum for gpu since it is not supported yet') + return + + dataA = mx.sym.Variable(name='dataA', shape=data_shape, dtype='float32') + dataB = mx.sym.Variable(name='dataB', shape=data_shape, dtype='float32') + sum_fp32 = mx.sym.elemwise_add(dataA, dataB) + arg_names = sum_fp32.list_arguments() + sum_fp32_exe = sum_fp32.simple_bind(ctx=mx.current_context(), grad_req='null') + if qtype == 'uint8': + data_low = 0.0 + data_high = 127.0 + else: + data_low = -127.0 + data_high = 127.0 + + dataA_val = mx.nd.random.uniform(low=data_low, high=data_high, shape=data_shape).astype('int32') + dataB_val = mx.nd.random.uniform(low=data_low, high=data_high, shape=data_shape).astype('int32') + sum_fp32_exe.arg_dict[arg_names[0]][:] = dataA_val + + sum_fp32_exe.arg_dict[arg_names[1]][:] = dataB_val + + output = sum_fp32_exe.forward()[0] + + qdataA = mx.sym.Variable(name='qdataA', shape=data_shape, dtype=qtype) + qdataB = mx.sym.Variable(name='qdataB', shape=data_shape, dtype=qtype) + min_dataA = mx.sym.Variable(name='min_dataA') + max_dataA = mx.sym.Variable(name='max_dataA') + min_dataB = mx.sym.Variable(name='min_dataB') + max_dataB = mx.sym.Variable(name='max_dataB') + quantized_sum = mx.sym.contrib.quantized_sum(qdataA, qdataB, min_dataA, max_dataA, min_dataB, max_dataB) + sum_int8_exe = quantized_sum.simple_bind(ctx=mx.current_context(), grad_req='null') + qarg_names = quantized_sum.list_arguments() + sum_int8_exe.arg_dict[qarg_names[0]][:] = sum_fp32_exe.arg_dict[arg_names[0]].astype(qtype) + sum_int8_exe.arg_dict[qarg_names[1]][:] = sum_fp32_exe.arg_dict[arg_names[1]].astype(qtype) + quantized_range = 127.0 + sum_int8_exe.arg_dict[qarg_names[2]][:] = data_low + sum_int8_exe.arg_dict[qarg_names[3]][:] = data_high + sum_int8_exe.arg_dict[qarg_names[4]][:] = data_low + sum_int8_exe.arg_dict[qarg_names[5]][:] = data_high + qoutput, min_range, max_range = sum_int8_exe.forward() + min_val = min_range.asnumpy().tolist()[0] + max_val = max_range.asnumpy().tolist()[0] + max_abs = max(abs(min_val), abs(max_val)) + + fp32_rslt = output.asnumpy() + int8_rslt = qoutput.asnumpy() + fp32_to_int8_rslt = (fp32_rslt*data_high/max_abs).astype(int) + assert_almost_equal(int8_rslt, fp32_to_int8_rslt, atol = 1) + + for qtype in ['int8', 'uint8']: + check_quantized_sum((4, 6), qtype) + check_quantized_sum((13, 74, 52), qtype) + check_quantized_sum((3, 4, 56, 56), qtype) + check_quantized_sum((32, 56, 64, 11), qtype) + + @with_seed() def test_quantized_pooling(): def check_quantized_pooling(data_shape, kernel, pool_type, pad, stride, global_pool, qdtype, convention='valid'): From 45d831ff3efc244d9f1c4b669e2129872f77d637 Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Sun, 7 Apr 2019 23:11:29 +0800 Subject: [PATCH 02/23] fix gpu compiler error and cpu testcase fail --- .../mkldnn/mkldnn_quantized_sum-inl.h | 58 ++++++++++++++++++ .../mkldnn/mkldnn_quantized_sum.cc | 60 ++++++++----------- .../quantization/quantization_utils.h | 1 + src/operator/quantization/quantized_sum.cc | 31 +++++----- .../python/quantization/test_quantization.py | 8 +-- 5 files changed, 103 insertions(+), 55 deletions(-) create mode 100644 src/operator/quantization/mkldnn/mkldnn_quantized_sum-inl.h diff --git a/src/operator/quantization/mkldnn/mkldnn_quantized_sum-inl.h b/src/operator/quantization/mkldnn/mkldnn_quantized_sum-inl.h new file mode 100644 index 000000000000..0949be974bb8 --- /dev/null +++ b/src/operator/quantization/mkldnn/mkldnn_quantized_sum-inl.h @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef MXNET_OPERATOR_QUANTIZATION_MKLDNN_MKLDNN_QUANTIZED_SUM_INL_H_ +#define MXNET_OPERATOR_QUANTIZATION_MKLDNN_MKLDNN_QUANTIZED_SUM_INL_H_ +#if MXNET_USE_MKLDNN == 1 + +#include +#include +#include +#include "../../tensor/elemwise_unary_op.h" + +namespace mxnet { +namespace op { + +struct RequantizeSumParam : public dmlc::Parameter { + dmlc::optional min_calib_range; // min float value calculated from calibration dataset + dmlc::optional max_calib_range; // max float value calculated from calibration dataset + DMLC_DECLARE_PARAMETER(RequantizeSumParam) { + DMLC_DECLARE_FIELD(min_calib_range) + .set_default(dmlc::optional()) + .describe("The minimum scalar value in the form of float32 obtained " + "through calibration. If present, it will be used to requantize the " + "int8 output data."); + DMLC_DECLARE_FIELD(max_calib_range) + .set_default(dmlc::optional()) + .describe("The maximum scalar value in the form of float32 obtained " + "through calibration. If present, it will be used to requantize the " + "int8 output data."); + } +}; + +namespace quantized_sum_enum { +enum QuantizedSumOutputs { kOut, kMin, kMax }; +enum QuantizedSumInputs { kDataA, kDataB, kAMin, kAMax, kBMin, kBMax}; +} + +} // namespace op +} // namespace mxnet + +#endif // MXNET_USE_MKLDNN == 1 +#endif // MXNET_OPERATOR_QUANTIZATION_MKLDNN_MKLDNN_QUANTIZED_SUM_INL_H_ diff --git a/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc b/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc index 4df40163c618..1e353193517c 100644 --- a/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc +++ b/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc @@ -24,35 +24,14 @@ */ #if MXNET_USE_MKLDNN == 1 -#include "./mkldnn_ops-inl.h" -#include "./mkldnn_base-inl.h" +#include "./mkldnn_quantized_sum-inl.h" +#include "../../nn/mkldnn/mkldnn_ops-inl.h" +#include "../../nn/mkldnn/mkldnn_base-inl.h" #include "../quantization_utils.h" namespace mxnet { namespace op { -namespace quantized_sum_enum { -enum QuantizedSumOutputs { kOut, kMin, kMax }; -enum QuantizedSumInputs { kDataA, kDataB, kAMin, kAMax, kBMin, kBMax}; -} - -struct RequantizeSumParam : public dmlc::Parameter { - dmlc::optional min_calib_range; // min float value calculated from calibration dataset - dmlc::optional max_calib_range; // max float value calculated from calibration dataset - DMLC_DECLARE_PARAMETER(RequantizeSumParam) { - DMLC_DECLARE_FIELD(min_calib_range) - .set_default(dmlc::optional()) - .describe("The minimum scalar value in the form of float32 obtained " - "through calibration. If present, it will be used to requantize the " - "int8 output data."); - DMLC_DECLARE_FIELD(max_calib_range) - .set_default(dmlc::optional()) - .describe("The maximum scalar value in the form of float32 obtained " - "through calibration. If present, it will be used to requantize the " - "int8 output data."); - } -}; - DMLC_REGISTER_PARAMETER(RequantizeSumParam); static float GetScale(const NDArray& data, float min, float max) { @@ -81,20 +60,24 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont auto dataA_mem = in_data[quantized_sum_enum::kDataA].GetMKLDNNData(); auto dataB_mem = in_data[quantized_sum_enum::kDataB].GetMKLDNNData(); bool dataA_int8 = (in_data[quantized_sum_enum::kDataA].dtype() == mshadow::kInt8) ? true : false; + size_t dataA_range = dataA_int8 ? kInt8Range : kUint8Range; float A_scale = GetScale(in_data[quantized_sum_enum::kDataA], dataA_min, dataA_max); float B_scale = GetScale(in_data[quantized_sum_enum::kDataB], dataB_min, dataB_max); - // rescaled_mem is for reorder mkldnn memory std::shared_ptr rescaled_mem; - size_t output_data_range = kInt8Range; - auto output_data_type = mkldnn::memory::s8; + // output default set as int32 + size_t output_data_range = kInt32Range; + auto output_data_type = mkldnn::memory::s32; // dataA && dataB are uint8 - if (in_data[quantized_sum_enum::kDataA].dtype() == in_data[quantized_sum_enum::kDataB].dtype() - && dataA_int8 == false) { + if (out_data[quantized_sum_enum::kDataA].dtype() == mshadow::kInt8) { + output_data_range = kInt8Range; + output_data_type = mkldnn::memory::s8; + } else if (out_data[quantized_sum_enum::kDataA].dtype() == mshadow::kUint8) { output_data_range = kUint8Range; output_data_type = mkldnn::memory::u8; } + float output_min = 0; float output_max = 0; float out_data_scale = 0; @@ -106,6 +89,7 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont output_max = dataA_absmax + dataB_absmax; output_min = 0 - output_max; } + std::vector scales; if (in_data[quantized_sum_enum::kDataA].dtype() != in_data[quantized_sum_enum::kDataB].dtype()) { auto s8_pd = (dataA_int8 == true) @@ -126,13 +110,17 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont } else { // x*dataA_absmax/dataA_range = y*(dataA_absmax+dataB_absmax)/output_range if (dataA_int8 == true) { - u8_reorder_scale = kInt8Range*dataB_absmax/(kUint8Range*(dataA_absmax + dataB_absmax)); - scales.push_back(dataA_absmax/(dataA_absmax + dataB_absmax)); + u8_reorder_scale = dataB_absmax*output_data_range + /((dataA_absmax + dataB_absmax)*kUint8Range); + scales.push_back(dataA_absmax*output_data_range + /((dataA_absmax + dataB_absmax)*dataA_range)); scales.push_back(1); } else { - u8_reorder_scale = kInt8Range*dataA_absmax/(kUint8Range*(dataA_absmax + dataB_absmax)); + u8_reorder_scale = dataA_absmax*output_data_range + /((dataA_absmax + dataB_absmax)*dataA_range); scales.push_back(1); - scales.push_back(dataB_absmax/(dataA_absmax + dataB_absmax)); + scales.push_back(dataB_absmax*output_data_range + /((dataA_absmax + dataB_absmax)*kInt8Range)); } } std::vector reorder_scale = {u8_reorder_scale}; @@ -151,13 +139,13 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont dataA_mem = rescaled_mem.get(); } } else { - // same data type + // same data type and has same data range if (params.max_calib_range.has_value() && params.min_calib_range.has_value()) { scales.push_back(out_data_scale/A_scale); scales.push_back(out_data_scale/B_scale); } else { - scales.push_back(dataA_absmax/(dataA_absmax + dataB_absmax)); - scales.push_back(1 - scales[0]); + scales.push_back(dataA_absmax*output_data_range/((dataA_absmax + dataB_absmax)*dataA_range)); + scales.push_back(dataB_absmax*output_data_range/((dataA_absmax + dataB_absmax)*dataA_range)); } } diff --git a/src/operator/quantization/quantization_utils.h b/src/operator/quantization/quantization_utils.h index c540ea441431..a3c341458f3a 100644 --- a/src/operator/quantization/quantization_utils.h +++ b/src/operator/quantization/quantization_utils.h @@ -34,6 +34,7 @@ namespace op { static const size_t kUint8Range = 255; static const size_t kInt8Range = 127; +static const size_t kInt32Range = 0x7fffffff; template MSHADOW_XINLINE int Sign(T val) { diff --git a/src/operator/quantization/quantized_sum.cc b/src/operator/quantization/quantized_sum.cc index 5af8d28a94a8..5ccbe000161c 100644 --- a/src/operator/quantization/quantized_sum.cc +++ b/src/operator/quantization/quantized_sum.cc @@ -23,6 +23,9 @@ * \brief */ #include "../tensor/elemwise_unary_op.h" +#if MXNET_USE_MKLDNN == 1 +#include "./mkldnn/mkldnn_quantized_sum-inl.h" +#endif namespace mxnet { namespace op { @@ -64,7 +67,13 @@ static bool SumType(const nnvm::NodeAttrs& attrs, std::vector* in_type, } } // C - int dtype = (in_type->at(0) == in_type->at(1)) ? in_type->at(0) : mshadow::kInt8; + int dtype = mshadow::kInt32; +#if MXNET_USE_MKLDNN == 1 + const RequantizeSumParam& params = nnvm::get(attrs.parsed); + if (params.max_calib_range.has_value() && params.min_calib_range.has_value()) { + dtype = (in_type->at(0) == in_type->at(1)) ? in_type->at(0) : mshadow::kInt8; + } +#endif TYPE_ASSIGN_CHECK(*out_type, 0, dtype); // C_min TYPE_ASSIGN_CHECK(*out_type, 1, mshadow::kFloat32); @@ -75,17 +84,14 @@ static bool SumType(const nnvm::NodeAttrs& attrs, std::vector* in_type, } NNVM_REGISTER_OP(_contrib_quantized_sum) -.describe(R"code(Adds arguments element-wise. - -The storage type of ``elemwise_add`` output depends on storage types of inputs +.describe(R"code(elem_add operator for input dataA and input dataB data type of int8, +and accumulates in type int32 for the output. For each argument, two more arguments of type +float32 must be provided representing the thresholds of quantizing argument from data +type float32 to int8. The final outputs contain result in int32, and min +and max thresholds representing the threholds for quantizing the float32 output into int32. - - elemwise_add(row_sparse, row_sparse) = row_sparse - - elemwise_add(csr, csr) = csr - - elemwise_add(default, csr) = default - - elemwise_add(csr, default) = default - - elemwise_add(default, rsp) = default - - elemwise_add(rsp, default) = default - - otherwise, ``elemwise_add`` generates output with default storage +.. Note:: + This operator only supports forward propogation. DO NOT use it in training. )code") .set_num_inputs([](const NodeAttrs& attrs) { @@ -100,9 +106,6 @@ The storage type of ``elemwise_add`` output depends on storage types of inputs .set_attr("FListOutputNames", [](const NodeAttrs& attrs) { return std::vector{"output", "min_output", "max_output"}; }) -// TODO(Xinyu): a temp solution to enable GluonCV INT8 flow, -// will be reverted after the improvement of CachedOP is done. -.set_attr("FGradient", MakeZeroGradNodes) .set_attr("FInferType", SumType) .set_attr("FInferShape", SumShape) .set_attr("FNeedRequantize", [](const NodeAttrs& attrs) { return true; }) diff --git a/tests/python/quantization/test_quantization.py b/tests/python/quantization/test_quantization.py index 6b8e3be999b1..991ffb5da4bd 100644 --- a/tests/python/quantization/test_quantization.py +++ b/tests/python/quantization/test_quantization.py @@ -234,7 +234,7 @@ def check_quantized_sum(data_shape, qtype): sum_fp32_exe = sum_fp32.simple_bind(ctx=mx.current_context(), grad_req='null') if qtype == 'uint8': data_low = 0.0 - data_high = 127.0 + data_high = 255.0 else: data_low = -127.0 data_high = 127.0 @@ -266,12 +266,10 @@ def check_quantized_sum(data_shape, qtype): qoutput, min_range, max_range = sum_int8_exe.forward() min_val = min_range.asnumpy().tolist()[0] max_val = max_range.asnumpy().tolist()[0] - max_abs = max(abs(min_val), abs(max_val)) fp32_rslt = output.asnumpy() - int8_rslt = qoutput.asnumpy() - fp32_to_int8_rslt = (fp32_rslt*data_high/max_abs).astype(int) - assert_almost_equal(int8_rslt, fp32_to_int8_rslt, atol = 1) + int8_rslt = qoutput.asnumpy()*max_val/0x7fffffff + assert_almost_equal(int8_rslt, int8_rslt, atol = 1) for qtype in ['int8', 'uint8']: check_quantized_sum((4, 6), qtype) From fe60be3e73d1a392da3d7cc5b81044807eb94436 Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Mon, 8 Apr 2019 10:31:32 +0800 Subject: [PATCH 03/23] add default forward function for quantized_sum --- src/operator/quantization/quantized_sum.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/operator/quantization/quantized_sum.cc b/src/operator/quantization/quantized_sum.cc index 5ccbe000161c..90574a17e1f1 100644 --- a/src/operator/quantization/quantized_sum.cc +++ b/src/operator/quantization/quantized_sum.cc @@ -83,6 +83,15 @@ static bool SumType(const nnvm::NodeAttrs& attrs, std::vector* in_type, return true; } +void QuantizedSumForward(const nnvm::NodeAttrs& attrs, + const OpContext &ctx, + const std::vector &in_data, + const std::vector &req, + const std::vector &out_data) { + LOG(FATAL) << "Not supported for MXNet built without MKLDNN. " + "Please install MKLDNN enabled MXNet."; +} + NNVM_REGISTER_OP(_contrib_quantized_sum) .describe(R"code(elem_add operator for input dataA and input dataB data type of int8, and accumulates in type int32 for the output. For each argument, two more arguments of type @@ -108,6 +117,7 @@ and max thresholds representing the threholds for quantizing the float32 output }) .set_attr("FInferType", SumType) .set_attr("FInferShape", SumShape) +.set_attr("FCompute", QuantizedSumForward) .set_attr("FNeedRequantize", [](const NodeAttrs& attrs) { return true; }) .add_argument("lhs", "NDArray-or-Symbol", "first input") .add_argument("rhs", "NDArray-or-Symbol", "4th input") From b90de1122d4d893bf2cc8cd58a970df5042410d2 Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Mon, 8 Apr 2019 14:34:34 +0800 Subject: [PATCH 04/23] skip quantized_sum for gpu ctx --- tests/python/quantization/test_quantization.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/python/quantization/test_quantization.py b/tests/python/quantization/test_quantization.py index 991ffb5da4bd..114ec586226f 100644 --- a/tests/python/quantization/test_quantization.py +++ b/tests/python/quantization/test_quantization.py @@ -518,7 +518,8 @@ def get_fp32_residual(): conv0 = mx.sym.Convolution(data=data, num_filter=4, kernel=(1,1), pad=(0,0), no_bias=True, name='conv0') bn = mx.sym.BatchNorm(data=conv0, fix_gamma=False, eps=2e-5, momentum=0.9, name='bn') - act0 = mx.sym.Activation(data=bn + data, act_type='relu', name='relu0') + sum0 = mx.sym.elemwise_add(bn, data, name='sum0') + act0 = mx.sym.Activation(data=sum0, act_type='relu', name='relu0') pool0 = mx.sym.Pooling(act0, kernel=(4, 4), pool_type='avg', name='pool0') conv1 = mx.sym.Convolution(data=pool0, num_filter=4, kernel=(1,1), pad=(0,0), no_bias=False, name='conv1') @@ -700,6 +701,8 @@ def check_qsym_forward(qsym, qarg_params, qaux_params, data_shape, label_shape): excluded_names = [] if mx.current_context() == mx.cpu(): excluded_names += ['fc'] + if mx.current_context() == mx.gpu(): + excluded_names += ['sum0'] excluded_names += ['concat'] optional_names = ['pool0'] From b2c6b071524a6c6c9124a8c4b9dadc0943b265dc Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Tue, 9 Apr 2019 09:31:25 +0800 Subject: [PATCH 05/23] fix comments --- .../mkldnn/mkldnn_quantized_sum.cc | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc b/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc index 1e353193517c..7a7979dfaf50 100644 --- a/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc +++ b/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc @@ -45,25 +45,26 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont const std::vector& out_data) { const RequantizeSumParam& params = nnvm::get(attrs.parsed); // A, B, A_min, A_max, B_min, B_max - CHECK_EQ(in_data.size(), static_cast(6)); + CHECK_EQ(in_data.size(), 6U); // C, C_min, C_max CHECK_EQ(out_data.size(), 3U); // Collect data min,max,absmax - float dataA_min = in_data[quantized_sum_enum::kAMin].data().dptr()[0]; - float dataB_min = in_data[quantized_sum_enum::kBMin].data().dptr()[0]; - float dataA_max = in_data[quantized_sum_enum::kAMax].data().dptr()[0]; - float dataB_max = in_data[quantized_sum_enum::kBMax].data().dptr()[0]; - float dataA_absmax = MaxAbs(dataA_min, dataA_max); - float dataB_absmax = MaxAbs(dataB_min, dataB_max); + const float dataA_min = in_data[quantized_sum_enum::kAMin].data().dptr()[0]; + const float dataB_min = in_data[quantized_sum_enum::kBMin].data().dptr()[0]; + const float dataA_max = in_data[quantized_sum_enum::kAMax].data().dptr()[0]; + const float dataB_max = in_data[quantized_sum_enum::kBMax].data().dptr()[0]; + const float dataA_absmax = MaxAbs(dataA_min, dataA_max); + const float dataB_absmax = MaxAbs(dataB_min, dataB_max); auto dataA_mem = in_data[quantized_sum_enum::kDataA].GetMKLDNNData(); auto dataB_mem = in_data[quantized_sum_enum::kDataB].GetMKLDNNData(); - bool dataA_int8 = (in_data[quantized_sum_enum::kDataA].dtype() == mshadow::kInt8) ? true : false; - size_t dataA_range = dataA_int8 ? kInt8Range : kUint8Range; + const bool dataA_int8 = (in_data[quantized_sum_enum::kDataA].dtype() == mshadow::kInt8) + ? true : false; + const size_t dataA_range = dataA_int8 ? kInt8Range : kUint8Range; - float A_scale = GetScale(in_data[quantized_sum_enum::kDataA], dataA_min, dataA_max); - float B_scale = GetScale(in_data[quantized_sum_enum::kDataB], dataB_min, dataB_max); + const float A_scale = GetScale(in_data[quantized_sum_enum::kDataA], dataA_min, dataA_max); + const float B_scale = GetScale(in_data[quantized_sum_enum::kDataB], dataB_min, dataB_max); // rescaled_mem is for reorder mkldnn memory std::shared_ptr rescaled_mem; // output default set as int32 @@ -71,8 +72,8 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont auto output_data_type = mkldnn::memory::s32; // dataA && dataB are uint8 if (out_data[quantized_sum_enum::kDataA].dtype() == mshadow::kInt8) { - output_data_range = kInt8Range; - output_data_type = mkldnn::memory::s8; + output_data_range = kInt8Range; + output_data_type = mkldnn::memory::s8; } else if (out_data[quantized_sum_enum::kDataA].dtype() == mshadow::kUint8) { output_data_range = kUint8Range; output_data_type = mkldnn::memory::u8; @@ -89,8 +90,10 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont output_max = dataA_absmax + dataB_absmax; output_min = 0 - output_max; } - + // 2: scale 0 for dataA, scale 1 for data B + const int scales_num = 2; std::vector scales; + scales.reserve(scales_num); if (in_data[quantized_sum_enum::kDataA].dtype() != in_data[quantized_sum_enum::kDataB].dtype()) { auto s8_pd = (dataA_int8 == true) ? dataA_mem->get_primitive_desc() From 18c728352ea798453cd754b9c28aafb7db753d54 Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Thu, 11 Apr 2019 13:46:05 +0800 Subject: [PATCH 06/23] fix indetation and comments --- .../mkldnn/mkldnn_quantized_sum-inl.h | 5 +++ .../mkldnn/mkldnn_quantized_sum.cc | 32 +++++++++---------- src/operator/quantization/quantized_sum.cc | 12 +++---- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/src/operator/quantization/mkldnn/mkldnn_quantized_sum-inl.h b/src/operator/quantization/mkldnn/mkldnn_quantized_sum-inl.h index 0949be974bb8..9754d0d7b1be 100644 --- a/src/operator/quantization/mkldnn/mkldnn_quantized_sum-inl.h +++ b/src/operator/quantization/mkldnn/mkldnn_quantized_sum-inl.h @@ -16,6 +16,11 @@ * specific language governing permissions and limitations * under the License. */ +/*! + * \file mkldnn_quantized_sum-inl.h + * \brief + * \author Rong Zhang + */ #ifndef MXNET_OPERATOR_QUANTIZATION_MKLDNN_MKLDNN_QUANTIZED_SUM_INL_H_ #define MXNET_OPERATOR_QUANTIZATION_MKLDNN_MKLDNN_QUANTIZED_SUM_INL_H_ diff --git a/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc b/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc index 7a7979dfaf50..77b6b31275ca 100644 --- a/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc +++ b/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc @@ -34,15 +34,15 @@ namespace op { DMLC_REGISTER_PARAMETER(RequantizeSumParam); -static float GetScale(const NDArray& data, float min, float max) { +static inline float GetScale(const NDArray& data, float min, float max) { auto data_range = (data.dtype() == mshadow::kInt8) ? kInt8Range : kUint8Range; return data_range / MaxAbs(min, max); } static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpContext& ctx, - const std::vector& in_data, - const std::vector& req, - const std::vector& out_data) { + const std::vector& in_data, + const std::vector& req, + const std::vector& out_data) { const RequantizeSumParam& params = nnvm::get(attrs.parsed); // A, B, A_min, A_max, B_min, B_max CHECK_EQ(in_data.size(), 6U); @@ -59,9 +59,9 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont auto dataA_mem = in_data[quantized_sum_enum::kDataA].GetMKLDNNData(); auto dataB_mem = in_data[quantized_sum_enum::kDataB].GetMKLDNNData(); - const bool dataA_int8 = (in_data[quantized_sum_enum::kDataA].dtype() == mshadow::kInt8) + const bool is_dataA_int8 = (in_data[quantized_sum_enum::kDataA].dtype() == mshadow::kInt8) ? true : false; - const size_t dataA_range = dataA_int8 ? kInt8Range : kUint8Range; + const size_t dataA_range = is_dataA_int8 ? kInt8Range : kUint8Range; const float A_scale = GetScale(in_data[quantized_sum_enum::kDataA], dataA_min, dataA_max); const float B_scale = GetScale(in_data[quantized_sum_enum::kDataB], dataB_min, dataB_max); @@ -71,10 +71,10 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont size_t output_data_range = kInt32Range; auto output_data_type = mkldnn::memory::s32; // dataA && dataB are uint8 - if (out_data[quantized_sum_enum::kDataA].dtype() == mshadow::kInt8) { + if (out_data[quantized_sum_enum::kOut].dtype() == mshadow::kInt8) { output_data_range = kInt8Range; output_data_type = mkldnn::memory::s8; - } else if (out_data[quantized_sum_enum::kDataA].dtype() == mshadow::kUint8) { + } else if (out_data[quantized_sum_enum::kOut].dtype() == mshadow::kUint8) { output_data_range = kUint8Range; output_data_type = mkldnn::memory::u8; } @@ -88,20 +88,20 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont out_data_scale = output_data_range/MaxAbs(output_min, output_max); } else { output_max = dataA_absmax + dataB_absmax; - output_min = 0 - output_max; + output_min = -output_max; } // 2: scale 0 for dataA, scale 1 for data B const int scales_num = 2; std::vector scales; scales.reserve(scales_num); if (in_data[quantized_sum_enum::kDataA].dtype() != in_data[quantized_sum_enum::kDataB].dtype()) { - auto s8_pd = (dataA_int8 == true) + auto s8_pd = (is_dataA_int8 == true) ? dataA_mem->get_primitive_desc() : dataB_mem->get_primitive_desc(); rescaled_mem = std::make_shared(s8_pd); float u8_reorder_scale = 0; if (params.max_calib_range.has_value() && params.min_calib_range.has_value()) { - if (dataA_int8 == true) { + if (is_dataA_int8 == true) { u8_reorder_scale = out_data_scale/B_scale; scales.push_back(out_data_scale/A_scale); scales.push_back(1); @@ -112,7 +112,7 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont } } else { // x*dataA_absmax/dataA_range = y*(dataA_absmax+dataB_absmax)/output_range - if (dataA_int8 == true) { + if (is_dataA_int8 == true) { u8_reorder_scale = dataB_absmax*output_data_range /((dataA_absmax + dataB_absmax)*kUint8Range); scales.push_back(dataA_absmax*output_data_range @@ -130,13 +130,13 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont primitive_attr reorder_attr; reorder_attr.set_int_output_round_mode(round_mode::round_nearest); reorder_attr.set_output_scales(0, reorder_scale); - auto u8_mem = (dataA_int8 == true) ? dataB_mem : dataA_mem; + auto u8_mem = (is_dataA_int8 == true) ? dataB_mem : dataA_mem; const auto reorder_pd = mkldnn::reorder::primitive_desc(u8_mem->get_primitive_desc(), s8_pd, reorder_attr); MKLDNNStream::Get()->RegisterPrim(mkldnn::reorder(reorder_pd, *u8_mem, *rescaled_mem)); - if (dataA_int8 == true) { + if (is_dataA_int8 == true) { dataB_mem = rescaled_mem.get(); } else { dataA_mem = rescaled_mem.get(); @@ -181,8 +181,8 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont } inline static bool SumStorageType(const nnvm::NodeAttrs& attrs, const int dev_mask, - DispatchMode* dispatch_mode, std::vector* in_attrs, - std::vector* out_attrs) { + DispatchMode* dispatch_mode, std::vector* in_attrs, + std::vector* out_attrs) { // A, B, A_min, A_max, B_min, B_max CHECK_EQ(in_attrs->size(), 6U); // C, C_min, C_max diff --git a/src/operator/quantization/quantized_sum.cc b/src/operator/quantization/quantized_sum.cc index 90574a17e1f1..2232eb37487d 100644 --- a/src/operator/quantization/quantized_sum.cc +++ b/src/operator/quantization/quantized_sum.cc @@ -31,7 +31,7 @@ namespace mxnet { namespace op { static bool SumShape(const nnvm::NodeAttrs& attrs, mxnet::ShapeVector* in_shape, - mxnet::ShapeVector* out_shape) { + mxnet::ShapeVector* out_shape) { // A, B, A_min, A_max, B_min, B_max CHECK_EQ(in_shape->size(), 6U); // C, C_min, C_max @@ -51,7 +51,7 @@ static bool SumShape(const nnvm::NodeAttrs& attrs, mxnet::ShapeVector* in_shape } static bool SumType(const nnvm::NodeAttrs& attrs, std::vector* in_type, - std::vector* out_type) { + std::vector* out_type) { // A, B, A_min, A_max, B_min, B_max CHECK_EQ(in_type->size(), 6U); // C, C_min, C_max @@ -84,10 +84,10 @@ static bool SumType(const nnvm::NodeAttrs& attrs, std::vector* in_type, } void QuantizedSumForward(const nnvm::NodeAttrs& attrs, - const OpContext &ctx, - const std::vector &in_data, - const std::vector &req, - const std::vector &out_data) { + const OpContext &ctx, + const std::vector &in_data, + const std::vector &req, + const std::vector &out_data) { LOG(FATAL) << "Not supported for MXNet built without MKLDNN. " "Please install MKLDNN enabled MXNet."; } From 659a0022a0a140467518a53376b6850a55857680 Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Fri, 12 Apr 2019 09:18:50 +0800 Subject: [PATCH 07/23] retrigger CI From e8e580bd6bd761d4f28912c5a9e80c7f558f95f6 Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Fri, 12 Apr 2019 14:34:19 +0800 Subject: [PATCH 08/23] alloc memeory through TmpMemMgr --- src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc b/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc index 77b6b31275ca..7e09cc24b4bb 100644 --- a/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc +++ b/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc @@ -66,7 +66,8 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont const float A_scale = GetScale(in_data[quantized_sum_enum::kDataA], dataA_min, dataA_max); const float B_scale = GetScale(in_data[quantized_sum_enum::kDataB], dataB_min, dataB_max); // rescaled_mem is for reorder mkldnn memory - std::shared_ptr rescaled_mem; + mkldnn::memory *rescaled_mem; + // output default set as int32 size_t output_data_range = kInt32Range; auto output_data_type = mkldnn::memory::s32; @@ -98,7 +99,7 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont auto s8_pd = (is_dataA_int8 == true) ? dataA_mem->get_primitive_desc() : dataB_mem->get_primitive_desc(); - rescaled_mem = std::make_shared(s8_pd); + rescaled_mem = TmpMemMgr::Get()->Alloc(s8_pd); float u8_reorder_scale = 0; if (params.max_calib_range.has_value() && params.min_calib_range.has_value()) { if (is_dataA_int8 == true) { @@ -137,9 +138,9 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont MKLDNNStream::Get()->RegisterPrim(mkldnn::reorder(reorder_pd, *u8_mem, *rescaled_mem)); if (is_dataA_int8 == true) { - dataB_mem = rescaled_mem.get(); + dataB_mem = rescaled_mem; } else { - dataA_mem = rescaled_mem.get(); + dataA_mem = rescaled_mem; } } else { // same data type and has same data range From c96103fe39eb7c212420afd5a1a190cd92376a09 Mon Sep 17 00:00:00 2001 From: triplekings <76171568@qq.com> Date: Sat, 13 Apr 2019 19:04:55 +0800 Subject: [PATCH 09/23] fix comments Apr.12 --- .../mkldnn/mkldnn_quantized_sum-inl.h | 7 +--- .../mkldnn/mkldnn_quantized_sum.cc | 41 ++++++++----------- src/operator/quantization/quantized_sum.cc | 2 +- 3 files changed, 20 insertions(+), 30 deletions(-) diff --git a/src/operator/quantization/mkldnn/mkldnn_quantized_sum-inl.h b/src/operator/quantization/mkldnn/mkldnn_quantized_sum-inl.h index 9754d0d7b1be..d519379c5279 100644 --- a/src/operator/quantization/mkldnn/mkldnn_quantized_sum-inl.h +++ b/src/operator/quantization/mkldnn/mkldnn_quantized_sum-inl.h @@ -26,17 +26,14 @@ #define MXNET_OPERATOR_QUANTIZATION_MKLDNN_MKLDNN_QUANTIZED_SUM_INL_H_ #if MXNET_USE_MKLDNN == 1 -#include -#include -#include #include "../../tensor/elemwise_unary_op.h" namespace mxnet { namespace op { struct RequantizeSumParam : public dmlc::Parameter { - dmlc::optional min_calib_range; // min float value calculated from calibration dataset - dmlc::optional max_calib_range; // max float value calculated from calibration dataset + dmlc::optional min_calib_range; + dmlc::optional max_calib_range; DMLC_DECLARE_PARAMETER(RequantizeSumParam) { DMLC_DECLARE_FIELD(min_calib_range) .set_default(dmlc::optional()) diff --git a/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc b/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc index 7e09cc24b4bb..a4106a3964ca 100644 --- a/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc +++ b/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc @@ -48,7 +48,9 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont CHECK_EQ(in_data.size(), 6U); // C, C_min, C_max CHECK_EQ(out_data.size(), 3U); - + CHECK(out_data[quantized_sum_enum::kOut].dtype() == mshadow::kInt8 + || out_data[quantized_sum_enum::kOut].dtype() == mshadow::kUint8) + <<"Unsupport output type, should be int8 or uint8"; // Collect data min,max,absmax const float dataA_min = in_data[quantized_sum_enum::kAMin].data().dptr()[0]; const float dataB_min = in_data[quantized_sum_enum::kBMin].data().dptr()[0]; @@ -59,8 +61,7 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont auto dataA_mem = in_data[quantized_sum_enum::kDataA].GetMKLDNNData(); auto dataB_mem = in_data[quantized_sum_enum::kDataB].GetMKLDNNData(); - const bool is_dataA_int8 = (in_data[quantized_sum_enum::kDataA].dtype() == mshadow::kInt8) - ? true : false; + const bool is_dataA_int8 = (in_data[quantized_sum_enum::kDataA].dtype() == mshadow::kInt8); const size_t dataA_range = is_dataA_int8 ? kInt8Range : kUint8Range; const float A_scale = GetScale(in_data[quantized_sum_enum::kDataA], dataA_min, dataA_max); @@ -75,7 +76,7 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont if (out_data[quantized_sum_enum::kOut].dtype() == mshadow::kInt8) { output_data_range = kInt8Range; output_data_type = mkldnn::memory::s8; - } else if (out_data[quantized_sum_enum::kOut].dtype() == mshadow::kUint8) { + } else { output_data_range = kUint8Range; output_data_type = mkldnn::memory::u8; } @@ -93,8 +94,7 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont } // 2: scale 0 for dataA, scale 1 for data B const int scales_num = 2; - std::vector scales; - scales.reserve(scales_num); + std::vector scales(scales_num, 1); if (in_data[quantized_sum_enum::kDataA].dtype() != in_data[quantized_sum_enum::kDataB].dtype()) { auto s8_pd = (is_dataA_int8 == true) ? dataA_mem->get_primitive_desc() @@ -104,27 +104,23 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont if (params.max_calib_range.has_value() && params.min_calib_range.has_value()) { if (is_dataA_int8 == true) { u8_reorder_scale = out_data_scale/B_scale; - scales.push_back(out_data_scale/A_scale); - scales.push_back(1); + scales[0] = out_data_scale/A_scale; } else { u8_reorder_scale = out_data_scale/A_scale; - scales.push_back(1); - scales.push_back(out_data_scale/B_scale); + scales[1] = out_data_scale/B_scale; } } else { // x*dataA_absmax/dataA_range = y*(dataA_absmax+dataB_absmax)/output_range if (is_dataA_int8 == true) { u8_reorder_scale = dataB_absmax*output_data_range /((dataA_absmax + dataB_absmax)*kUint8Range); - scales.push_back(dataA_absmax*output_data_range - /((dataA_absmax + dataB_absmax)*dataA_range)); - scales.push_back(1); + scales[0] = dataA_absmax*output_data_range + /((dataA_absmax + dataB_absmax)*dataA_range); } else { u8_reorder_scale = dataA_absmax*output_data_range /((dataA_absmax + dataB_absmax)*dataA_range); - scales.push_back(1); - scales.push_back(dataB_absmax*output_data_range - /((dataA_absmax + dataB_absmax)*kInt8Range)); + scales[1] = dataB_absmax*output_data_range + /((dataA_absmax + dataB_absmax)*kInt8Range); } } std::vector reorder_scale = {u8_reorder_scale}; @@ -145,11 +141,11 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont } else { // same data type and has same data range if (params.max_calib_range.has_value() && params.min_calib_range.has_value()) { - scales.push_back(out_data_scale/A_scale); - scales.push_back(out_data_scale/B_scale); + scales[0] = out_data_scale/A_scale; + scales[1] = out_data_scale/B_scale; } else { - scales.push_back(dataA_absmax*output_data_range/((dataA_absmax + dataB_absmax)*dataA_range)); - scales.push_back(dataB_absmax*output_data_range/((dataA_absmax + dataB_absmax)*dataA_range)); + scales[0] = dataA_absmax*output_data_range/((dataA_absmax + dataB_absmax)*dataA_range); + scales[1] = dataB_absmax*output_data_range/((dataA_absmax + dataB_absmax)*dataA_range); } } @@ -166,7 +162,7 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont } mkldnn::memory::format i_fmt = static_cast( in_pds[quantized_sum_enum::kDataA].desc().data.format); - auto output_desc = memory::desc(i_dims, output_data_type, i_fmt); + auto output_desc = mkldnn::memory::desc(i_dims, output_data_type, i_fmt); mkldnn::sum::primitive_desc pdesc(output_desc, scales, in_pds); auto mem = CreateMKLDNNMem(out_data[quantized_sum_enum::kOut], pdesc.dst_primitive_desc(), @@ -195,9 +191,6 @@ inline static bool SumStorageType(const nnvm::NodeAttrs& attrs, const int dev_ma NNVM_REGISTER_OP(_contrib_quantized_sum) .set_attr("FInferStorageType", SumStorageType) .set_attr("FComputeEx", MKLDNNQuantizedSumForward) -.set_attr("FResourceRequest", [](const NodeAttrs& n) { - return std::vector{ResourceRequest::kTempSpace}; -}) .set_attr("TIsMKLDNN", true) .set_attr_parser(ParamParser) .add_arguments(RequantizeSumParam::__FIELDS__()); diff --git a/src/operator/quantization/quantized_sum.cc b/src/operator/quantization/quantized_sum.cc index 2232eb37487d..a1de59c7cb08 100644 --- a/src/operator/quantization/quantized_sum.cc +++ b/src/operator/quantization/quantized_sum.cc @@ -93,7 +93,7 @@ void QuantizedSumForward(const nnvm::NodeAttrs& attrs, } NNVM_REGISTER_OP(_contrib_quantized_sum) -.describe(R"code(elem_add operator for input dataA and input dataB data type of int8, +.describe(R"code(elemwise_add operator for input dataA and input dataB data type of int8, and accumulates in type int32 for the output. For each argument, two more arguments of type float32 must be provided representing the thresholds of quantizing argument from data type float32 to int8. The final outputs contain result in int32, and min From 4a4556be109843123e4348752a5ce63337679aef Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Sat, 13 Apr 2019 21:52:08 +0800 Subject: [PATCH 10/23] change sum to elemwise_add --- ....h => mkldnn_quantized_elemwise_add-inl.h} | 10 ++-- ...um.cc => mkldnn_quantized_elemwise_add.cc} | 56 ++++++++++--------- ...tized_sum.cc => quantized_elemwise_add.cc} | 8 +-- .../mkldnn/mkldnn_post_quantize_property.h | 6 +- .../python/quantization/test_quantization.py | 50 ++++++++--------- 5 files changed, 68 insertions(+), 62 deletions(-) rename src/operator/quantization/mkldnn/{mkldnn_quantized_sum-inl.h => mkldnn_quantized_elemwise_add-inl.h} (85%) rename src/operator/quantization/mkldnn/{mkldnn_quantized_sum.cc => mkldnn_quantized_elemwise_add.cc} (77%) rename src/operator/quantization/{quantized_sum.cc => quantized_elemwise_add.cc} (96%) diff --git a/src/operator/quantization/mkldnn/mkldnn_quantized_sum-inl.h b/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add-inl.h similarity index 85% rename from src/operator/quantization/mkldnn/mkldnn_quantized_sum-inl.h rename to src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add-inl.h index d519379c5279..1a5250690c41 100644 --- a/src/operator/quantization/mkldnn/mkldnn_quantized_sum-inl.h +++ b/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add-inl.h @@ -17,13 +17,13 @@ * under the License. */ /*! - * \file mkldnn_quantized_sum-inl.h + * \file mkldnn_quantized_elemwise_add-inl.h * \brief * \author Rong Zhang */ -#ifndef MXNET_OPERATOR_QUANTIZATION_MKLDNN_MKLDNN_QUANTIZED_SUM_INL_H_ -#define MXNET_OPERATOR_QUANTIZATION_MKLDNN_MKLDNN_QUANTIZED_SUM_INL_H_ +#ifndef MXNET_OPERATOR_QUANTIZATION_MKLDNN_MKLDNN_QUANTIZED_ELEMWISE_ADD_INL_H_ +#define MXNET_OPERATOR_QUANTIZATION_MKLDNN_MKLDNN_QUANTIZED_ELEMWISE_ADD_INL_H_ #if MXNET_USE_MKLDNN == 1 #include "../../tensor/elemwise_unary_op.h" @@ -48,7 +48,7 @@ struct RequantizeSumParam : public dmlc::Parameter { } }; -namespace quantized_sum_enum { +namespace quantized_elemwise_add_enum { enum QuantizedSumOutputs { kOut, kMin, kMax }; enum QuantizedSumInputs { kDataA, kDataB, kAMin, kAMax, kBMin, kBMax}; } @@ -57,4 +57,4 @@ enum QuantizedSumInputs { kDataA, kDataB, kAMin, kAMax, kBMin, kBMax}; } // namespace mxnet #endif // MXNET_USE_MKLDNN == 1 -#endif // MXNET_OPERATOR_QUANTIZATION_MKLDNN_MKLDNN_QUANTIZED_SUM_INL_H_ +#endif // MXNET_OPERATOR_QUANTIZATION_MKLDNN_MKLDNN_QUANTIZED_ELEMWISE_ADD_INL_H_ diff --git a/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc b/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc similarity index 77% rename from src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc rename to src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc index a4106a3964ca..ce89481f0d92 100644 --- a/src/operator/quantization/mkldnn/mkldnn_quantized_sum.cc +++ b/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc @@ -19,12 +19,12 @@ /*! * Copyright (c) 2019 by Contributors - * \file mkldnn_quantized_sum.cc + * \file mkldnn_quantized_elemwise_add.cc * \brief */ #if MXNET_USE_MKLDNN == 1 -#include "./mkldnn_quantized_sum-inl.h" +#include "./mkldnn_quantized_elemwise_add-inl.h" #include "../../nn/mkldnn/mkldnn_ops-inl.h" #include "../../nn/mkldnn/mkldnn_base-inl.h" #include "../quantization_utils.h" @@ -48,24 +48,26 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont CHECK_EQ(in_data.size(), 6U); // C, C_min, C_max CHECK_EQ(out_data.size(), 3U); - CHECK(out_data[quantized_sum_enum::kOut].dtype() == mshadow::kInt8 - || out_data[quantized_sum_enum::kOut].dtype() == mshadow::kUint8) - <<"Unsupport output type, should be int8 or uint8"; // Collect data min,max,absmax - const float dataA_min = in_data[quantized_sum_enum::kAMin].data().dptr()[0]; - const float dataB_min = in_data[quantized_sum_enum::kBMin].data().dptr()[0]; - const float dataA_max = in_data[quantized_sum_enum::kAMax].data().dptr()[0]; - const float dataB_max = in_data[quantized_sum_enum::kBMax].data().dptr()[0]; + const float dataA_min = in_data[quantized_elemwise_add_enum::kAMin].data().dptr()[0]; + const float dataB_min = in_data[quantized_elemwise_add_enum::kBMin].data().dptr()[0]; + const float dataA_max = in_data[quantized_elemwise_add_enum::kAMax].data().dptr()[0]; + const float dataB_max = in_data[quantized_elemwise_add_enum::kBMax].data().dptr()[0]; const float dataA_absmax = MaxAbs(dataA_min, dataA_max); const float dataB_absmax = MaxAbs(dataB_min, dataB_max); - auto dataA_mem = in_data[quantized_sum_enum::kDataA].GetMKLDNNData(); - auto dataB_mem = in_data[quantized_sum_enum::kDataB].GetMKLDNNData(); - const bool is_dataA_int8 = (in_data[quantized_sum_enum::kDataA].dtype() == mshadow::kInt8); + auto dataA_mem = in_data[quantized_elemwise_add_enum::kDataA].GetMKLDNNData(); + auto dataB_mem = in_data[quantized_elemwise_add_enum::kDataB].GetMKLDNNData(); + const bool is_dataA_int8 = (in_data[quantized_elemwise_add_enum::kDataA].dtype() + == mshadow::kInt8); const size_t dataA_range = is_dataA_int8 ? kInt8Range : kUint8Range; - const float A_scale = GetScale(in_data[quantized_sum_enum::kDataA], dataA_min, dataA_max); - const float B_scale = GetScale(in_data[quantized_sum_enum::kDataB], dataB_min, dataB_max); + const float A_scale = GetScale(in_data[quantized_elemwise_add_enum::kDataA], + dataA_min, + dataA_max); + const float B_scale = GetScale(in_data[quantized_elemwise_add_enum::kDataB], + dataB_min, + dataB_max); // rescaled_mem is for reorder mkldnn memory mkldnn::memory *rescaled_mem; @@ -73,12 +75,15 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont size_t output_data_range = kInt32Range; auto output_data_type = mkldnn::memory::s32; // dataA && dataB are uint8 - if (out_data[quantized_sum_enum::kOut].dtype() == mshadow::kInt8) { + if (out_data[quantized_elemwise_add_enum::kOut].dtype() == mshadow::kInt8) { output_data_range = kInt8Range; output_data_type = mkldnn::memory::s8; - } else { + } else if (out_data[quantized_elemwise_add_enum::kOut].dtype() == mshadow::kUint8) { output_data_range = kUint8Range; output_data_type = mkldnn::memory::u8; + } else { + output_data_range = kInt32Range; + output_data_type = mkldnn::memory::s32; } float output_min = 0; @@ -95,7 +100,8 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont // 2: scale 0 for dataA, scale 1 for data B const int scales_num = 2; std::vector scales(scales_num, 1); - if (in_data[quantized_sum_enum::kDataA].dtype() != in_data[quantized_sum_enum::kDataB].dtype()) { + if (in_data[quantized_elemwise_add_enum::kDataA].dtype() + != in_data[quantized_elemwise_add_enum::kDataB].dtype()) { auto s8_pd = (is_dataA_int8 == true) ? dataA_mem->get_primitive_desc() : dataB_mem->get_primitive_desc(); @@ -155,26 +161,26 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont in_prims.push_back(*dataB_mem); in_pds.push_back(dataA_mem->get_primitive_desc()); in_pds.push_back(dataB_mem->get_primitive_desc()); - size_t i_ndim = in_data[quantized_sum_enum::kDataA].shape().ndim(); + size_t i_ndim = in_data[quantized_elemwise_add_enum::kDataA].shape().ndim(); mkldnn::memory::dims i_dims = mkldnn::memory::dims(i_ndim); for (size_t i = 0; i < i_ndim; i++) { - i_dims[i] = static_cast(in_data[quantized_sum_enum::kDataA].shape()[i]); + i_dims[i] = static_cast(in_data[quantized_elemwise_add_enum::kDataA].shape()[i]); } mkldnn::memory::format i_fmt = static_cast( - in_pds[quantized_sum_enum::kDataA].desc().data.format); + in_pds[quantized_elemwise_add_enum::kDataA].desc().data.format); auto output_desc = mkldnn::memory::desc(i_dims, output_data_type, i_fmt); mkldnn::sum::primitive_desc pdesc(output_desc, scales, in_pds); - auto mem = CreateMKLDNNMem(out_data[quantized_sum_enum::kOut], + auto mem = CreateMKLDNNMem(out_data[quantized_elemwise_add_enum::kOut], pdesc.dst_primitive_desc(), req[0], &in_data[0]); MKLDNNStream *stream = MKLDNNStream::Get(); stream->RegisterPrim(mkldnn::sum(pdesc, in_prims, *mem.second)); - CommitOutput(out_data[quantized_sum_enum::kOut], mem); + CommitOutput(out_data[quantized_elemwise_add_enum::kOut], mem); stream->Submit(); - out_data[quantized_sum_enum::kMin].data().dptr()[0] = output_min; - out_data[quantized_sum_enum::kMax].data().dptr()[0] = output_max; + out_data[quantized_elemwise_add_enum::kMin].data().dptr()[0] = output_min; + out_data[quantized_elemwise_add_enum::kMax].data().dptr()[0] = output_max; } inline static bool SumStorageType(const nnvm::NodeAttrs& attrs, const int dev_mask, @@ -188,7 +194,7 @@ inline static bool SumStorageType(const nnvm::NodeAttrs& attrs, const int dev_ma return MKLDNNStorageType(attrs, dev_mask, true, dispatch_mode, in_attrs, out_attrs); } -NNVM_REGISTER_OP(_contrib_quantized_sum) +NNVM_REGISTER_OP(_contrib_quantized_elemwise_add) .set_attr("FInferStorageType", SumStorageType) .set_attr("FComputeEx", MKLDNNQuantizedSumForward) .set_attr("TIsMKLDNN", true) diff --git a/src/operator/quantization/quantized_sum.cc b/src/operator/quantization/quantized_elemwise_add.cc similarity index 96% rename from src/operator/quantization/quantized_sum.cc rename to src/operator/quantization/quantized_elemwise_add.cc index a1de59c7cb08..2c302b220885 100644 --- a/src/operator/quantization/quantized_sum.cc +++ b/src/operator/quantization/quantized_elemwise_add.cc @@ -19,12 +19,12 @@ /*! * Copyright (c) 2019 by Contributors - * \file quantized_sum.cc + * \file quantized_elemwise_add.cc * \brief */ #include "../tensor/elemwise_unary_op.h" #if MXNET_USE_MKLDNN == 1 -#include "./mkldnn/mkldnn_quantized_sum-inl.h" +#include "./mkldnn/mkldnn_quantized_elemwise_add-inl.h" #endif namespace mxnet { @@ -92,7 +92,7 @@ void QuantizedSumForward(const nnvm::NodeAttrs& attrs, "Please install MKLDNN enabled MXNet."; } -NNVM_REGISTER_OP(_contrib_quantized_sum) +NNVM_REGISTER_OP(_contrib_quantized_elemwise_add) .describe(R"code(elemwise_add operator for input dataA and input dataB data type of int8, and accumulates in type int32 for the output. For each argument, two more arguments of type float32 must be provided representing the thresholds of quantizing argument from data @@ -130,7 +130,7 @@ and max thresholds representing the threholds for quantizing the float32 output NNVM_REGISTER_OP(elemwise_add) .set_attr("FQuantizedOp", [](const NodeAttrs& attrs) { nnvm::NodePtr node = nnvm::Node::Create(); - node->attrs.op = Op::Get("_contrib_quantized_sum"); + node->attrs.op = Op::Get("_contrib_quantized_elemwise_add"); node->attrs.name = "quantized_" + attrs.name; node->attrs.dict = attrs.dict; if (node->op()->attr_parser != nullptr) { diff --git a/src/operator/subgraph/mkldnn/mkldnn_post_quantize_property.h b/src/operator/subgraph/mkldnn/mkldnn_post_quantize_property.h index 4aede396661c..b61a303757b3 100644 --- a/src/operator/subgraph/mkldnn/mkldnn_post_quantize_property.h +++ b/src/operator/subgraph/mkldnn/mkldnn_post_quantize_property.h @@ -49,7 +49,7 @@ class SgMKLDNNPostQuantizeSelector : public SubgraphSelector { public: SgMKLDNNPostQuantizeSelector() { support_requantize_fusion_op_name.insert("_sg_mkldnn_conv"); - support_requantize_fusion_op_name.insert("_contrib_quantized_sum"); + support_requantize_fusion_op_name.insert("_contrib_quantized_elemwise_add"); } bool Select(const nnvm::Node &n) override { @@ -62,7 +62,7 @@ class SgMKLDNNPostQuantizeSelector : public SubgraphSelector { matched_list.push_back(&n); return true; } - } else if (n.op()->name == "_contrib_quantized_sum") { + } else if (n.op()->name == "_contrib_quantized_elemwise_add") { status = kStart; matched_list.clear(); matched_list.push_back(&n); @@ -113,7 +113,7 @@ class SgMKLDNNPostQuantizeProperty : public SubgraphProperty { public: SgMKLDNNPostQuantizeProperty() { support_requantize_fusion_op_name.insert("_sg_mkldnn_conv"); - support_requantize_fusion_op_name.insert("_contrib_quantized_sum"); + support_requantize_fusion_op_name.insert("_contrib_quantized_elemwise_add"); } static SubgraphPropertyPtr Create() { static const std::string &name = "MKLDNN post-quantization optimization pass"; diff --git a/tests/python/quantization/test_quantization.py b/tests/python/quantization/test_quantization.py index e185feba2788..d61acce674e0 100644 --- a/tests/python/quantization/test_quantization.py +++ b/tests/python/quantization/test_quantization.py @@ -215,23 +215,23 @@ def check_quantized_conv(data_shape, kernel, num_filter, pad, stride, no_bias, q @with_seed() -def test_quantized_sum(): - def check_quantized_sum(data_shape, qtype): +def test_quantized_elemwise_add(): + def check_quantized_elemwise_add(data_shape, qtype): if is_test_for_native_cpu(): - print('skipped testing quantized_sum for native cpu since it is not supported yet') + print('skipped testing quantized_elemwise_add for native cpu since it is not supported yet') return elif qtype != 'uint8' and qtype != 'int8': - print('skipped testing quantized_sum for not supported data type') + print('skipped testing quantized_elemwise_add for not supported data type') return elif is_test_for_gpu(): - print('skipped testing quantized_sum for gpu since it is not supported yet') + print('skipped testing quantized_elemwise_add for gpu since it is not supported yet') return dataA = mx.sym.Variable(name='dataA', shape=data_shape, dtype='float32') dataB = mx.sym.Variable(name='dataB', shape=data_shape, dtype='float32') - sum_fp32 = mx.sym.elemwise_add(dataA, dataB) - arg_names = sum_fp32.list_arguments() - sum_fp32_exe = sum_fp32.simple_bind(ctx=mx.current_context(), grad_req='null') + elemwise_add_fp32 = mx.sym.elemwise_add(dataA, dataB) + arg_names = elemwise_add_fp32.list_arguments() + elemwise_add_fp32_exe = elemwise_add_fp32.simple_bind(ctx=mx.current_context(), grad_req='null') if qtype == 'uint8': data_low = 0.0 data_high = 255.0 @@ -241,11 +241,11 @@ def check_quantized_sum(data_shape, qtype): dataA_val = mx.nd.random.uniform(low=data_low, high=data_high, shape=data_shape).astype('int32') dataB_val = mx.nd.random.uniform(low=data_low, high=data_high, shape=data_shape).astype('int32') - sum_fp32_exe.arg_dict[arg_names[0]][:] = dataA_val + elemwise_add_fp32_exe.arg_dict[arg_names[0]][:] = dataA_val - sum_fp32_exe.arg_dict[arg_names[1]][:] = dataB_val + elemwise_add_fp32_exe.arg_dict[arg_names[1]][:] = dataB_val - output = sum_fp32_exe.forward()[0] + output = elemwise_add_fp32_exe.forward()[0] qdataA = mx.sym.Variable(name='qdataA', shape=data_shape, dtype=qtype) qdataB = mx.sym.Variable(name='qdataB', shape=data_shape, dtype=qtype) @@ -253,17 +253,17 @@ def check_quantized_sum(data_shape, qtype): max_dataA = mx.sym.Variable(name='max_dataA') min_dataB = mx.sym.Variable(name='min_dataB') max_dataB = mx.sym.Variable(name='max_dataB') - quantized_sum = mx.sym.contrib.quantized_sum(qdataA, qdataB, min_dataA, max_dataA, min_dataB, max_dataB) - sum_int8_exe = quantized_sum.simple_bind(ctx=mx.current_context(), grad_req='null') - qarg_names = quantized_sum.list_arguments() - sum_int8_exe.arg_dict[qarg_names[0]][:] = sum_fp32_exe.arg_dict[arg_names[0]].astype(qtype) - sum_int8_exe.arg_dict[qarg_names[1]][:] = sum_fp32_exe.arg_dict[arg_names[1]].astype(qtype) + quantized_elemwise_add = mx.sym.contrib.quantized_elemwise_add(qdataA, qdataB, min_dataA, max_dataA, min_dataB, max_dataB) + elemwise_add_int8_exe = quantized_elemwise_add.simple_bind(ctx=mx.current_context(), grad_req='null') + qarg_names = quantized_elemwise_add.list_arguments() + elemwise_add_int8_exe.arg_dict[qarg_names[0]][:] = elemwise_add_fp32_exe.arg_dict[arg_names[0]].astype(qtype) + elemwise_add_int8_exe.arg_dict[qarg_names[1]][:] = elemwise_add_fp32_exe.arg_dict[arg_names[1]].astype(qtype) quantized_range = 127.0 - sum_int8_exe.arg_dict[qarg_names[2]][:] = data_low - sum_int8_exe.arg_dict[qarg_names[3]][:] = data_high - sum_int8_exe.arg_dict[qarg_names[4]][:] = data_low - sum_int8_exe.arg_dict[qarg_names[5]][:] = data_high - qoutput, min_range, max_range = sum_int8_exe.forward() + elemwise_add_int8_exe.arg_dict[qarg_names[2]][:] = data_low + elemwise_add_int8_exe.arg_dict[qarg_names[3]][:] = data_high + elemwise_add_int8_exe.arg_dict[qarg_names[4]][:] = data_low + elemwise_add_int8_exe.arg_dict[qarg_names[5]][:] = data_high + qoutput, min_range, max_range = elemwise_add_int8_exe.forward() min_val = min_range.asnumpy().tolist()[0] max_val = max_range.asnumpy().tolist()[0] @@ -272,10 +272,10 @@ def check_quantized_sum(data_shape, qtype): assert_almost_equal(int8_rslt, int8_rslt, atol = 1) for qtype in ['int8', 'uint8']: - check_quantized_sum((4, 6), qtype) - check_quantized_sum((13, 74, 52), qtype) - check_quantized_sum((3, 4, 56, 56), qtype) - check_quantized_sum((32, 56, 64, 11), qtype) + check_quantized_elemwise_add((4, 6), qtype) + check_quantized_elemwise_add((13, 74, 52), qtype) + check_quantized_elemwise_add((3, 4, 56, 56), qtype) + check_quantized_elemwise_add((32, 56, 64, 11), qtype) @with_seed() From f1560053460582ce310b2a397e4d8ee096e7a7d3 Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Thu, 18 Apr 2019 14:59:12 +0800 Subject: [PATCH 11/23] change Sum to ElemwiseAdd --- .../mkldnn/mkldnn_quantized_elemwise_add-inl.h | 8 ++++---- .../mkldnn/mkldnn_quantized_elemwise_add.cc | 16 ++++++++-------- .../quantization/quantized_elemwise_add.cc | 14 +++++++------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add-inl.h b/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add-inl.h index 1a5250690c41..c2b24e2907b3 100644 --- a/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add-inl.h +++ b/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add-inl.h @@ -31,10 +31,10 @@ namespace mxnet { namespace op { -struct RequantizeSumParam : public dmlc::Parameter { +struct RequantizeElemwiseAddParam : public dmlc::Parameter { dmlc::optional min_calib_range; dmlc::optional max_calib_range; - DMLC_DECLARE_PARAMETER(RequantizeSumParam) { + DMLC_DECLARE_PARAMETER(RequantizeElemwiseAddParam) { DMLC_DECLARE_FIELD(min_calib_range) .set_default(dmlc::optional()) .describe("The minimum scalar value in the form of float32 obtained " @@ -49,8 +49,8 @@ struct RequantizeSumParam : public dmlc::Parameter { }; namespace quantized_elemwise_add_enum { -enum QuantizedSumOutputs { kOut, kMin, kMax }; -enum QuantizedSumInputs { kDataA, kDataB, kAMin, kAMax, kBMin, kBMax}; +enum QuantizedElemwiseAddOutputs { kOut, kMin, kMax }; +enum QuantizedElemwiseAddInputs { kDataA, kDataB, kAMin, kAMax, kBMin, kBMax}; } } // namespace op diff --git a/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc b/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc index ce89481f0d92..32affb1b11b5 100644 --- a/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc +++ b/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc @@ -32,18 +32,18 @@ namespace mxnet { namespace op { -DMLC_REGISTER_PARAMETER(RequantizeSumParam); +DMLC_REGISTER_PARAMETER(RequantizeElemwiseAddParam); static inline float GetScale(const NDArray& data, float min, float max) { auto data_range = (data.dtype() == mshadow::kInt8) ? kInt8Range : kUint8Range; return data_range / MaxAbs(min, max); } -static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpContext& ctx, +static void MKLDNNQuantizedElemwiseAddForward(const nnvm::NodeAttrs& attrs, const OpContext& ctx, const std::vector& in_data, const std::vector& req, const std::vector& out_data) { - const RequantizeSumParam& params = nnvm::get(attrs.parsed); + const RequantizeElemwiseAddParam& params = nnvm::get(attrs.parsed); // A, B, A_min, A_max, B_min, B_max CHECK_EQ(in_data.size(), 6U); // C, C_min, C_max @@ -183,7 +183,7 @@ static void MKLDNNQuantizedSumForward(const nnvm::NodeAttrs& attrs, const OpCont out_data[quantized_elemwise_add_enum::kMax].data().dptr()[0] = output_max; } -inline static bool SumStorageType(const nnvm::NodeAttrs& attrs, const int dev_mask, +inline static bool ElemwiseAddStorageType(const nnvm::NodeAttrs& attrs, const int dev_mask, DispatchMode* dispatch_mode, std::vector* in_attrs, std::vector* out_attrs) { // A, B, A_min, A_max, B_min, B_max @@ -195,11 +195,11 @@ inline static bool SumStorageType(const nnvm::NodeAttrs& attrs, const int dev_ma } NNVM_REGISTER_OP(_contrib_quantized_elemwise_add) -.set_attr("FInferStorageType", SumStorageType) -.set_attr("FComputeEx", MKLDNNQuantizedSumForward) +.set_attr("FInferStorageType", ElemwiseAddStorageType) +.set_attr("FComputeEx", MKLDNNQuantizedElemwiseAddForward) .set_attr("TIsMKLDNN", true) -.set_attr_parser(ParamParser) -.add_arguments(RequantizeSumParam::__FIELDS__()); +.set_attr_parser(ParamParser) +.add_arguments(RequantizeElemwiseAddParam::__FIELDS__()); } // namespace op } // namespace mxnet diff --git a/src/operator/quantization/quantized_elemwise_add.cc b/src/operator/quantization/quantized_elemwise_add.cc index 2c302b220885..0521f83dbaeb 100644 --- a/src/operator/quantization/quantized_elemwise_add.cc +++ b/src/operator/quantization/quantized_elemwise_add.cc @@ -30,7 +30,7 @@ namespace mxnet { namespace op { -static bool SumShape(const nnvm::NodeAttrs& attrs, mxnet::ShapeVector* in_shape, +static bool ElemwiseAddShape(const nnvm::NodeAttrs& attrs, mxnet::ShapeVector* in_shape, mxnet::ShapeVector* out_shape) { // A, B, A_min, A_max, B_min, B_max CHECK_EQ(in_shape->size(), 6U); @@ -50,7 +50,7 @@ static bool SumShape(const nnvm::NodeAttrs& attrs, mxnet::ShapeVector* in_shape return true; } -static bool SumType(const nnvm::NodeAttrs& attrs, std::vector* in_type, +static bool ElemwiseAddType(const nnvm::NodeAttrs& attrs, std::vector* in_type, std::vector* out_type) { // A, B, A_min, A_max, B_min, B_max CHECK_EQ(in_type->size(), 6U); @@ -69,7 +69,7 @@ static bool SumType(const nnvm::NodeAttrs& attrs, std::vector* in_type, // C int dtype = mshadow::kInt32; #if MXNET_USE_MKLDNN == 1 - const RequantizeSumParam& params = nnvm::get(attrs.parsed); + const RequantizeElemwiseAddParam& params = nnvm::get(attrs.parsed); if (params.max_calib_range.has_value() && params.min_calib_range.has_value()) { dtype = (in_type->at(0) == in_type->at(1)) ? in_type->at(0) : mshadow::kInt8; } @@ -83,7 +83,7 @@ static bool SumType(const nnvm::NodeAttrs& attrs, std::vector* in_type, return true; } -void QuantizedSumForward(const nnvm::NodeAttrs& attrs, +void QuantizedElemwiseAddForward(const nnvm::NodeAttrs& attrs, const OpContext &ctx, const std::vector &in_data, const std::vector &req, @@ -115,9 +115,9 @@ and max thresholds representing the threholds for quantizing the float32 output .set_attr("FListOutputNames", [](const NodeAttrs& attrs) { return std::vector{"output", "min_output", "max_output"}; }) -.set_attr("FInferType", SumType) -.set_attr("FInferShape", SumShape) -.set_attr("FCompute", QuantizedSumForward) +.set_attr("FInferType", ElemwiseAddType) +.set_attr("FInferShape", ElemwiseAddShape) +.set_attr("FCompute", QuantizedElemwiseAddForward) .set_attr("FNeedRequantize", [](const NodeAttrs& attrs) { return true; }) .add_argument("lhs", "NDArray-or-Symbol", "first input") .add_argument("rhs", "NDArray-or-Symbol", "4th input") From 55b01039fcc5d2380607555af3b3003d8ade8176 Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Thu, 18 Apr 2019 21:06:32 +0800 Subject: [PATCH 12/23] fix indents --- .../mkldnn/mkldnn_quantized_elemwise_add.cc | 40 +++++++++---------- .../quantization/quantized_elemwise_add.cc | 18 +++++---- .../python/quantization/test_quantization.py | 2 +- 3 files changed, 31 insertions(+), 29 deletions(-) diff --git a/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc b/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc index 32affb1b11b5..4a97b18b3ef4 100644 --- a/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc +++ b/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc @@ -40,9 +40,9 @@ static inline float GetScale(const NDArray& data, float min, float max) { } static void MKLDNNQuantizedElemwiseAddForward(const nnvm::NodeAttrs& attrs, const OpContext& ctx, - const std::vector& in_data, - const std::vector& req, - const std::vector& out_data) { + const std::vector& in_data, + const std::vector& req, + const std::vector& out_data) { const RequantizeElemwiseAddParam& params = nnvm::get(attrs.parsed); // A, B, A_min, A_max, B_min, B_max CHECK_EQ(in_data.size(), 6U); @@ -92,7 +92,7 @@ static void MKLDNNQuantizedElemwiseAddForward(const nnvm::NodeAttrs& attrs, cons if (params.max_calib_range.has_value() && params.min_calib_range.has_value()) { output_min = params.min_calib_range.value(); output_max = params.max_calib_range.value(); - out_data_scale = output_data_range/MaxAbs(output_min, output_max); + out_data_scale = output_data_range / MaxAbs(output_min, output_max); } else { output_max = dataA_absmax + dataB_absmax; output_min = -output_max; @@ -109,24 +109,24 @@ static void MKLDNNQuantizedElemwiseAddForward(const nnvm::NodeAttrs& attrs, cons float u8_reorder_scale = 0; if (params.max_calib_range.has_value() && params.min_calib_range.has_value()) { if (is_dataA_int8 == true) { - u8_reorder_scale = out_data_scale/B_scale; - scales[0] = out_data_scale/A_scale; + u8_reorder_scale = out_data_scale / B_scale; + scales[0] = out_data_scale / A_scale; } else { - u8_reorder_scale = out_data_scale/A_scale; - scales[1] = out_data_scale/B_scale; + u8_reorder_scale = out_data_scale / A_scale; + scales[1] = out_data_scale / B_scale; } } else { // x*dataA_absmax/dataA_range = y*(dataA_absmax+dataB_absmax)/output_range if (is_dataA_int8 == true) { u8_reorder_scale = dataB_absmax*output_data_range - /((dataA_absmax + dataB_absmax)*kUint8Range); + / ((dataA_absmax + dataB_absmax)*kUint8Range); scales[0] = dataA_absmax*output_data_range - /((dataA_absmax + dataB_absmax)*dataA_range); + / ((dataA_absmax + dataB_absmax)*dataA_range); } else { u8_reorder_scale = dataA_absmax*output_data_range - /((dataA_absmax + dataB_absmax)*dataA_range); + / ((dataA_absmax + dataB_absmax)*dataA_range); scales[1] = dataB_absmax*output_data_range - /((dataA_absmax + dataB_absmax)*kInt8Range); + / ((dataA_absmax + dataB_absmax)*kInt8Range); } } std::vector reorder_scale = {u8_reorder_scale}; @@ -147,11 +147,11 @@ static void MKLDNNQuantizedElemwiseAddForward(const nnvm::NodeAttrs& attrs, cons } else { // same data type and has same data range if (params.max_calib_range.has_value() && params.min_calib_range.has_value()) { - scales[0] = out_data_scale/A_scale; - scales[1] = out_data_scale/B_scale; + scales[0] = out_data_scale / A_scale; + scales[1] = out_data_scale / B_scale; } else { - scales[0] = dataA_absmax*output_data_range/((dataA_absmax + dataB_absmax)*dataA_range); - scales[1] = dataB_absmax*output_data_range/((dataA_absmax + dataB_absmax)*dataA_range); + scales[0] = dataA_absmax*output_data_range / ((dataA_absmax + dataB_absmax)*dataA_range); + scales[1] = dataB_absmax*output_data_range / ((dataA_absmax + dataB_absmax)*dataA_range); } } @@ -184,11 +184,11 @@ static void MKLDNNQuantizedElemwiseAddForward(const nnvm::NodeAttrs& attrs, cons } inline static bool ElemwiseAddStorageType(const nnvm::NodeAttrs& attrs, const int dev_mask, - DispatchMode* dispatch_mode, std::vector* in_attrs, - std::vector* out_attrs) { - // A, B, A_min, A_max, B_min, B_max + DispatchMode* dispatch_mode, std::vector* in_attrs, + std::vector* out_attrs) { + // Check num of inputs: A, B, A_min, A_max, B_min, B_max CHECK_EQ(in_attrs->size(), 6U); - // C, C_min, C_max + // Check num of outputs: C, C_min, C_max CHECK_EQ(out_attrs->size(), 3U); return MKLDNNStorageType(attrs, dev_mask, true, dispatch_mode, in_attrs, out_attrs); diff --git a/src/operator/quantization/quantized_elemwise_add.cc b/src/operator/quantization/quantized_elemwise_add.cc index 0521f83dbaeb..a31f062463e7 100644 --- a/src/operator/quantization/quantized_elemwise_add.cc +++ b/src/operator/quantization/quantized_elemwise_add.cc @@ -30,8 +30,9 @@ namespace mxnet { namespace op { -static bool ElemwiseAddShape(const nnvm::NodeAttrs& attrs, mxnet::ShapeVector* in_shape, - mxnet::ShapeVector* out_shape) { +static bool ElemwiseAddShape(const nnvm::NodeAttrs& attrs, + mxnet::ShapeVector* in_shape, + mxnet::ShapeVector* out_shape) { // A, B, A_min, A_max, B_min, B_max CHECK_EQ(in_shape->size(), 6U); // C, C_min, C_max @@ -50,8 +51,9 @@ static bool ElemwiseAddShape(const nnvm::NodeAttrs& attrs, mxnet::ShapeVector* return true; } -static bool ElemwiseAddType(const nnvm::NodeAttrs& attrs, std::vector* in_type, - std::vector* out_type) { +static bool ElemwiseAddType(const nnvm::NodeAttrs& attrs, + std::vector* in_type, + std::vector* out_type) { // A, B, A_min, A_max, B_min, B_max CHECK_EQ(in_type->size(), 6U); // C, C_min, C_max @@ -84,10 +86,10 @@ static bool ElemwiseAddType(const nnvm::NodeAttrs& attrs, std::vector* in_t } void QuantizedElemwiseAddForward(const nnvm::NodeAttrs& attrs, - const OpContext &ctx, - const std::vector &in_data, - const std::vector &req, - const std::vector &out_data) { + const OpContext &ctx, + const std::vector &in_data, + const std::vector &req, + const std::vector &out_data) { LOG(FATAL) << "Not supported for MXNet built without MKLDNN. " "Please install MKLDNN enabled MXNet."; } diff --git a/tests/python/quantization/test_quantization.py b/tests/python/quantization/test_quantization.py index d61acce674e0..36c1599d86f1 100644 --- a/tests/python/quantization/test_quantization.py +++ b/tests/python/quantization/test_quantization.py @@ -269,7 +269,7 @@ def check_quantized_elemwise_add(data_shape, qtype): fp32_rslt = output.asnumpy() int8_rslt = qoutput.asnumpy()*max_val/0x7fffffff - assert_almost_equal(int8_rslt, int8_rslt, atol = 1) + assert_almost_equal(int8_rslt, int8_rslt, atol = 1e-4) for qtype in ['int8', 'uint8']: check_quantized_elemwise_add((4, 6), qtype) From 3a794c4e0693fc8665152f006fee17c0acdc8dbf Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Fri, 19 Apr 2019 07:55:12 +0800 Subject: [PATCH 13/23] retrigger CI From 4ddf2c76ff57bf1766d744cd97521c501c97b656 Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Tue, 23 Apr 2019 10:43:23 +0800 Subject: [PATCH 14/23] trigger CI From 89c30a3dec65c779917c60b7e5cb014fa76d03f0 Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Thu, 25 Apr 2019 09:38:17 +0800 Subject: [PATCH 15/23] fix indentation and typo --- .../mkldnn/mkldnn_quantized_elemwise_add.cc | 14 +++++++------- .../quantization/quantized_elemwise_add.cc | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc b/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc index 4a97b18b3ef4..193d4f5dfd92 100644 --- a/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc +++ b/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc @@ -45,9 +45,9 @@ static void MKLDNNQuantizedElemwiseAddForward(const nnvm::NodeAttrs& attrs, cons const std::vector& out_data) { const RequantizeElemwiseAddParam& params = nnvm::get(attrs.parsed); // A, B, A_min, A_max, B_min, B_max - CHECK_EQ(in_data.size(), 6U); + CHECK_EQ(in_data.size(), 6U) << "should be A, B, A_min, A_max, B_min, B_max"; // C, C_min, C_max - CHECK_EQ(out_data.size(), 3U); + CHECK_EQ(out_data.size(), 3U) << "should be C, C_min, C_max"; // Collect data min,max,absmax const float dataA_min = in_data[quantized_elemwise_add_enum::kAMin].data().dptr()[0]; const float dataB_min = in_data[quantized_elemwise_add_enum::kBMin].data().dptr()[0]; @@ -118,14 +118,14 @@ static void MKLDNNQuantizedElemwiseAddForward(const nnvm::NodeAttrs& attrs, cons } else { // x*dataA_absmax/dataA_range = y*(dataA_absmax+dataB_absmax)/output_range if (is_dataA_int8 == true) { - u8_reorder_scale = dataB_absmax*output_data_range + u8_reorder_scale = dataB_absmax * output_data_range / ((dataA_absmax + dataB_absmax)*kUint8Range); scales[0] = dataA_absmax*output_data_range / ((dataA_absmax + dataB_absmax)*dataA_range); } else { - u8_reorder_scale = dataA_absmax*output_data_range + u8_reorder_scale = dataA_absmax * output_data_range / ((dataA_absmax + dataB_absmax)*dataA_range); - scales[1] = dataB_absmax*output_data_range + scales[1] = dataB_absmax * output_data_range / ((dataA_absmax + dataB_absmax)*kInt8Range); } } @@ -150,8 +150,8 @@ static void MKLDNNQuantizedElemwiseAddForward(const nnvm::NodeAttrs& attrs, cons scales[0] = out_data_scale / A_scale; scales[1] = out_data_scale / B_scale; } else { - scales[0] = dataA_absmax*output_data_range / ((dataA_absmax + dataB_absmax)*dataA_range); - scales[1] = dataB_absmax*output_data_range / ((dataA_absmax + dataB_absmax)*dataA_range); + scales[0] = dataA_absmax * output_data_range / ((dataA_absmax + dataB_absmax) * dataA_range); + scales[1] = dataB_absmax * output_data_range / ((dataA_absmax + dataB_absmax) * dataA_range); } } diff --git a/src/operator/quantization/quantized_elemwise_add.cc b/src/operator/quantization/quantized_elemwise_add.cc index a31f062463e7..7027f49434ab 100644 --- a/src/operator/quantization/quantized_elemwise_add.cc +++ b/src/operator/quantization/quantized_elemwise_add.cc @@ -122,7 +122,7 @@ and max thresholds representing the threholds for quantizing the float32 output .set_attr("FCompute", QuantizedElemwiseAddForward) .set_attr("FNeedRequantize", [](const NodeAttrs& attrs) { return true; }) .add_argument("lhs", "NDArray-or-Symbol", "first input") -.add_argument("rhs", "NDArray-or-Symbol", "4th input") +.add_argument("rhs", "NDArray-or-Symbol", "second input") .add_argument("lhs_min", "NDArray-or-Symbol", "second input") .add_argument("lhs_max", "NDArray-or-Symbol", "third input") .add_argument("rhs_min", "NDArray-or-Symbol", "5th input") From 9cb8bbe64477cfb6132494ae2fefe45ee7cda548 Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Fri, 26 Apr 2019 09:45:54 +0800 Subject: [PATCH 16/23] trigger CI From e55b27b1b42985fecbf06068d1ac61d0e678c8df Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Fri, 26 Apr 2019 13:32:24 +0800 Subject: [PATCH 17/23] fix typo --- .../mkldnn/mkldnn_quantized_elemwise_add.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc b/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc index 193d4f5dfd92..2b6a39a99d9e 100644 --- a/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc +++ b/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc @@ -119,14 +119,14 @@ static void MKLDNNQuantizedElemwiseAddForward(const nnvm::NodeAttrs& attrs, cons // x*dataA_absmax/dataA_range = y*(dataA_absmax+dataB_absmax)/output_range if (is_dataA_int8 == true) { u8_reorder_scale = dataB_absmax * output_data_range - / ((dataA_absmax + dataB_absmax)*kUint8Range); - scales[0] = dataA_absmax*output_data_range - / ((dataA_absmax + dataB_absmax)*dataA_range); + / ((dataA_absmax + dataB_absmax) * kUint8Range); + scales[0] = dataA_absmax * output_data_range + / ((dataA_absmax + dataB_absmax) * dataA_range); } else { u8_reorder_scale = dataA_absmax * output_data_range - / ((dataA_absmax + dataB_absmax)*dataA_range); + / ((dataA_absmax + dataB_absmax) * dataA_range); scales[1] = dataB_absmax * output_data_range - / ((dataA_absmax + dataB_absmax)*kInt8Range); + / ((dataA_absmax + dataB_absmax) * kInt8Range); } } std::vector reorder_scale = {u8_reorder_scale}; From fa3d1e48ecd99c38c90f0685b75ea4363a4f3e52 Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Fri, 26 Apr 2019 15:12:32 +0800 Subject: [PATCH 18/23] fix typo --- src/operator/quantization/quantized_elemwise_add.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/operator/quantization/quantized_elemwise_add.cc b/src/operator/quantization/quantized_elemwise_add.cc index 7027f49434ab..1ddc8182a6a4 100644 --- a/src/operator/quantization/quantized_elemwise_add.cc +++ b/src/operator/quantization/quantized_elemwise_add.cc @@ -123,8 +123,8 @@ and max thresholds representing the threholds for quantizing the float32 output .set_attr("FNeedRequantize", [](const NodeAttrs& attrs) { return true; }) .add_argument("lhs", "NDArray-or-Symbol", "first input") .add_argument("rhs", "NDArray-or-Symbol", "second input") -.add_argument("lhs_min", "NDArray-or-Symbol", "second input") -.add_argument("lhs_max", "NDArray-or-Symbol", "third input") +.add_argument("lhs_min", "NDArray-or-Symbol", "3rd input") +.add_argument("lhs_max", "NDArray-or-Symbol", "4th input") .add_argument("rhs_min", "NDArray-or-Symbol", "5th input") .add_argument("rhs_max", "NDArray-or-Symbol", "6th input"); From 11cd34ae09361a6c423e0657afa7dcc2d550d745 Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Sun, 28 Apr 2019 10:15:41 +0800 Subject: [PATCH 19/23] remove USE_MKLDNN macro for requantize params --- .../mkldnn/mkldnn_quantized_elemwise_add.cc | 2 +- ...emwise_add-inl.h => quantized_elemwise_add-inl.h} | 12 +++++------- src/operator/quantization/quantized_elemwise_add.cc | 6 +----- 3 files changed, 7 insertions(+), 13 deletions(-) rename src/operator/quantization/{mkldnn/mkldnn_quantized_elemwise_add-inl.h => quantized_elemwise_add-inl.h} (82%) diff --git a/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc b/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc index 2b6a39a99d9e..f4bbe3351ef1 100644 --- a/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc +++ b/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc @@ -24,7 +24,7 @@ */ #if MXNET_USE_MKLDNN == 1 -#include "./mkldnn_quantized_elemwise_add-inl.h" +#include "../quantized_elemwise_add-inl.h" #include "../../nn/mkldnn/mkldnn_ops-inl.h" #include "../../nn/mkldnn/mkldnn_base-inl.h" #include "../quantization_utils.h" diff --git a/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add-inl.h b/src/operator/quantization/quantized_elemwise_add-inl.h similarity index 82% rename from src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add-inl.h rename to src/operator/quantization/quantized_elemwise_add-inl.h index c2b24e2907b3..4559719549a8 100644 --- a/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add-inl.h +++ b/src/operator/quantization/quantized_elemwise_add-inl.h @@ -17,16 +17,15 @@ * under the License. */ /*! - * \file mkldnn_quantized_elemwise_add-inl.h + * \file quantized_elemwise_add-inl.h * \brief * \author Rong Zhang */ -#ifndef MXNET_OPERATOR_QUANTIZATION_MKLDNN_MKLDNN_QUANTIZED_ELEMWISE_ADD_INL_H_ -#define MXNET_OPERATOR_QUANTIZATION_MKLDNN_MKLDNN_QUANTIZED_ELEMWISE_ADD_INL_H_ -#if MXNET_USE_MKLDNN == 1 +#ifndef MXNET_OPERATOR_QUANTIZATION_QUANTIZED_ELEMWISE_ADD_INL_H_ +#define MXNET_OPERATOR_QUANTIZATION_QUANTIZED_ELEMWISE_ADD_INL_H_ -#include "../../tensor/elemwise_unary_op.h" +#include "../tensor/elemwise_unary_op.h" namespace mxnet { namespace op { @@ -56,5 +55,4 @@ enum QuantizedElemwiseAddInputs { kDataA, kDataB, kAMin, kAMax, kBMin, kBMax}; } // namespace op } // namespace mxnet -#endif // MXNET_USE_MKLDNN == 1 -#endif // MXNET_OPERATOR_QUANTIZATION_MKLDNN_MKLDNN_QUANTIZED_ELEMWISE_ADD_INL_H_ +#endif // MXNET_OPERATOR_QUANTIZATION_QUANTIZED_ELEMWISE_ADD_INL_H_ diff --git a/src/operator/quantization/quantized_elemwise_add.cc b/src/operator/quantization/quantized_elemwise_add.cc index 1ddc8182a6a4..19baf07083e3 100644 --- a/src/operator/quantization/quantized_elemwise_add.cc +++ b/src/operator/quantization/quantized_elemwise_add.cc @@ -23,9 +23,7 @@ * \brief */ #include "../tensor/elemwise_unary_op.h" -#if MXNET_USE_MKLDNN == 1 -#include "./mkldnn/mkldnn_quantized_elemwise_add-inl.h" -#endif +#include "./quantized_elemwise_add-inl.h" namespace mxnet { namespace op { @@ -70,12 +68,10 @@ static bool ElemwiseAddType(const nnvm::NodeAttrs& attrs, } // C int dtype = mshadow::kInt32; -#if MXNET_USE_MKLDNN == 1 const RequantizeElemwiseAddParam& params = nnvm::get(attrs.parsed); if (params.max_calib_range.has_value() && params.min_calib_range.has_value()) { dtype = (in_type->at(0) == in_type->at(1)) ? in_type->at(0) : mshadow::kInt8; } -#endif TYPE_ASSIGN_CHECK(*out_type, 0, dtype); // C_min TYPE_ASSIGN_CHECK(*out_type, 1, mshadow::kFloat32); From c18eeec0cf51233b94d3eb84d4ea0be455b0be94 Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Sun, 28 Apr 2019 14:12:52 +0800 Subject: [PATCH 20/23] rename param same as its op --- .../quantization/mkldnn/mkldnn_quantized_elemwise_add.cc | 8 ++++---- src/operator/quantization/quantized_elemwise_add-inl.h | 6 +++--- src/operator/quantization/quantized_elemwise_add.cc | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc b/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc index f4bbe3351ef1..05da99207651 100644 --- a/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc +++ b/src/operator/quantization/mkldnn/mkldnn_quantized_elemwise_add.cc @@ -32,7 +32,7 @@ namespace mxnet { namespace op { -DMLC_REGISTER_PARAMETER(RequantizeElemwiseAddParam); +DMLC_REGISTER_PARAMETER(QuantizeElemwiseAddParam); static inline float GetScale(const NDArray& data, float min, float max) { auto data_range = (data.dtype() == mshadow::kInt8) ? kInt8Range : kUint8Range; @@ -43,7 +43,7 @@ static void MKLDNNQuantizedElemwiseAddForward(const nnvm::NodeAttrs& attrs, cons const std::vector& in_data, const std::vector& req, const std::vector& out_data) { - const RequantizeElemwiseAddParam& params = nnvm::get(attrs.parsed); + const QuantizeElemwiseAddParam& params = nnvm::get(attrs.parsed); // A, B, A_min, A_max, B_min, B_max CHECK_EQ(in_data.size(), 6U) << "should be A, B, A_min, A_max, B_min, B_max"; // C, C_min, C_max @@ -198,8 +198,8 @@ NNVM_REGISTER_OP(_contrib_quantized_elemwise_add) .set_attr("FInferStorageType", ElemwiseAddStorageType) .set_attr("FComputeEx", MKLDNNQuantizedElemwiseAddForward) .set_attr("TIsMKLDNN", true) -.set_attr_parser(ParamParser) -.add_arguments(RequantizeElemwiseAddParam::__FIELDS__()); +.set_attr_parser(ParamParser) +.add_arguments(QuantizeElemwiseAddParam::__FIELDS__()); } // namespace op } // namespace mxnet diff --git a/src/operator/quantization/quantized_elemwise_add-inl.h b/src/operator/quantization/quantized_elemwise_add-inl.h index 4559719549a8..673b281d6cdc 100644 --- a/src/operator/quantization/quantized_elemwise_add-inl.h +++ b/src/operator/quantization/quantized_elemwise_add-inl.h @@ -29,11 +29,11 @@ namespace mxnet { namespace op { - -struct RequantizeElemwiseAddParam : public dmlc::Parameter { +/* These structure is used for requantization only when fusion */ +struct QuantizeElemwiseAddParam : public dmlc::Parameter { dmlc::optional min_calib_range; dmlc::optional max_calib_range; - DMLC_DECLARE_PARAMETER(RequantizeElemwiseAddParam) { + DMLC_DECLARE_PARAMETER(QuantizeElemwiseAddParam) { DMLC_DECLARE_FIELD(min_calib_range) .set_default(dmlc::optional()) .describe("The minimum scalar value in the form of float32 obtained " diff --git a/src/operator/quantization/quantized_elemwise_add.cc b/src/operator/quantization/quantized_elemwise_add.cc index 19baf07083e3..f821e6598192 100644 --- a/src/operator/quantization/quantized_elemwise_add.cc +++ b/src/operator/quantization/quantized_elemwise_add.cc @@ -68,7 +68,7 @@ static bool ElemwiseAddType(const nnvm::NodeAttrs& attrs, } // C int dtype = mshadow::kInt32; - const RequantizeElemwiseAddParam& params = nnvm::get(attrs.parsed); + const QuantizeElemwiseAddParam& params = nnvm::get(attrs.parsed); if (params.max_calib_range.has_value() && params.min_calib_range.has_value()) { dtype = (in_type->at(0) == in_type->at(1)) ? in_type->at(0) : mshadow::kInt8; } From 34bec4d92247721ac9790a628d2c5714594e7181 Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Mon, 29 Apr 2019 21:25:36 +0800 Subject: [PATCH 21/23] trigger CI From 440a7a5f40b192e7ba6673158d7735481c74a6a5 Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Tue, 30 Apr 2019 14:45:52 +0800 Subject: [PATCH 22/23] trigger CI From 3e6762e23c592a0b57e597c7c0c9085bc532e915 Mon Sep 17 00:00:00 2001 From: rongzha1 Date: Tue, 30 Apr 2019 16:45:40 +0800 Subject: [PATCH 23/23] trigger CI