From f39fa747ad6e9af438c612ca1b0e3a9dde5969ab Mon Sep 17 00:00:00 2001
From: Zhennan Qin <zhennan.qin@intel.com>
Date: Thu, 28 Feb 2019 13:55:49 +0800
Subject: [PATCH 01/11] Enhance PartitionGraph

---
 src/c_api/c_api_symbolic.cc                   |  17 +-
 src/c_api/c_api_test.cc                       |  19 +-
 src/executor/graph_executor.cc                | 306 ++++++++++++------
 ...onv_property.cc => mkldnn_conv_property.h} |  12 +-
 ...c => mkldnn_post_quantize_conv_property.h} |  14 +-
 .../mkldnn/mkldnn_subgraph_property.cc        |  34 ++
 src/operator/subgraph/subgraph_property.h     |  35 +-
 7 files changed, 305 insertions(+), 132 deletions(-)
 rename src/operator/subgraph/mkldnn/{mkldnn_conv_property.cc => mkldnn_conv_property.h} (94%)
 rename src/operator/subgraph/mkldnn/{mkldnn_conv_post_quantize_property.cc => mkldnn_post_quantize_conv_property.h} (90%)
 create mode 100644 src/operator/subgraph/mkldnn/mkldnn_subgraph_property.cc

diff --git a/src/c_api/c_api_symbolic.cc b/src/c_api/c_api_symbolic.cc
index 9f0d2834fcce..ae7c0bd625bd 100644
--- a/src/c_api/c_api_symbolic.cc
+++ b/src/c_api/c_api_symbolic.cc
@@ -722,14 +722,15 @@ int MXGenBackendSubgraph(SymbolHandle sym_handle, const char *backend,
   API_BEGIN();
   nnvm::Symbol *sym = static_cast<nnvm::Symbol *>(sym_handle);
   *s = sym->Copy();
-  nnvm::Graph g = Symbol2Graph(*s);
-  mxnet::op::SubgraphPropertyPtr property =
-      mxnet::op::SubgraphPropertyRegistry::Get()->CreateSubgraphProperty(
-          backend);
-  g.attrs["subgraph_property"] =
-      std::make_shared<nnvm::any>(std::move(property));
-  g = ApplyPass(std::move(g), "PartitionGraph");
-  s->outputs = g.outputs;
+  std::vector<mxnet::op::SubgraphPropertyPtr> properties =
+      mxnet::op::SubgraphPropertyRegistry::Get()->CreateSubgraphProperty(backend);
+  for (auto property : properties) {
+    nnvm::Graph g = Symbol2Graph(*s);
+    property->SetAttr("graph", g);
+    g.attrs["subgraph_property"] = std::make_shared<nnvm::any>(std::move(property));
+    g = nnvm::ApplyPass(std::move(g), "PartitionGraph");
+    s->outputs = g.outputs;
+  }
   *ret_sym_handle = s;
   API_END_HANDLE_ERROR(delete s);
 }
diff --git a/src/c_api/c_api_test.cc b/src/c_api/c_api_test.cc
index 623faa71adc9..70829db3d4a5 100644
--- a/src/c_api/c_api_test.cc
+++ b/src/c_api/c_api_test.cc
@@ -40,16 +40,19 @@ int MXPartitionGraphByOpNames(SymbolHandle sym_handle,
   }
   nnvm::Symbol* sym = static_cast<nnvm::Symbol*>(sym_handle);
   *s = sym->Copy();
-  nnvm::Graph g;
-  g.outputs = s->outputs;
   if (!op_name_set.empty()) {
-    mxnet::op::SubgraphPropertyPtr property
-        = mxnet::op::SubgraphPropertyRegistry::Get()->CreateSubgraphProperty(prop_name);
-    property->SetAttr("op_names", op_name_set);
-    g.attrs["subgraph_property"] = std::make_shared<nnvm::any>(std::move(property));
+    std::vector<mxnet::op::SubgraphPropertyPtr> properties =
+        mxnet::op::SubgraphPropertyRegistry::Get()->CreateSubgraphProperty(prop_name);
+    for (auto property : properties) {
+      nnvm::Graph g;
+      g.outputs = s->outputs;
+      property->SetAttr("graph", g);
+      property->SetAttr("op_names", op_name_set);
+      g.attrs["subgraph_property"] = std::make_shared<nnvm::any>(std::move(property));
+      g = nnvm::ApplyPass(std::move(g), "PartitionGraph");
+      s->outputs = g.outputs;
+    }
   }
-  g = nnvm::ApplyPass(std::move(g), "PartitionGraph");
-  s->outputs = g.outputs;
   *ret_sym_handle = s;
   API_END_HANDLE_ERROR(delete s);
 }
diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc
index 8302dc133c64..036cb23b848c 100644
--- a/src/executor/graph_executor.cc
+++ b/src/executor/graph_executor.cc
@@ -1499,7 +1499,7 @@ static nnvm::Graph InferForwardAttrs(nnvm::Graph g,
 // Given input attr arrays, partition the graph using the backend name equal to prop_name.
 // This is a common function for bind and simple_bind flows.
 static nnvm::Symbol PartitionGraph(const nnvm::Symbol& src,
-                                   const std::string& prop_name,
+                                   mxnet::op::SubgraphPropertyPtr subgraph_prop,
                                    const nnvm::ShapeVector& arg_shapes,
                                    const nnvm::DTypeVector& arg_dtypes,
                                    const StorageTypeVector& arg_stypes,
@@ -1507,21 +1507,12 @@ static nnvm::Symbol PartitionGraph(const nnvm::Symbol& src,
                                    const std::map<std::string, Context>& ctx_map,
                                    const std::vector<Context>& in_arg_ctxes,
                                    const std::vector<Context>& aux_state_ctxes) {
-  auto subgraph_prop = op::SubgraphPropertyRegistry::Get()->CreateSubgraphProperty(prop_name);
   nnvm::Symbol ret = src.Copy();
   nnvm::Graph g;
   g.outputs = ret.outputs;
-  g = InferForwardAttrs(g, arg_shapes, arg_dtypes, arg_stypes, default_ctx,
-                        ctx_map, in_arg_ctxes, aux_state_ctxes);
+  g = InferForwardAttrs(g, arg_shapes, arg_dtypes, arg_stypes, default_ctx, ctx_map, in_arg_ctxes,
+                        aux_state_ctxes);
   subgraph_prop->SetAttr("graph", g);
-  auto it = op::SubgraphPropertyOpNameSet::Get()->find(prop_name);
-  // assign a op name set to the subgraph property if it has been provided by users
-  if (it != op::SubgraphPropertyOpNameSet::Get()->end()) {
-    LOG(INFO) << "SubgraphPropertyOpNameSet for subgraph property " << prop_name
-              << " has been assigned a value. Please make sure it is initialized"
-                 " only for the testing purpose.";
-    subgraph_prop->SetAttr("op_names", it->second);
-  }
   g.attrs["subgraph_property"] = std::make_shared<nnvm::any>(std::move(subgraph_prop));
   g = ApplyPass(std::move(g), "PartitionGraph");
   ret.outputs = g.outputs;
@@ -1530,96 +1521,226 @@ static nnvm::Symbol PartitionGraph(const nnvm::Symbol& src,
 
 // Given input attr dicts, partition the graph using the backend name equal to prop_name.
 // This is for simple_bind flow.
-static nnvm::Symbol PartitionGraph(const nnvm::Symbol& src,
-                                   const std::string& prop_name,
+static nnvm::Symbol PartitionGraph(const nnvm::Symbol& src, const std::string& prop_name,
                                    const std::unordered_map<std::string, TShape>& arg_shape_map,
                                    const std::unordered_map<std::string, int>& arg_dtype_map,
                                    const std::unordered_map<std::string, int>& arg_stype_map,
                                    const Context& default_ctx,
                                    const std::map<std::string, Context>& ctx_map,
-                                   const std::vector<Context>& in_arg_ctxes,
-                                   const std::vector<Context>& aux_state_ctxes) {
-  const std::vector<std::string> input_names = src.ListInputNames(Symbol::kAll);
-  nnvm::ShapeVector arg_shapes(input_names.size(), TShape());
-  nnvm::DTypeVector arg_dtypes(input_names.size(), -1);
-  StorageTypeVector arg_stypes(input_names.size(), kUndefinedStorage);
-  for (size_t i = 0; i < input_names.size(); ++i) {
-    auto it1 = arg_shape_map.find(input_names[i]);
-    if (arg_shape_map.end() != it1) {
-      arg_shapes[i] = it1->second;
+                                   std::vector<Context>* in_arg_ctxes,
+                                   std::vector<Context>* arg_grad_ctxes,
+                                   std::vector<OpReqType>* grad_req_types,
+                                   std::vector<Context>* aux_state_ctxes) {
+  // setup map for in_arg_ctxes, arg_grad_ctxes, aux_state_ctxes and grad_req_types
+  std::unordered_map<std::string, Context> in_arg_ctx_map;
+  std::unordered_map<std::string, Context> arg_grad_ctx_map;
+  std::unordered_map<std::string, Context> aux_state_ctx_map;
+  std::unordered_map<std::string, OpReqType> grad_req_type_map;
+
+  auto arg_names = src.ListInputNames(nnvm::Symbol::kReadOnlyArgs);
+  auto aux_names = src.ListInputNames(nnvm::Symbol::kAuxiliaryStates);
+  for (size_t i = 0; i < arg_names.size(); ++i) {
+    auto name = arg_names[i];
+    in_arg_ctx_map[name] = in_arg_ctxes->at(i);
+    arg_grad_ctx_map[name] = arg_grad_ctxes->at(i);
+    grad_req_type_map[name] = grad_req_types->at(i);
+  }
+
+  for (size_t i = 0; i < aux_names.size(); ++i) {
+    aux_state_ctx_map[aux_names[i]] = aux_state_ctxes->at(i);
+  }
+
+  bool need_grad = false;
+  for (OpReqType req : *grad_req_types) {
+    if (req != kNullOp) {
+      need_grad = true;
+      break;
     }
-    auto it2 = arg_dtype_map.find(input_names[i]);
-    if (arg_dtype_map.end() != it2) {
-      arg_dtypes[i] = it2->second;
+  }
+  nnvm::Symbol ret = src.Copy();
+  std::unordered_set<std::string> op_names_set;
+  auto it = op::SubgraphPropertyOpNameSet::Get()->find(prop_name);
+  // assign a op name set to the subgraph property if it has been provided by users
+  if (it != op::SubgraphPropertyOpNameSet::Get()->end()) {
+    LOG(INFO) << "SubgraphPropertyOpNameSet for subgraph property " << prop_name
+              << " has been assigned a value. Please make sure it is initialized"
+                 " only for the testing purpose.";
+    op_names_set = it->second;
+  }
+  std::vector<mxnet::op::SubgraphPropertyPtr> properties =
+      op::SubgraphPropertyRegistry::Get()->CreateSubgraphProperty(prop_name);
+  for (auto subgraph_prop : properties) {
+    if (subgraph_prop->HasAttr("inference_only") &&
+        subgraph_prop->GetAttr<bool>("inference_only") == true) {
+      if (need_grad) {
+        auto full_name = subgraph_prop->HasAttr("prop_name")
+                             ? subgraph_prop->GetAttr<std::string>("prop_name")
+                             : prop_name;
+        LOG(INFO) << "Skip subgraph " << full_name << " as it requires `grad_req=null`.";
+        continue;
+      }
     }
-    auto it3 = arg_stype_map.find(input_names[i]);
-    if (arg_stype_map.end() != it3) {
-      arg_stypes[i] = it3->second;
+    subgraph_prop->SetAttr("op_names", op_names_set);
+    const std::vector<std::string> input_names = ret.ListInputNames(Symbol::kAll);
+    nnvm::ShapeVector arg_shapes(input_names.size(), TShape());
+    nnvm::DTypeVector arg_dtypes(input_names.size(), -1);
+    StorageTypeVector arg_stypes(input_names.size(), kUndefinedStorage);
+    for (size_t i = 0; i < input_names.size(); ++i) {
+      const auto& input_name = input_names[i];
+      auto it1 = arg_shape_map.find(input_name);
+      if (arg_shape_map.end() != it1) {
+        arg_shapes[i] = it1->second;
+      }
+      auto it2 = arg_dtype_map.find(input_name);
+      if (arg_dtype_map.end() != it2) {
+        arg_dtypes[i] = it2->second;
+      }
+      auto it3 = arg_stype_map.find(input_name);
+      if (arg_stype_map.end() != it3) {
+        arg_stypes[i] = it3->second;
+      }
+    }
+    ret = PartitionGraph(ret, subgraph_prop, arg_shapes, arg_dtypes, arg_stypes, default_ctx,
+                         ctx_map, *in_arg_ctxes, *aux_state_ctxes);
+    // Reorder in_arg_ctxes, arg_grad_ctxes, aux_state_ctxes and grad_req_types according to
+    // partitioned symbol input sequence
+    in_arg_ctxes->clear();
+    arg_grad_ctxes->clear();
+    aux_state_ctxes->clear();
+    grad_req_types->clear();
+    auto new_arg_names = ret.ListInputNames(nnvm::Symbol::kReadOnlyArgs);
+    auto new_aux_names = ret.ListInputNames(nnvm::Symbol::kAuxiliaryStates);
+    for (auto arg_name : new_arg_names) {
+      CHECK(in_arg_ctx_map.count(arg_name));
+      in_arg_ctxes->push_back(in_arg_ctx_map[arg_name]);
+      arg_grad_ctxes->push_back(arg_grad_ctx_map[arg_name]);
+      grad_req_types->push_back(grad_req_type_map[arg_name]);
+    }
+    for (auto arg_name : new_aux_names) {
+      CHECK(aux_state_ctx_map.count(arg_name));
+      aux_state_ctxes->push_back(aux_state_ctx_map[arg_name]);
     }
   }
-  return PartitionGraph(src, prop_name, arg_shapes, arg_dtypes, arg_stypes,
-                        default_ctx, ctx_map, in_arg_ctxes, aux_state_ctxes);
+  return ret;
 }
 
 // Given input ndarrays, partition the graph using the backend name equal to prop_name.
 // This is for bind flow.
-static nnvm::Symbol PartitionGraph(const nnvm::Symbol& src,
-                                   const std::string& prop_name,
-                                   std::vector<NDArray> *in_args,
-                                   const std::vector<NDArray> &aux_states,
+static nnvm::Symbol PartitionGraph(const nnvm::Symbol& src, const std::string& prop_name,
                                    const Context& default_ctx,
-                                   const std::map<std::string, Context>& ctx_map) {
-  const std::vector<std::string> input_names = src.ListInputNames(Symbol::kAll);
+                                   const std::map<std::string, Context>& ctx_map,
+                                   std::vector<NDArray>* in_args,
+                                   std::vector<NDArray>* arg_grad_store,
+                                   std::vector<OpReqType>* grad_req_type,
+                                   std::vector<NDArray>* aux_states) {
+  // setup map for in_args, arg_grad_store, grad_req_type and aux_states
+  std::unordered_map<std::string, NDArray> in_args_map;
+  std::unordered_map<std::string, NDArray> arg_grad_store_map;
+  std::unordered_map<std::string, OpReqType> grad_req_type_map;
+  std::unordered_map<std::string, NDArray> aux_states_map;
   const std::vector<std::string> arg_names = src.ListInputNames(nnvm::Symbol::kReadOnlyArgs);
   const std::vector<std::string> aux_names = src.ListInputNames(nnvm::Symbol::kAuxiliaryStates);
-  CHECK_EQ(arg_names.size(), in_args->size());
-  CHECK_EQ(aux_names.size(), aux_states.size());
-  nnvm::ShapeVector arg_shapes;  // all input shapes
-  arg_shapes.reserve(input_names.size());
-  nnvm::DTypeVector arg_dtypes;  // all input dtypes
-  arg_dtypes.reserve(input_names.size());
-  StorageTypeVector arg_stypes;  // all input stypes
-  arg_stypes.reserve(input_names.size());
-  std::vector<Context> in_arg_ctxes(in_args->size());
-  std::vector<Context> aux_state_ctxes(aux_states.size());
+  for (size_t i = 0; i < arg_names.size(); ++i) {
+    auto name = arg_names[i];
+    in_args_map[name] = in_args->at(i);
+    arg_grad_store_map[name] = arg_grad_store->at(i);
+    grad_req_type_map[name] = grad_req_type->at(i);
+  }
 
-  size_t i1 = 0, i2 = 0;
-  for (const auto& input_name : input_names) {
-    if (i2 < aux_names.size() && aux_names[i2] == input_name) {
-      arg_shapes.push_back(aux_states[i2].shape());
-      arg_dtypes.push_back(aux_states[i2].dtype());
-      arg_stypes.push_back(aux_states[i2].storage_type());
-      aux_state_ctxes[i2] = aux_states[i2].ctx();
-      ++i2;
-    } else {
-      CHECK(i1 < arg_names.size());
-      CHECK_EQ(arg_names[i1], input_name);
-      arg_shapes.push_back(in_args->at(i1).shape());
-      arg_dtypes.push_back(in_args->at(i1).dtype());
-      arg_stypes.push_back(in_args->at(i1).storage_type());
-      in_arg_ctxes[i1] = in_args->at(i1).ctx();
-      ++i1;
+  for (size_t i = 0; i < aux_names.size(); ++i) {
+    aux_states_map[aux_names[i]] = aux_states->at(i);
+  }
+
+  bool need_grad = false;
+  for (OpReqType req : *grad_req_type) {
+    if (req != kNullOp) {
+      need_grad = true;
+      break;
     }
   }
+  nnvm::Symbol ret = src.Copy();
+  std::unordered_set<std::string> op_names_set;
+  auto it = op::SubgraphPropertyOpNameSet::Get()->find(prop_name);
+  // assign a op name set to the subgraph property if it has been provided by users
+  if (it != op::SubgraphPropertyOpNameSet::Get()->end()) {
+    LOG(INFO) << "SubgraphPropertyOpNameSet for subgraph property " << prop_name
+              << " has been assigned a value. Please make sure it is initialized"
+                 " only for the testing purpose.";
+    op_names_set = it->second;
+  }
+  std::vector<mxnet::op::SubgraphPropertyPtr> properties =
+      op::SubgraphPropertyRegistry::Get()->CreateSubgraphProperty(prop_name);
+  for (auto subgraph_prop : properties) {
+    if (subgraph_prop->HasAttr("inference_only") &&
+        subgraph_prop->GetAttr<bool>("inference_only") == true) {
+      if (need_grad) {
+        auto full_name = subgraph_prop->HasAttr("prop_name")
+                             ? subgraph_prop->GetAttr<std::string>("prop_name")
+                             : prop_name;
+        LOG(INFO) << "Skip subgraph " << full_name << " as it requires `grad_req=null`.";
+        continue;
+      }
+    }
+    subgraph_prop->SetAttr("op_names", op_names_set);
+    const std::vector<std::string> input_names = ret.ListInputNames(Symbol::kAll);
+    const std::vector<std::string> arg_names = ret.ListInputNames(nnvm::Symbol::kReadOnlyArgs);
+    const std::vector<std::string> aux_names = ret.ListInputNames(nnvm::Symbol::kAuxiliaryStates);
+    CHECK_EQ(arg_names.size(), in_args_map.size());
+    CHECK_EQ(aux_names.size(), aux_states_map.size());
+    nnvm::ShapeVector arg_shapes;  // all input shapes
+    arg_shapes.reserve(input_names.size());
+    nnvm::DTypeVector arg_dtypes;  // all input dtypes
+    arg_dtypes.reserve(input_names.size());
+    StorageTypeVector arg_stypes;  // all input stypes
+    arg_stypes.reserve(input_names.size());
+    std::vector<Context> in_arg_ctxes(in_args_map.size());
+    std::vector<Context> aux_state_ctxes(aux_states_map.size());
+
+    size_t i1 = 0, i2 = 0;
+    for (const auto& input_name : input_names) {
+      if (i2 < aux_names.size() && aux_names[i2] == input_name) {
+        const auto &aux_st = aux_states_map[input_name];
+        arg_shapes.push_back(aux_st.shape());
+        arg_dtypes.push_back(aux_st.dtype());
+        arg_stypes.push_back(aux_st.storage_type());
+        aux_state_ctxes[i2] = aux_st.ctx();
+        ++i2;
+      } else {
+        CHECK(i1 < arg_names.size());
+        CHECK_EQ(arg_names[i1], input_name);
+        const auto &in_arg = in_args_map[input_name];
+        arg_shapes.push_back(in_arg.shape());
+        arg_dtypes.push_back(in_arg.dtype());
+        arg_stypes.push_back(in_arg.storage_type());
+        in_arg_ctxes[i1] = in_arg.ctx();
+        ++i1;
+      }
+    }
 
-  // setup in_args_map
-  std::unordered_map<std::string, NDArray> in_args_map;
-  for (size_t i = 0; i < in_args->size(); ++i) {
-    in_args_map[arg_names[i]] = in_args->at(i);
-  }
-  auto result = PartitionGraph(src, prop_name, arg_shapes, arg_dtypes, arg_stypes, default_ctx,
-                               ctx_map, in_arg_ctxes, aux_state_ctxes);
-  // Reorder in_args into new_in_args according to partitioned symbol input sequence
-  std::vector<NDArray> new_in_args(in_args->size());
-  // get new symbol in_arg names
-  std::vector<std::string> new_arg_names = result.ListInputNames(nnvm::Symbol::kReadOnlyArgs);
+    ret = PartitionGraph(ret, subgraph_prop, arg_shapes, arg_dtypes, arg_stypes, default_ctx,
+                         ctx_map, in_arg_ctxes, aux_state_ctxes);
+  }
+  // Reorder in_args, arg_grad_store, grad_req_type and aux_states according to partitioned symbol
+  // input sequence
+  const auto new_arg_names = ret.ListInputNames(nnvm::Symbol::kReadOnlyArgs);
+  const auto new_aux_names = ret.ListInputNames(nnvm::Symbol::kAuxiliaryStates);
+  CHECK_EQ(arg_names.size(), new_arg_names.size());
   CHECK_EQ(arg_names.size(), new_arg_names.size());
   in_args->clear();
+  arg_grad_store->clear();
+  grad_req_type->clear();
+  aux_states->clear();
   for (auto arg_name : new_arg_names) {
     CHECK(in_args_map.count(arg_name));
     in_args->push_back(in_args_map[arg_name]);
+    arg_grad_store->push_back(arg_grad_store_map[arg_name]);
+    grad_req_type->push_back(grad_req_type_map[arg_name]);
+  }
+  for (auto arg_name : new_aux_names) {
+    CHECK(aux_states_map.count(arg_name));
+    aux_states->push_back(aux_states_map[arg_name]);
   }
-  return result;
+  return ret;
 }
 }  // namespace exec
 
@@ -1640,17 +1761,18 @@ Executor *Executor::SimpleBind(nnvm::Symbol symbol,
                                std::unordered_map<std::string, NDArray>* shared_buffer,
                                Executor* shared_exec) {
   auto exec = new exec::GraphExecutor();
+  std::vector<Context> tmp_in_arg_ctxes = in_arg_ctxes;
+  std::vector<Context> tmp_arg_grad_ctxes = arg_grad_ctxes;
+  std::vector<Context> tmp_aux_state_ctxes = aux_state_ctxes;
+  std::vector<OpReqType> tmp_grad_req_types = grad_req_types;
   if (!exec->subgraph_property().empty()) {
     symbol = exec::PartitionGraph(symbol, exec->subgraph_property(), arg_shape_map, arg_dtype_map,
-                                  arg_stype_map, default_ctx, group2ctx, in_arg_ctxes,
-                                  aux_state_ctxes);
-  }
-  exec->Init(symbol, default_ctx, group2ctx,
-             in_arg_ctxes, arg_grad_ctxes, aux_state_ctxes,
-             arg_shape_map, arg_dtype_map, arg_stype_map,
-             grad_req_types, shared_arg_names,
-             in_args, arg_grads, aux_states,
-             shared_buffer, shared_exec);
+                                  arg_stype_map, default_ctx, group2ctx, &tmp_in_arg_ctxes,
+                                  &tmp_arg_grad_ctxes, &tmp_grad_req_types, &tmp_aux_state_ctxes);
+  }
+  exec->Init(symbol, default_ctx, group2ctx, tmp_in_arg_ctxes, tmp_arg_grad_ctxes,
+             tmp_aux_state_ctxes, arg_shape_map, arg_dtype_map, arg_stype_map, tmp_grad_req_types,
+             shared_arg_names, in_args, arg_grads, aux_states, shared_buffer, shared_exec);
   return exec;
 }
 
@@ -1664,13 +1786,17 @@ Executor *Executor::Bind(nnvm::Symbol symbol,
                          Executor* shared_exec) {
   auto exec = new exec::GraphExecutor();
   std::vector<NDArray> tmp_in_args = in_args;
+  std::vector<NDArray> tmp_arg_grad_store = arg_grad_store;
+  std::vector<OpReqType> tmp_grad_req_type = grad_req_type;
+  std::vector<NDArray> tmp_aux_states = aux_states;
+
   if (!exec->subgraph_property().empty()) {
-    symbol = exec::PartitionGraph(symbol, exec->subgraph_property(), &tmp_in_args, aux_states,
-                                  default_ctx, group2ctx);
+    symbol = exec::PartitionGraph(symbol, exec->subgraph_property(), default_ctx, group2ctx,
+                                  &tmp_in_args, &tmp_arg_grad_store, &tmp_grad_req_type,
+                                  &tmp_aux_states);
   }
-  exec->Init(symbol, default_ctx, group2ctx,
-             tmp_in_args, arg_grad_store, grad_req_type, aux_states,
-             reinterpret_cast<Executor*>(shared_exec));
+  exec->Init(symbol, default_ctx, group2ctx, tmp_in_args, tmp_arg_grad_store, tmp_grad_req_type,
+             tmp_aux_states, reinterpret_cast<Executor*>(shared_exec));
   return exec;
 }
 }  // namespace mxnet
diff --git a/src/operator/subgraph/mkldnn/mkldnn_conv_property.cc b/src/operator/subgraph/mkldnn/mkldnn_conv_property.h
similarity index 94%
rename from src/operator/subgraph/mkldnn/mkldnn_conv_property.cc
rename to src/operator/subgraph/mkldnn/mkldnn_conv_property.h
index e462191c2898..b89e4b347ed9 100644
--- a/src/operator/subgraph/mkldnn/mkldnn_conv_property.cc
+++ b/src/operator/subgraph/mkldnn/mkldnn_conv_property.h
@@ -17,8 +17,12 @@
  * under the License.
  */
 
+#ifndef MXNET_OPERATOR_SUBGRAPH_MKLDNN_MKLDNN_CONV_PROPERTY_H_
+#define MXNET_OPERATOR_SUBGRAPH_MKLDNN_MKLDNN_CONV_PROPERTY_H_
 #if MXNET_USE_MKLDNN == 1
 
+#include <string>
+#include <vector>
 #include "../common.h"
 #include "../subgraph_property.h"
 #include "../../nn/activation-inl.h"
@@ -149,7 +153,10 @@ class SgMKLDNNConvProperty : public SubgraphProperty {
     }
   }
   static SubgraphPropertyPtr Create() {
-    return std::make_shared<SgMKLDNNConvProperty>();
+    auto property = std::make_shared<SgMKLDNNConvProperty>();
+    property->SetAttr<std::string>("prop_name", "MKLDNN Convolution optimization pass");
+    property->SetAttr<bool>("inference_only", true);
+    return property;
   }
   nnvm::NodePtr CreateSubgraphNode(const nnvm::Symbol &sym,
                                    const int subgraph_id = 0) const override {
@@ -241,9 +248,8 @@ class SgMKLDNNConvProperty : public SubgraphProperty {
   int disable_conv_sum;
 };
 
-MXNET_REGISTER_SUBGRAPH_PROPERTY(MKLDNN, SgMKLDNNConvProperty);
-
 }  // namespace op
 }  // namespace mxnet
 
 #endif  // if MXNET_USE_MKLDNN == 1
+#endif  // MXNET_OPERATOR_SUBGRAPH_MKLDNN_MKLDNN_CONV_PROPERTY_H_
diff --git a/src/operator/subgraph/mkldnn/mkldnn_conv_post_quantize_property.cc b/src/operator/subgraph/mkldnn/mkldnn_post_quantize_conv_property.h
similarity index 90%
rename from src/operator/subgraph/mkldnn/mkldnn_conv_post_quantize_property.cc
rename to src/operator/subgraph/mkldnn/mkldnn_post_quantize_conv_property.h
index fc68287b039d..90a02c152422 100644
--- a/src/operator/subgraph/mkldnn/mkldnn_conv_post_quantize_property.cc
+++ b/src/operator/subgraph/mkldnn/mkldnn_post_quantize_conv_property.h
@@ -16,9 +16,12 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
+#ifndef MXNET_OPERATOR_SUBGRAPH_MKLDNN_MKLDNN_POST_QUANTIZE_CONV_PROPERTY_H_
+#define MXNET_OPERATOR_SUBGRAPH_MKLDNN_MKLDNN_POST_QUANTIZE_CONV_PROPERTY_H_
 #if MXNET_USE_MKLDNN == 1
 
+#include <string>
+#include <vector>
 #include "../common.h"
 #include "../subgraph_property.h"
 #include "../../nn/mkldnn/mkldnn_convolution-inl.h"
@@ -107,7 +110,11 @@ class SgMKLDNNConvPostQuantizeProperty : public SubgraphProperty {
     }
   }
   static SubgraphPropertyPtr Create() {
-    return std::make_shared<SgMKLDNNConvPostQuantizeProperty>();
+    auto property = std::make_shared<SgMKLDNNConvPostQuantizeProperty>();
+    property->SetAttr<std::string>("prop_name",
+                                   "MKLDNN Convolution post-quantization optimization pass");
+    property->SetAttr<bool>("inference_only", true);
+    return property;
   }
   nnvm::NodePtr CreateSubgraphNode(const nnvm::Symbol &sym,
                                    const int subgraph_id = 0) const override {
@@ -155,9 +162,8 @@ class SgMKLDNNConvPostQuantizeProperty : public SubgraphProperty {
   int disable_all;
 };
 
-MXNET_REGISTER_SUBGRAPH_PROPERTY(MKLDNN_POST_QUANTIZE, SgMKLDNNConvPostQuantizeProperty);
-
 }  // namespace op
 }  // namespace mxnet
 
 #endif  // if MXNET_USE_MKLDNN == 1
+#endif  // MXNET_OPERATOR_SUBGRAPH_MKLDNN_MKLDNN_POST_QUANTIZE_CONV_PROPERTY_H_
diff --git a/src/operator/subgraph/mkldnn/mkldnn_subgraph_property.cc b/src/operator/subgraph/mkldnn/mkldnn_subgraph_property.cc
new file mode 100644
index 000000000000..ce570baf3591
--- /dev/null
+++ b/src/operator/subgraph/mkldnn/mkldnn_subgraph_property.cc
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#if MXNET_USE_MKLDNN == 1
+
+#include "mkldnn_conv_property.h"
+#include "mkldnn_post_quantize_conv_property.h"
+
+namespace mxnet {
+namespace op {
+
+MXNET_REGISTER_SUBGRAPH_PROPERTY(MKLDNN, SgMKLDNNConvProperty);
+MXNET_REGISTER_SUBGRAPH_PROPERTY(MKLDNN_POST_QUANTIZE, SgMKLDNNConvPostQuantizeProperty);
+
+}  // namespace op
+}  // namespace mxnet
+
+#endif  // MXNET_USE_MKLDNN == 1
diff --git a/src/operator/subgraph/subgraph_property.h b/src/operator/subgraph/subgraph_property.h
index e9fdd6619275..d16f298ebedb 100644
--- a/src/operator/subgraph/subgraph_property.h
+++ b/src/operator/subgraph/subgraph_property.h
@@ -145,7 +145,13 @@ class SubgraphProperty {
     CHECK(it != attrs_.end()) << "Cannot find attribute " << name << " in SubgraphProperty";
     return nnvm::get<T>(*it->second);
   }
-
+  /*!
+   * \brief Check if the attr exists.
+   */
+  bool HasAttr(const std::string& name) const {
+    auto it = attrs_.find(name);
+    return it != attrs_.end();
+  }
  protected:
   std::unordered_map<std::string, std::shared_ptr<nnvm::any>> attrs_;
 };
@@ -160,35 +166,26 @@ class SubgraphPropertyRegistry {
     return &inst;
   }
 
-  SubgraphPropertyPtr CreateSubgraphProperty(const std::string& name) {
+  std::vector<SubgraphPropertyPtr> CreateSubgraphProperty(const std::string& name) {
     auto it = prop_fn_map_.find(name);
     CHECK(it != prop_fn_map_.end()) << "SubgraphProperty " << name
                                     << " is not found in SubgraphPropertyRegistry";
-    return it->second();
-  }
-
-  SubgraphPropertyCreateFn __REGISTER_OR_GET__(const std::string& name,
-                                               SubgraphPropertyCreateFn fn) {
-    if (prop_fn_map_.count(name) == 0U) {
-      return __REGISTER__(name, fn);
-    } else {
-      return prop_fn_map_.at(name);
-    }
+    std::vector<SubgraphPropertyPtr> ret;
+    ret.reserve(it->second.size());
+    for (auto i : it->second) ret.emplace_back(i());
+    return ret;
   }
 
- private:
   SubgraphPropertyCreateFn __REGISTER__(const std::string& name, SubgraphPropertyCreateFn fn) {
-    CHECK_EQ(prop_fn_map_.count(name), 0U) << "Subgraph property " << name
-                                           << " has been registered";
-    prop_fn_map_[name] = fn;
-    return prop_fn_map_[name];
+    prop_fn_map_[name].push_back(fn);
+    return fn;
   }
 
   SubgraphPropertyRegistry() = default;
   SubgraphPropertyRegistry(const SubgraphPropertyRegistry&) = delete;
   SubgraphPropertyRegistry(SubgraphPropertyRegistry&&) = delete;
   SubgraphPropertyRegistry& operator=(const SubgraphPropertyRegistry&) = delete;
-  std::unordered_map<std::string, SubgraphPropertyCreateFn> prop_fn_map_;
+  std::unordered_map<std::string, std::vector<SubgraphPropertyCreateFn>> prop_fn_map_;
 };
 
 // This op name set is for setting the names of operators that should be grouped into
@@ -200,7 +197,7 @@ typedef dmlc::ThreadLocalStore<std::unordered_map<std::string, std::unordered_se
 
 #define MXNET_REGISTER_SUBGRAPH_PROPERTY(Name, SubgraphPropertyType) \
   static DMLC_ATTRIBUTE_UNUSED auto __make_ ## SubgraphPropertyType ## _ ## Name ## __ = \
-    SubgraphPropertyRegistry::Get()->__REGISTER_OR_GET__(#Name, &SubgraphPropertyType::Create)
+    SubgraphPropertyRegistry::Get()->__REGISTER__(#Name, &SubgraphPropertyType::Create)
 
 }  // namespace op
 }  // namespace mxnet

From 30e3564d8322e019035600bfef11178a28840637 Mon Sep 17 00:00:00 2001
From: Zhennan Qin <zhennan.qin@intel.com>
Date: Thu, 14 Mar 2019 15:56:45 +0800
Subject: [PATCH 02/11] Fix lint

---
 .../subgraph/mkldnn/mkldnn_fc_post_quantize_property.h       | 5 +++--
 src/operator/subgraph/mkldnn/mkldnn_fc_property.h            | 1 +
 src/operator/subgraph/subgraph_property.h                    | 1 +
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/operator/subgraph/mkldnn/mkldnn_fc_post_quantize_property.h b/src/operator/subgraph/mkldnn/mkldnn_fc_post_quantize_property.h
index 6d32ca754a44..14ddf2d0abd1 100644
--- a/src/operator/subgraph/mkldnn/mkldnn_fc_post_quantize_property.h
+++ b/src/operator/subgraph/mkldnn/mkldnn_fc_post_quantize_property.h
@@ -28,10 +28,11 @@
 #define MXNET_OPERATOR_SUBGRAPH_MKLDNN_MKLDNN_FC_POST_QUANTIZE_PROPERTY_H_
 #if MXNET_USE_MKLDNN == 1
 
-#include "../common.h"
-#include "../subgraph_property.h"
+#include <vector>
 #include "../../nn/fully_connected-inl.h"
 #include "../../quantization/requantize-inl.h"
+#include "../common.h"
+#include "../subgraph_property.h"
 
 namespace mxnet {
 namespace op {
diff --git a/src/operator/subgraph/mkldnn/mkldnn_fc_property.h b/src/operator/subgraph/mkldnn/mkldnn_fc_property.h
index 81fdb80a7996..7913ad56c361 100644
--- a/src/operator/subgraph/mkldnn/mkldnn_fc_property.h
+++ b/src/operator/subgraph/mkldnn/mkldnn_fc_property.h
@@ -28,6 +28,7 @@
 #define MXNET_OPERATOR_SUBGRAPH_MKLDNN_MKLDNN_FC_PROPERTY_H_
 #if MXNET_USE_MKLDNN == 1
 
+#include <vector>
 #include "../common.h"
 #include "../subgraph_property.h"
 
diff --git a/src/operator/subgraph/subgraph_property.h b/src/operator/subgraph/subgraph_property.h
index d16f298ebedb..cad5e8a5597e 100644
--- a/src/operator/subgraph/subgraph_property.h
+++ b/src/operator/subgraph/subgraph_property.h
@@ -152,6 +152,7 @@ class SubgraphProperty {
     auto it = attrs_.find(name);
     return it != attrs_.end();
   }
+
  protected:
   std::unordered_map<std::string, std::shared_ptr<nnvm::any>> attrs_;
 };

From 0111e95fafab134eb8761ad7dc9191f331aad211 Mon Sep 17 00:00:00 2001
From: Zhennan Qin <zhennan.qin@intel.com>
Date: Fri, 15 Mar 2019 09:20:20 +0800
Subject: [PATCH 03/11] Fix test

---
 src/operator/subgraph/mkldnn/mkldnn_fc.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/operator/subgraph/mkldnn/mkldnn_fc.cc b/src/operator/subgraph/mkldnn/mkldnn_fc.cc
index 94e2bda1e16c..2e02b44cbe5d 100644
--- a/src/operator/subgraph/mkldnn/mkldnn_fc.cc
+++ b/src/operator/subgraph/mkldnn/mkldnn_fc.cc
@@ -427,6 +427,9 @@ NNVM_REGISTER_OP(_sg_mkldnn_fully_connected)
 .set_attr<FCreateOpState>("FCreateOpState", CreateSgMKLDNNFCState)
 .set_attr<FStatefulComputeEx>("FStatefulComputeEx<cpu>", SgMKLDNNFCForward)
 .set_attr<bool>("TIsMKLDNN", true)
+// TODO(Xinyu): a temp solution to enable GluonCV INT8 flow,
+// will be reverted after the improvement of CachedOP is done.
+.set_attr<nnvm::FGradient>("FGradient", MakeZeroGradNodes)
 .set_attr<FResourceRequest>("FResourceRequest", [](const NodeAttrs& n) {
   return std::vector<ResourceRequest>{ResourceRequest::kTempSpace};
 })

From f3b5e980581c94c9d36b5a9f02cbb923be9ba7bd Mon Sep 17 00:00:00 2001
From: Zhennan Qin <zhennan.qin@intel.com>
Date: Fri, 15 Mar 2019 12:40:38 +0800
Subject: [PATCH 04/11] Run CI


From acb76b0c386542405ac038c6e9e0267684bfff20 Mon Sep 17 00:00:00 2001
From: Zhennan Qin <zhennan.qin@intel.com>
Date: Wed, 20 Mar 2019 13:04:11 +0800
Subject: [PATCH 05/11] Change subgraph property register

---
 src/executor/graph_executor.cc                |  8 ++---
 .../mkldnn_conv_post_quantize_property.h      | 24 +++++---------
 .../subgraph/mkldnn/mkldnn_conv_property.h    | 15 ++++-----
 .../mkldnn/mkldnn_fc_post_quantize_property.h | 18 ++++------
 .../subgraph/mkldnn/mkldnn_fc_property.h      | 33 ++++++++-----------
 src/operator/subgraph/partition_graph.cc      |  4 +++
 src/operator/subgraph/subgraph_property.h     | 19 +++++++++--
 7 files changed, 60 insertions(+), 61 deletions(-)

diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc
index 84962beba54b..53b5941778c9 100644
--- a/src/executor/graph_executor.cc
+++ b/src/executor/graph_executor.cc
@@ -1519,8 +1519,8 @@ static nnvm::Symbol PartitionGraph(const nnvm::Symbol& src,
     if (subgraph_prop->HasAttr("inference_only") &&
         subgraph_prop->GetAttr<bool>("inference_only") == true) {
       if (need_grad) {
-        auto full_name = subgraph_prop->HasAttr("prop_name")
-                             ? subgraph_prop->GetAttr<std::string>("prop_name")
+        auto full_name = subgraph_prop->HasAttr("property_name")
+                             ? subgraph_prop->GetAttr<std::string>("property_name")
                              : prop_name;
         LOG(INFO) << "Skip subgraph " << full_name << " as it requires `grad_req=null`.";
         continue;
@@ -1620,8 +1620,8 @@ static nnvm::Symbol PartitionGraph(const nnvm::Symbol& src, const std::string& p
     if (subgraph_prop->HasAttr("inference_only") &&
         subgraph_prop->GetAttr<bool>("inference_only") == true) {
       if (need_grad) {
-        auto full_name = subgraph_prop->HasAttr("prop_name")
-                             ? subgraph_prop->GetAttr<std::string>("prop_name")
+        auto full_name = subgraph_prop->HasAttr("property_name")
+                             ? subgraph_prop->GetAttr<std::string>("property_name")
                              : prop_name;
         LOG(INFO) << "Skip subgraph " << full_name << " as it requires `grad_req=null`.";
         continue;
diff --git a/src/operator/subgraph/mkldnn/mkldnn_conv_post_quantize_property.h b/src/operator/subgraph/mkldnn/mkldnn_conv_post_quantize_property.h
index 606886f3a70c..5ee23db2a5ab 100644
--- a/src/operator/subgraph/mkldnn/mkldnn_conv_post_quantize_property.h
+++ b/src/operator/subgraph/mkldnn/mkldnn_conv_post_quantize_property.h
@@ -41,16 +41,14 @@ class SgMKLDNNConvPostQuantizeSelector : public SubgraphSelector {
   };
 
  private:
-  bool disable_all;
   SelectStatus status;
   std::vector<const nnvm::Node *> matched_list;
 
  public:
-  explicit SgMKLDNNConvPostQuantizeSelector(int dis_all)
-      : disable_all(dis_all) {}
+  SgMKLDNNConvPostQuantizeSelector() {}
 
   bool Select(const nnvm::Node &n) override {
-    if ((!disable_all) && n.op() && n.op()->name == "_sg_mkldnn_conv") {
+    if (n.op() && n.op()->name == "_sg_mkldnn_conv") {
       auto const &param = nnvm::get<MKLDNNConvFusionParam>(n.attrs.parsed);
       if (param.full_conv_param.mkldnn_param.quantized) {
         status = kStart;
@@ -101,21 +99,15 @@ class SgMKLDNNConvPostQuantizeSelector : public SubgraphSelector {
 
 class SgMKLDNNConvPostQuantizeProperty : public SubgraphProperty {
  public:
-  SgMKLDNNConvPostQuantizeProperty() {
-    disable_all = dmlc::GetEnv("MXNET_DISABLE_MKLDNN_OPT", 0);
-    if (disable_all) {
-      LOG(INFO) << "MKLDNN Convolution post-quantization optimization pass is disabled.";
-    } else {
-      LOG(INFO) << "Start to execute MKLDNN Convolution post-quantization optimization pass.";
-    }
-  }
+  SgMKLDNNConvPostQuantizeProperty() {}
+
   static SubgraphPropertyPtr Create() {
     auto property = std::make_shared<SgMKLDNNConvPostQuantizeProperty>();
-    property->SetAttr<std::string>("property_name",
-                                   "MKLDNN Convolution post-quantization optimization pass");
+    property->SetAttr<std::string>("property_name", name_);
     property->SetAttr<bool>("inference_only", true);
     return property;
   }
+
   nnvm::NodePtr CreateSubgraphNode(const nnvm::Symbol &sym,
                                    const int subgraph_id = 0) const override {
     nnvm::NodePtr conv_node = nullptr;
@@ -145,7 +137,7 @@ class SgMKLDNNConvPostQuantizeProperty : public SubgraphProperty {
 
   SubgraphSelectorPtr CreateSubgraphSelector() const override {
     auto selector =
-        std::make_shared<SgMKLDNNConvPostQuantizeSelector>(disable_all);
+        std::make_shared<SgMKLDNNConvPostQuantizeSelector>();
     return selector;
   }
 
@@ -159,7 +151,7 @@ class SgMKLDNNConvPostQuantizeProperty : public SubgraphProperty {
   }
 
  private:
-  int disable_all;
+  constexpr static char *name_ = "MKLDNN Convolution post-quantization optimization pass";
 };
 
 }  // namespace op
diff --git a/src/operator/subgraph/mkldnn/mkldnn_conv_property.h b/src/operator/subgraph/mkldnn/mkldnn_conv_property.h
index b89e4b347ed9..5837fa8e905b 100644
--- a/src/operator/subgraph/mkldnn/mkldnn_conv_property.h
+++ b/src/operator/subgraph/mkldnn/mkldnn_conv_property.h
@@ -140,21 +140,19 @@ class SgMKLDNNConvSelector : public SubgraphSelector {
 class SgMKLDNNConvProperty : public SubgraphProperty {
  public:
   SgMKLDNNConvProperty() {
-    disable_all = dmlc::GetEnv("MXNET_DISABLE_MKLDNN_OPT", 0);
     disable_conv_bn = dmlc::GetEnv("MXNET_DISABLE_MKLDNN_FUSE_CONV_BN", 0);
     disable_conv_relu = dmlc::GetEnv("MXNET_DISABLE_MKLDNN_FUSE_CONV_RELU", 0);
     disable_conv_sum = dmlc::GetEnv("MXNET_DISABLE_MKLDNN_FUSE_CONV_SUM", 0);
 
-    disable_all = disable_all || (disable_conv_bn && disable_conv_relu && disable_conv_sum);
-    if (disable_all) {
-      LOG(INFO) << "MKLDNN Convolution optimization pass is disabled.";
-    } else {
-      LOG(INFO) << "Start to execute MKLDNN Convolution optimization pass.";
-    }
+    disable_all = disable_conv_bn && disable_conv_relu && disable_conv_sum;
   }
   static SubgraphPropertyPtr Create() {
+    if (dmlc::GetEnv("MXNET_DISABLE_MKLDNN_CONV_OPT", 0)) {
+      LOG(INFO) << name_ << "is disabled.";
+      return nullptr;
+    }
     auto property = std::make_shared<SgMKLDNNConvProperty>();
-    property->SetAttr<std::string>("prop_name", "MKLDNN Convolution optimization pass");
+    property->SetAttr<std::string>("property_name", name_);
     property->SetAttr<bool>("inference_only", true);
     return property;
   }
@@ -242,6 +240,7 @@ class SgMKLDNNConvProperty : public SubgraphProperty {
   }
 
  private:
+  constexpr static char* name_ = "MKLDNN Convolution optimization pass";
   int disable_all;
   int disable_conv_bn;
   int disable_conv_relu;
diff --git a/src/operator/subgraph/mkldnn/mkldnn_fc_post_quantize_property.h b/src/operator/subgraph/mkldnn/mkldnn_fc_post_quantize_property.h
index 14ddf2d0abd1..a241d54cdd48 100644
--- a/src/operator/subgraph/mkldnn/mkldnn_fc_post_quantize_property.h
+++ b/src/operator/subgraph/mkldnn/mkldnn_fc_post_quantize_property.h
@@ -28,6 +28,7 @@
 #define MXNET_OPERATOR_SUBGRAPH_MKLDNN_MKLDNN_FC_POST_QUANTIZE_PROPERTY_H_
 #if MXNET_USE_MKLDNN == 1
 
+#include <string>
 #include <vector>
 #include "../../nn/fully_connected-inl.h"
 #include "../../quantization/requantize-inl.h"
@@ -136,20 +137,15 @@ class SgMKLDNNFCPostQuantizeSelector : public SubgraphSelector {
 class SgMKLDNNFCPostQuantizeProperty : public SubgraphProperty {
  public:
   SgMKLDNNFCPostQuantizeProperty() {
-    disable_all = dmlc::GetEnv("MXNET_DISABLE_MKLDNN_POST_OPT", false);
     disable_fuse_all = dmlc::GetEnv("MXNET_DISABLE_MKLDNN_QFC_FUSE_ALL", false);
     disable_float_output = dmlc::GetEnv("MXNET_DISABLE_MKLDNN_QFC_FLOAT_OUTPUT", false);
-
-    disable_all = disable_all || disable_fuse_all;
-    if (disable_all) {
-      LOG(INFO) << "MKLDNN FullyConnected post-quantization optimization pass is disabled.";
-    } else {
-      LOG(INFO) << "Start to execute MKLDNN FullyConected post-quantization optimization pass.";
-    }
   }
 
   static SubgraphPropertyPtr Create() {
-    return std::make_shared<SgMKLDNNFCPostQuantizeProperty>();
+    auto property = std::make_shared<SgMKLDNNFCPostQuantizeProperty>();
+    property->SetAttr<std::string>("property_name", name_);
+    property->SetAttr<bool>("inference_only", true);
+    return property;
   }
 
   nnvm::NodePtr CreateSubgraphNode(const nnvm::Symbol &sym,
@@ -192,7 +188,7 @@ class SgMKLDNNFCPostQuantizeProperty : public SubgraphProperty {
 
   SubgraphSelectorPtr CreateSubgraphSelector() const override {
     auto selector =
-        std::make_shared<SgMKLDNNFCPostQuantizeSelector>(disable_all,
+        std::make_shared<SgMKLDNNFCPostQuantizeSelector>(disable_fuse_all,
                                                          disable_float_output);
     return selector;
   }
@@ -207,7 +203,7 @@ class SgMKLDNNFCPostQuantizeProperty : public SubgraphProperty {
   }
 
  private:
-  bool disable_all;
+  constexpr static char* name_ = "MKLDNN FullyConected post-quantization optimization pass";
   bool disable_fuse_all;
   bool disable_float_output;
 };
diff --git a/src/operator/subgraph/mkldnn/mkldnn_fc_property.h b/src/operator/subgraph/mkldnn/mkldnn_fc_property.h
index 7913ad56c361..a2deb90a2679 100644
--- a/src/operator/subgraph/mkldnn/mkldnn_fc_property.h
+++ b/src/operator/subgraph/mkldnn/mkldnn_fc_property.h
@@ -28,6 +28,7 @@
 #define MXNET_OPERATOR_SUBGRAPH_MKLDNN_MKLDNN_FC_PROPERTY_H_
 #if MXNET_USE_MKLDNN == 1
 
+#include <string>
 #include <vector>
 #include "../common.h"
 #include "../subgraph_property.h"
@@ -45,19 +46,16 @@ class SgMKLDNNFCSelector : public SubgraphSelector {
   };
 
  private:
-  bool disable_all;
   bool disable_fc_relu;
   SelectStatus status;
   std::vector<const nnvm::Node *> matched_list;
 
  public:
-  SgMKLDNNFCSelector(const bool dis_all, const bool dis_fc_relu)
-      : disable_all(dis_all),
-        disable_fc_relu(dis_fc_relu) {}
+  explicit SgMKLDNNFCSelector(const bool dis_fc_relu) : disable_fc_relu(dis_fc_relu) {}
 
   bool Select(const nnvm::Node &n) override {
     if (n.op() == Op::Get("FullyConnected")) {
-      status = disable_all ? kSuccess : kStart;
+      status = disable_fc_relu ? kSuccess : kStart;
       matched_list.clear();
       matched_list.push_back(&n);
       return true;
@@ -89,8 +87,7 @@ class SgMKLDNNFCSelector : public SubgraphSelector {
 
     switch (status) {
       case kStart:
-        if ((!disable_fc_relu) &&
-            new_node.op() == Op::Get("Activation") &&
+        if (new_node.op() == Op::Get("Activation") &&
             new_node.attrs.dict.at("act_type") == "relu") {
           matched_list.push_back(&new_node);
           status = kSuccess;
@@ -123,19 +120,18 @@ class SgMKLDNNFCSelector : public SubgraphSelector {
 class SgMKLDNNFCProperty : public SubgraphProperty {
  public:
   SgMKLDNNFCProperty() {
-    disable_all = dmlc::GetEnv("MXNET_DISABLE_MKLDNN_OPT", false);
     disable_fc_relu = dmlc::GetEnv("MXNET_DISABLE_MKLDNN_FUSE_FC_RELU", false);
-
-    disable_all = disable_all || disable_fc_relu;
-    if (disable_all) {
-      LOG(INFO) << "MKLDNN FullyConnected optimization pass is disabled.";
-    } else {
-      LOG(INFO) << "Start to execute MKLDNN FullyConnected optimization pass.";
-    }
   }
 
   static SubgraphPropertyPtr Create() {
-    return std::make_shared<SgMKLDNNFCProperty>();
+    if (dmlc::GetEnv("MXNET_DISABLE_MKLDNN_FC_OPT", 0)) {
+      LOG(INFO) << name_ << "is disabled.";
+      return nullptr;
+    }
+    auto property = std::make_shared<SgMKLDNNFCProperty>();
+    property->SetAttr<std::string>("property_name", name_);
+    property->SetAttr<bool>("inference_only", true);
+    return property;
   }
 
   nnvm::NodePtr CreateSubgraphNode(const nnvm::Symbol &sym,
@@ -168,8 +164,7 @@ class SgMKLDNNFCProperty : public SubgraphProperty {
   }
 
   SubgraphSelectorPtr CreateSubgraphSelector() const override {
-    auto selector = std::make_shared<SgMKLDNNFCSelector>(
-        disable_all, disable_fc_relu);
+    auto selector = std::make_shared<SgMKLDNNFCSelector>(disable_fc_relu);
     return selector;
   }
 
@@ -184,7 +179,7 @@ class SgMKLDNNFCProperty : public SubgraphProperty {
   }
 
  private:
-  bool disable_all;
+  constexpr static char* name_ = "MKLDNN FullyConnected optimization pass";
   bool disable_fc_relu;
 };
 
diff --git a/src/operator/subgraph/partition_graph.cc b/src/operator/subgraph/partition_graph.cc
index 90a14caa510b..4c7552ec9bfe 100644
--- a/src/operator/subgraph/partition_graph.cc
+++ b/src/operator/subgraph/partition_graph.cc
@@ -740,6 +740,10 @@ Graph PartitionGraph(Graph&& g) {
   }
   using namespace sg;
   const SubgraphPropertyPtr& subg_prop = g.GetAttr<SubgraphPropertyPtr>("subgraph_property");
+  const std::string& prop_name = subg_prop->HasAttr("property_name")
+                                     ? subg_prop->GetAttr<std::string>("property_name")
+                                     : "partition graph";
+  LOG(INFO) << "start to execute " << prop_name << ".";
   // top sort NodeEntry of all the nodes' inputs
   std::unordered_map<const nnvm::NodeEntry*, size_t> entry_top_order_map;
   TopSortEntries(g, &entry_top_order_map);
diff --git a/src/operator/subgraph/subgraph_property.h b/src/operator/subgraph/subgraph_property.h
index cad5e8a5597e..8b87c8bedec0 100644
--- a/src/operator/subgraph/subgraph_property.h
+++ b/src/operator/subgraph/subgraph_property.h
@@ -173,7 +173,14 @@ class SubgraphPropertyRegistry {
                                     << " is not found in SubgraphPropertyRegistry";
     std::vector<SubgraphPropertyPtr> ret;
     ret.reserve(it->second.size());
-    for (auto i : it->second) ret.emplace_back(i());
+     for (auto i : it->second) {
+       auto ptr_it = prop_ptr_map_.find(i);
+       if (ptr_it == prop_ptr_map_.end()) {
+         prop_ptr_map_[i] = i();
+         ptr_it = prop_ptr_map_.find(i);
+       }
+       if (ptr_it->second) ret.emplace_back(ptr_it->second);
+    }
     return ret;
   }
 
@@ -187,6 +194,7 @@ class SubgraphPropertyRegistry {
   SubgraphPropertyRegistry(SubgraphPropertyRegistry&&) = delete;
   SubgraphPropertyRegistry& operator=(const SubgraphPropertyRegistry&) = delete;
   std::unordered_map<std::string, std::vector<SubgraphPropertyCreateFn>> prop_fn_map_;
+  std::unordered_map<SubgraphPropertyCreateFn, SubgraphPropertyPtr> prop_ptr_map_;
 };
 
 // This op name set is for setting the names of operators that should be grouped into
@@ -196,9 +204,14 @@ class SubgraphPropertyRegistry {
 typedef dmlc::ThreadLocalStore<std::unordered_map<std::string, std::unordered_set<std::string>>>
   SubgraphPropertyOpNameSet;
 
+#define DECLEARPROPERTY1(NAME, SubgraphPropertyType, X) \
+  DMLC_ATTRIBUTE_UNUSED auto __make_##SubgraphPropertyType##_##Name##_##X##__
+#define DECLEARPROPERTY(NAME, SubgraphPropertyType, X) \
+  DECLEARPROPERTY1(NAME, SubgraphPropertyType, X)
+
 #define MXNET_REGISTER_SUBGRAPH_PROPERTY(Name, SubgraphPropertyType) \
-  static DMLC_ATTRIBUTE_UNUSED auto __make_ ## SubgraphPropertyType ## _ ## Name ## __ = \
-    SubgraphPropertyRegistry::Get()->__REGISTER__(#Name, &SubgraphPropertyType::Create)
+  static DECLEARPROPERTY(Name, SubgraphPropertyType, __LINE__) =     \
+      SubgraphPropertyRegistry::Get()->__REGISTER__(#Name, &SubgraphPropertyType::Create)
 
 }  // namespace op
 }  // namespace mxnet

From f4f87bd11e5515505548aae63cbf5f1b2872fd10 Mon Sep 17 00:00:00 2001
From: Zhennan Qin <zhennan.qin@intel.com>
Date: Wed, 20 Mar 2019 14:31:26 +0800
Subject: [PATCH 06/11] Change doc

---
 docs/tutorials/c++/subgraphAPI.md | 44 ++++++++++++++++++++++++++++---
 1 file changed, 41 insertions(+), 3 deletions(-)

diff --git a/docs/tutorials/c++/subgraphAPI.md b/docs/tutorials/c++/subgraphAPI.md
index b834df8741b5..07d5829e9ed9 100644
--- a/docs/tutorials/c++/subgraphAPI.md
+++ b/docs/tutorials/c++/subgraphAPI.md
@@ -97,23 +97,61 @@ class SgProperty : public SubgraphProperty {
     return n;
   }
   SubgraphSelectorPtr CreateSubgraphSelector() const override {
-    return std::make_shared<SgSelector>();
+    auto property = std::make_shared<CreateSubgraphSelector>();
+    property->SetAttr<std::string>("property_name", name_);
+    property->SetAttr<bool>("inference_only", true); // Optional, only for inference_only pass.
+    return property;
   }
+
+ private:
+  constexpr static char* name_ = "subgraph example pass";
+
 };
 ```
+`SetAttr` is optional and developer can define their own attributes to control property behavior.
+There're 2 built-in attributes that used by MXNet executor.
+
+`property_name`  : std::string, name of this property.
+
+`inference_only` : bool, apply this property only for inference. Property will be skiped when need_grad=True. Default `false` if this attribute isn't defined.
 
-After defining the subgraph property, we need to register it.
+After defining the subgraph property, we need to register it in .cc file.
 
 ```C++
 MXNET_REGISTER_SUBGRAPH_PROPERTY(SgTest, SgProperty);
 ```
 
-After compiling this subgraph mechanism into MXNet, we can use the environment variable `MXNET_SUBGRAPH_BACKEND` to activate it.
+It's possible to register multiple properties for same backend. In practice, we recommend to put each property definition into .h file, and register backend in single .cc file. Property will be executed according to the register order.
+
+```C++
+#include "SgProperty.h" // Define SgProperty class
+#include "SgProperty2.h" // Define SgProperty2 class
+#include "SgProperty3.h" // Define SgProperty3 class
+
+MXNET_REGISTER_SUBGRAPH_PROPERTY(SgTest, SgProperty);  // Execution order 1.
+MXNET_REGISTER_SUBGRAPH_PROPERTY(SgTest, SgProperty2); // Execution order 2.
+MXNET_REGISTER_SUBGRAPH_PROPERTY(SgTest, SgProperty3); // Execution order 3.
+```
+
+After compiling this subgraph mechanism into MXNet, we can use the environment variable `MXNET_SUBGRAPH_BACKEND` to activate it during symbol bind.
 
 ```bash
 export MXNET_SUBGRAPH_BACKEND=SgTest
 ```
 
+Or you can use python symbol API `get_backend_symbol` to run all properties registered for this backend and get returned symbol.
+
+```Python
+sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch)
+sym = sym.get_backend_symbol('SgTest')
+```
+
+When `SgProperty` is activated, a message will be shown in terminal as
+
+```bash
+start to execute subgraph example pass.
+```
+
 This tutorial shows a simple example of how to use the subgraph API to search for patterns in an NNVM graph.
 Intested users can try different pattern matching rules (i.e., define their own `SubgraphSelector`) and
 attach different operators to execute the subgraphs.

From 347fd99f2006d09115af025e3a0e96105940f727 Mon Sep 17 00:00:00 2001
From: Zhennan Qin <zhennan.qin@intel.com>
Date: Wed, 20 Mar 2019 15:02:10 +0800
Subject: [PATCH 07/11] Fix name

---
 docs/tutorials/c++/subgraphAPI.md                        | 6 +-----
 .../subgraph/mkldnn/mkldnn_conv_post_quantize_property.h | 9 +++------
 src/operator/subgraph/mkldnn/mkldnn_conv_property.h      | 6 +++---
 .../subgraph/mkldnn/mkldnn_fc_post_quantize_property.h   | 4 ++--
 src/operator/subgraph/mkldnn/mkldnn_fc_property.h        | 6 +++---
 5 files changed, 12 insertions(+), 19 deletions(-)

diff --git a/docs/tutorials/c++/subgraphAPI.md b/docs/tutorials/c++/subgraphAPI.md
index 07d5829e9ed9..6b1b477b8021 100644
--- a/docs/tutorials/c++/subgraphAPI.md
+++ b/docs/tutorials/c++/subgraphAPI.md
@@ -98,14 +98,10 @@ class SgProperty : public SubgraphProperty {
   }
   SubgraphSelectorPtr CreateSubgraphSelector() const override {
     auto property = std::make_shared<CreateSubgraphSelector>();
-    property->SetAttr<std::string>("property_name", name_);
+    property->SetAttr<std::string>("property_name", "subgraph example pass"); // Optional, better to have it.
     property->SetAttr<bool>("inference_only", true); // Optional, only for inference_only pass.
     return property;
   }
-
- private:
-  constexpr static char* name_ = "subgraph example pass";
-
 };
 ```
 `SetAttr` is optional and developer can define their own attributes to control property behavior.
diff --git a/src/operator/subgraph/mkldnn/mkldnn_conv_post_quantize_property.h b/src/operator/subgraph/mkldnn/mkldnn_conv_post_quantize_property.h
index 5ee23db2a5ab..f9033f48d413 100644
--- a/src/operator/subgraph/mkldnn/mkldnn_conv_post_quantize_property.h
+++ b/src/operator/subgraph/mkldnn/mkldnn_conv_post_quantize_property.h
@@ -102,8 +102,9 @@ class SgMKLDNNConvPostQuantizeProperty : public SubgraphProperty {
   SgMKLDNNConvPostQuantizeProperty() {}
 
   static SubgraphPropertyPtr Create() {
+    static const std::string &name = "MKLDNN Convolution post-quantization optimization pass";
     auto property = std::make_shared<SgMKLDNNConvPostQuantizeProperty>();
-    property->SetAttr<std::string>("property_name", name_);
+    property->SetAttr<std::string>("property_name", name);
     property->SetAttr<bool>("inference_only", true);
     return property;
   }
@@ -136,8 +137,7 @@ class SgMKLDNNConvPostQuantizeProperty : public SubgraphProperty {
   }
 
   SubgraphSelectorPtr CreateSubgraphSelector() const override {
-    auto selector =
-        std::make_shared<SgMKLDNNConvPostQuantizeSelector>();
+    auto selector = std::make_shared<SgMKLDNNConvPostQuantizeSelector>();
     return selector;
   }
 
@@ -149,9 +149,6 @@ class SgMKLDNNConvPostQuantizeProperty : public SubgraphProperty {
       *entry_ptr = nnvm::NodeEntry{n, entry_ptr->index, 0};
     }
   }
-
- private:
-  constexpr static char *name_ = "MKLDNN Convolution post-quantization optimization pass";
 };
 
 }  // namespace op
diff --git a/src/operator/subgraph/mkldnn/mkldnn_conv_property.h b/src/operator/subgraph/mkldnn/mkldnn_conv_property.h
index 5837fa8e905b..4d6242f35c70 100644
--- a/src/operator/subgraph/mkldnn/mkldnn_conv_property.h
+++ b/src/operator/subgraph/mkldnn/mkldnn_conv_property.h
@@ -147,12 +147,13 @@ class SgMKLDNNConvProperty : public SubgraphProperty {
     disable_all = disable_conv_bn && disable_conv_relu && disable_conv_sum;
   }
   static SubgraphPropertyPtr Create() {
+    static const std::string &name = "MKLDNN convolution optimization pass";
     if (dmlc::GetEnv("MXNET_DISABLE_MKLDNN_CONV_OPT", 0)) {
-      LOG(INFO) << name_ << "is disabled.";
+      LOG(INFO) << name << "is disabled.";
       return nullptr;
     }
     auto property = std::make_shared<SgMKLDNNConvProperty>();
-    property->SetAttr<std::string>("property_name", name_);
+    property->SetAttr<std::string>("property_name", name);
     property->SetAttr<bool>("inference_only", true);
     return property;
   }
@@ -240,7 +241,6 @@ class SgMKLDNNConvProperty : public SubgraphProperty {
   }
 
  private:
-  constexpr static char* name_ = "MKLDNN Convolution optimization pass";
   int disable_all;
   int disable_conv_bn;
   int disable_conv_relu;
diff --git a/src/operator/subgraph/mkldnn/mkldnn_fc_post_quantize_property.h b/src/operator/subgraph/mkldnn/mkldnn_fc_post_quantize_property.h
index a241d54cdd48..f8d7ee1da6c9 100644
--- a/src/operator/subgraph/mkldnn/mkldnn_fc_post_quantize_property.h
+++ b/src/operator/subgraph/mkldnn/mkldnn_fc_post_quantize_property.h
@@ -142,8 +142,9 @@ class SgMKLDNNFCPostQuantizeProperty : public SubgraphProperty {
   }
 
   static SubgraphPropertyPtr Create() {
+    static const std::string &name = "MKLDNN FullyConected post-quantization optimization pass";
     auto property = std::make_shared<SgMKLDNNFCPostQuantizeProperty>();
-    property->SetAttr<std::string>("property_name", name_);
+    property->SetAttr<std::string>("property_name", name);
     property->SetAttr<bool>("inference_only", true);
     return property;
   }
@@ -203,7 +204,6 @@ class SgMKLDNNFCPostQuantizeProperty : public SubgraphProperty {
   }
 
  private:
-  constexpr static char* name_ = "MKLDNN FullyConected post-quantization optimization pass";
   bool disable_fuse_all;
   bool disable_float_output;
 };
diff --git a/src/operator/subgraph/mkldnn/mkldnn_fc_property.h b/src/operator/subgraph/mkldnn/mkldnn_fc_property.h
index a2deb90a2679..8f9b5a468d8f 100644
--- a/src/operator/subgraph/mkldnn/mkldnn_fc_property.h
+++ b/src/operator/subgraph/mkldnn/mkldnn_fc_property.h
@@ -124,12 +124,13 @@ class SgMKLDNNFCProperty : public SubgraphProperty {
   }
 
   static SubgraphPropertyPtr Create() {
+    static const std::string &name = "MKLDNN FullyConnected optimization pass";
     if (dmlc::GetEnv("MXNET_DISABLE_MKLDNN_FC_OPT", 0)) {
-      LOG(INFO) << name_ << "is disabled.";
+      LOG(INFO) << name << "is disabled.";
       return nullptr;
     }
     auto property = std::make_shared<SgMKLDNNFCProperty>();
-    property->SetAttr<std::string>("property_name", name_);
+    property->SetAttr<std::string>("property_name", name);
     property->SetAttr<bool>("inference_only", true);
     return property;
   }
@@ -179,7 +180,6 @@ class SgMKLDNNFCProperty : public SubgraphProperty {
   }
 
  private:
-  constexpr static char* name_ = "MKLDNN FullyConnected optimization pass";
   bool disable_fc_relu;
 };
 

From 6abd1820167f7a93440b27c01d3858ec209d9d63 Mon Sep 17 00:00:00 2001
From: Zhennan Qin <zhennan.qin@intel.com>
Date: Wed, 20 Mar 2019 20:29:18 +0800
Subject: [PATCH 08/11] Run CI


From 26eef52f9b0cdad61c019def0a0ba1e83a148da6 Mon Sep 17 00:00:00 2001
From: Zhennan Qin <zhennan.qin@intel.com>
Date: Thu, 21 Mar 2019 09:44:10 +0800
Subject: [PATCH 09/11] Add env var in doc

---
 MKLDNN_README.md                                    | 8 ++++++++
 src/operator/subgraph/mkldnn/mkldnn_conv_property.h | 2 +-
 src/operator/subgraph/mkldnn/mkldnn_fc_property.h   | 2 +-
 src/operator/subgraph/subgraph_property.h           | 8 ++++----
 4 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/MKLDNN_README.md b/MKLDNN_README.md
index 214fc83985fb..43f8dee54293 100644
--- a/MKLDNN_README.md
+++ b/MKLDNN_README.md
@@ -306,6 +306,14 @@ Graph optimization by subgraph feature are available in master branch. You can b
 ```
 export MXNET_SUBGRAPH_BACKEND=MKLDNN
 ```
+
+When `MKLDNN` backend is enabled, advanced control options are avaliable:
+
+```
+export MXNET_DISABLE_MKLDNN_CONV_OPT=1 # disable MKLDNN convolution optimization pass
+export MXNET_DISABLE_MKLDNN_FC_OPT=1   # disable MKLDNN FullyConnected optimization pass
+```
+
 
 This limitations of this experimental feature are:
 
diff --git a/src/operator/subgraph/mkldnn/mkldnn_conv_property.h b/src/operator/subgraph/mkldnn/mkldnn_conv_property.h
index 4d6242f35c70..7fe4727a4990 100644
--- a/src/operator/subgraph/mkldnn/mkldnn_conv_property.h
+++ b/src/operator/subgraph/mkldnn/mkldnn_conv_property.h
@@ -149,7 +149,7 @@ class SgMKLDNNConvProperty : public SubgraphProperty {
   static SubgraphPropertyPtr Create() {
     static const std::string &name = "MKLDNN convolution optimization pass";
     if (dmlc::GetEnv("MXNET_DISABLE_MKLDNN_CONV_OPT", 0)) {
-      LOG(INFO) << name << "is disabled.";
+      LOG(INFO) << name << " is disabled.";
       return nullptr;
     }
     auto property = std::make_shared<SgMKLDNNConvProperty>();
diff --git a/src/operator/subgraph/mkldnn/mkldnn_fc_property.h b/src/operator/subgraph/mkldnn/mkldnn_fc_property.h
index 8f9b5a468d8f..04e140c72d86 100644
--- a/src/operator/subgraph/mkldnn/mkldnn_fc_property.h
+++ b/src/operator/subgraph/mkldnn/mkldnn_fc_property.h
@@ -126,7 +126,7 @@ class SgMKLDNNFCProperty : public SubgraphProperty {
   static SubgraphPropertyPtr Create() {
     static const std::string &name = "MKLDNN FullyConnected optimization pass";
     if (dmlc::GetEnv("MXNET_DISABLE_MKLDNN_FC_OPT", 0)) {
-      LOG(INFO) << name << "is disabled.";
+      LOG(INFO) << name << " is disabled.";
       return nullptr;
     }
     auto property = std::make_shared<SgMKLDNNFCProperty>();
diff --git a/src/operator/subgraph/subgraph_property.h b/src/operator/subgraph/subgraph_property.h
index 8b87c8bedec0..813fbd34f63a 100644
--- a/src/operator/subgraph/subgraph_property.h
+++ b/src/operator/subgraph/subgraph_property.h
@@ -204,13 +204,13 @@ class SubgraphPropertyRegistry {
 typedef dmlc::ThreadLocalStore<std::unordered_map<std::string, std::unordered_set<std::string>>>
   SubgraphPropertyOpNameSet;
 
-#define DECLEARPROPERTY1(NAME, SubgraphPropertyType, X) \
+#define DECLAREPROPERTY1(NAME, SubgraphPropertyType, X) \
   DMLC_ATTRIBUTE_UNUSED auto __make_##SubgraphPropertyType##_##Name##_##X##__
-#define DECLEARPROPERTY(NAME, SubgraphPropertyType, X) \
-  DECLEARPROPERTY1(NAME, SubgraphPropertyType, X)
+#define DECLAREPROPERTY(NAME, SubgraphPropertyType, X) \
+  DECLAREPROPERTY1(NAME, SubgraphPropertyType, X)
 
 #define MXNET_REGISTER_SUBGRAPH_PROPERTY(Name, SubgraphPropertyType) \
-  static DECLEARPROPERTY(Name, SubgraphPropertyType, __LINE__) =     \
+  static DECLAREPROPERTY(Name, SubgraphPropertyType, __LINE__) =     \
       SubgraphPropertyRegistry::Get()->__REGISTER__(#Name, &SubgraphPropertyType::Create)
 
 }  // namespace op

From 2745620cde331f21b107f97dcf2730fa02affd3a Mon Sep 17 00:00:00 2001
From: Zhennan Qin <zhennan.qin@intel.com>
Date: Sat, 23 Mar 2019 08:32:50 +0800
Subject: [PATCH 10/11] Address comments.

---
 src/executor/graph_executor.cc            |  3 ++-
 src/operator/subgraph/subgraph_property.h | 10 +++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc
index 53b5941778c9..28862a49fae1 100644
--- a/src/executor/graph_executor.cc
+++ b/src/executor/graph_executor.cc
@@ -1522,7 +1522,8 @@ static nnvm::Symbol PartitionGraph(const nnvm::Symbol& src,
         auto full_name = subgraph_prop->HasAttr("property_name")
                              ? subgraph_prop->GetAttr<std::string>("property_name")
                              : prop_name;
-        LOG(INFO) << "Skip subgraph " << full_name << " as it requires `grad_req=null`.";
+        LOG(INFO) << "skip partitioning graph with subgraph property " << full_name
+                  << " as it requires `grad_req=null`.";
         continue;
       }
     }
diff --git a/src/operator/subgraph/subgraph_property.h b/src/operator/subgraph/subgraph_property.h
index 813fbd34f63a..4c05c089426f 100644
--- a/src/operator/subgraph/subgraph_property.h
+++ b/src/operator/subgraph/subgraph_property.h
@@ -204,13 +204,13 @@ class SubgraphPropertyRegistry {
 typedef dmlc::ThreadLocalStore<std::unordered_map<std::string, std::unordered_set<std::string>>>
   SubgraphPropertyOpNameSet;
 
-#define DECLAREPROPERTY1(NAME, SubgraphPropertyType, X) \
-  DMLC_ATTRIBUTE_UNUSED auto __make_##SubgraphPropertyType##_##Name##_##X##__
-#define DECLAREPROPERTY(NAME, SubgraphPropertyType, X) \
-  DECLAREPROPERTY1(NAME, SubgraphPropertyType, X)
+#define DECLARE_PROPERTY_EX(NAME, SubgraphPropertyType, X) \
+  static const DMLC_ATTRIBUTE_UNUSED auto __make_##SubgraphPropertyType##_##Name##_##X##__
+#define DECLARE_PROPERTY(NAME, SubgraphPropertyType, X) \
+  DECLARE_PROPERTY_EX(NAME, SubgraphPropertyType, X)
 
 #define MXNET_REGISTER_SUBGRAPH_PROPERTY(Name, SubgraphPropertyType) \
-  static DECLAREPROPERTY(Name, SubgraphPropertyType, __LINE__) =     \
+  DECLARE_PROPERTY(Name, SubgraphPropertyType, __LINE__) =           \
       SubgraphPropertyRegistry::Get()->__REGISTER__(#Name, &SubgraphPropertyType::Create)
 
 }  // namespace op

From 695b3ed6f0842dbcd225daebc411c159bfc73792 Mon Sep 17 00:00:00 2001
From: Zhennan Qin <zhennan.qin@intel.com>
Date: Sat, 23 Mar 2019 10:09:56 +0800
Subject: [PATCH 11/11] run CI