diff --git a/nnvm b/nnvm
index 0d64855f741e..ddf3c17e3455 160000
--- a/nnvm
+++ b/nnvm
@@ -1 +1 @@
-Subproject commit 0d64855f741e0482be7a3dfecde05f290bfec85d
+Subproject commit ddf3c17e3455db9cd10f5b18bc9753a146971819
diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc
index 8c7aaa7311d1..a02df3da16e1 100644
--- a/src/executor/graph_executor.cc
+++ b/src/executor/graph_executor.cc
@@ -91,26 +91,31 @@ nnvm::NodeEntry AggregateGradient(std::vector<nnvm::NodeEntry>&& v) {
   static const Op* ewise_sum_op = Op::Get("ElementWiseSum");
   static const Op* identity_op = Op::Get("identity");
   static const Op* zeros_op = Op::Get("_zeros");
-  // remove zero in the sum.
+  static const Op* zeros_like_op = Op::Get("zeros_like");
+
+  if (v.size() == 0) {
+    nnvm::NodePtr ng = nnvm::Node::Create();
+    ng->attrs.op = zeros_op;
+    ng->attrs.name = "zeros";
+    ng->attrs.op->attr_parser(&(ng->attrs));
+    return nnvm::NodeEntry{ng, 0, 0};
+  }
+
+  // remove zero in the sum. at least keep 1.
   size_t begin = 0;
   for (size_t i = 0; i < v.size(); ++i) {
-    if (v[i].node->op() != zeros_op) {
+    if (v[i].node->op() != zeros_op && v[i].node->op() != zeros_like_op) {
       if (begin != i) {
         v[begin] = std::move(v[i]);
       }
       ++begin;
     }
   }
+  if (begin == 0) begin = 1;
   v.resize(begin);
 
   if (v.size() == 1) {
     return std::move(v[0]);
-  } else if (v.size() == 0) {
-    nnvm::NodePtr ng = nnvm::Node::Create();
-    ng->attrs.op = zeros_op;
-    ng->attrs.name = "zeros";
-    ng->attrs.op->attr_parser(&(ng->attrs));
-    return nnvm::NodeEntry{ng, 0, 0};
   } else {
     if (v.size() < inplace_sum_cap) {
       nnvm::NodePtr sum_node = nnvm::Node::Create();
@@ -216,10 +221,16 @@ nnvm::Graph GraphExecutor::InitFullGraph(
     if (type == "CuDNNBatchNorm") return false;
     return true;
   };
+
+  std::vector<const nnvm::Op*> zero_ops;
+  zero_ops.push_back(nnvm::Op::Get("zeros_like"));
+  zero_ops.push_back(nnvm::Op::Get("_zeros"));
+
   // take gradient
   nnvm::Graph g_grad = nnvm::pass::Gradient(
       g, symbol.outputs, xs, head_grad_entry_,
-      AggregateGradient, need_mirror);
+      AggregateGradient, need_mirror, nullptr,
+      zero_ops);
   CHECK_EQ(g_grad.outputs.size(), xs.size());
   for (const auto &e : g_grad.outputs) {
     g.outputs.push_back(e);
diff --git a/src/initialize.cc b/src/initialize.cc
index e2d337390701..607b24493f8a 100644
--- a/src/initialize.cc
+++ b/src/initialize.cc
@@ -3,6 +3,7 @@
  * \file initialize.cc
  * \brief initialize mxnet library
  */
+#include <signal.h>
 #include <dmlc/logging.h>
 #include <mxnet/engine.h>
 
@@ -10,10 +11,31 @@
 
 namespace mxnet {
 
+void segfault_logger(int sig) {
+  const int MAX_STACK_SIZE = 10;
+  void *stack[MAX_STACK_SIZE];
+
+  fprintf(stderr, "\nSegmentation fault: %d\n\n", sig);
+
+#if DMLC_LOG_STACK_TRACE
+  int nframes = backtrace(stack, MAX_STACK_SIZE);
+  fprintf(stderr, "Stack trace returned %d entries:\n", nframes);
+  char **msgs = backtrace_symbols(stack, nframes);
+  if (msgs != nullptr) {
+    for (int i = 0; i < nframes; ++i) {
+      fprintf(stderr, "[bt] (%d) %s\n", i, msgs[i]);
+    }
+  }
+#endif  // DMLC_LOG_STACK_TRACE
+
+  exit(1);
+}
+
 class LibraryInitializer {
  public:
   LibraryInitializer() {
     dmlc::InitLogging("mxnet");
+    signal(SIGSEGV, segfault_logger);
 #if MXNET_USE_PROFILER
     // ensure profiler's constructor are called before atexit.
     engine::Profiler::Get();
@@ -26,8 +48,12 @@ class LibraryInitializer {
     });
 #endif
   }
+
+  static LibraryInitializer* Get();
 };
 
-static LibraryInitializer __library_init;
+LibraryInitializer* LibraryInitializer::Get() {
+  static LibraryInitializer inst;
+  return &inst;
+}
 }  // namespace mxnet
-
diff --git a/src/operator/elemwise_op_common.h b/src/operator/elemwise_op_common.h
index 693222bba9d1..129cb3360dce 100644
--- a/src/operator/elemwise_op_common.h
+++ b/src/operator/elemwise_op_common.h
@@ -74,11 +74,7 @@ struct ElemwiseGradUseIn {
   const char *op_name;
   std::vector<nnvm::NodeEntry> operator()(const nnvm::NodePtr& n,
                                           const std::vector<nnvm::NodeEntry>& ograds) {
-    std::vector<nnvm::NodeEntry> heads(ograds.begin(), ograds.end());
-    for (auto& h : n->inputs) {
-      heads.push_back(h);
-    }
-    return MakeGradNode(op_name, n, heads, n->attrs.dict);
+    return MakeNonlossGradNode(op_name, n, ograds, n->inputs, n->attrs.dict);
   }
 };
 
@@ -87,12 +83,12 @@ struct ElemwiseGradUseOut {
   const char *op_name;
   std::vector<nnvm::NodeEntry> operator()(const nnvm::NodePtr& n,
                                           const std::vector<nnvm::NodeEntry>& ograds) {
-    std::vector<nnvm::NodeEntry> heads(ograds.begin(), ograds.end());
+    std::vector<nnvm::NodeEntry> heads;
     index_t n_out = n->num_outputs();
     for (index_t i = 0; i < n_out; ++i) {
       heads.emplace_back(nnvm::NodeEntry{n, i, 0});
     }
-    return MakeGradNode(op_name, n, heads, n->attrs.dict);
+    return MakeNonlossGradNode(op_name, n, ograds, heads, n->attrs.dict);
   }
 };
 
@@ -101,7 +97,7 @@ struct ElemwiseGradUseNone {
   const char *op_name;
   std::vector<nnvm::NodeEntry> operator()(const nnvm::NodePtr& n,
                                           const std::vector<nnvm::NodeEntry>& ograds) {
-    return MakeGradNode(op_name, n, ograds, n->attrs.dict);
+    return MakeNonlossGradNode(op_name, n, ograds, {}, n->attrs.dict);
   }
 };
 
diff --git a/src/operator/operator_common.h b/src/operator/operator_common.h
index d03fac5121ee..862c39b3b9b4 100644
--- a/src/operator/operator_common.h
+++ b/src/operator/operator_common.h
@@ -171,21 +171,42 @@ inline bool type_assign(int *y, const int& x) {
 #endif
 
 
-// quick helper to make node
-inline std::vector<nnvm::NodeEntry> MakeGradNode(
-    const char* op_name,
-    const nnvm::NodePtr& n,
-    std::vector<nnvm::NodeEntry> inputs,
-    std::unordered_map<std::string, std::string> dict) {
-  nnvm::NodePtr p = nnvm::Node::Create();
+// make a new node with operator op_name. Inputs are not filled.
+inline nnvm::NodePtr MakeNode(
+    const char* op_name, const std::string& name,
+    std::vector<nnvm::NodeEntry> const * inputs,
+    std::unordered_map<std::string, std::string> const * dict,
+    nnvm::NodePtr const * fwd_node) {
+  auto p = nnvm::Node::Create();
   p->attrs.op = nnvm::Op::Get(op_name);
-  p->attrs.name = n->attrs.name + "_backward";
-  p->attrs.dict = std::move(dict);
+  p->attrs.name = name;
+  if (dict != nullptr) p->attrs.dict = *dict;
+  if (inputs != nullptr) p->inputs = *inputs;
+  if (fwd_node != nullptr) {
+    p->control_deps.emplace_back(*fwd_node);
+  }
   if (p->op()->attr_parser != nullptr) {
     p->op()->attr_parser(&(p->attrs));
   }
-  p->control_deps.emplace_back(n);
-  p->inputs = std::move(inputs);
+  return p;
+}
+
+inline nnvm::NodePtr MakeNode(
+    const char* op_name, const std::string& name,
+    const std::vector<nnvm::NodeEntry>& inputs,
+    std::unordered_map<std::string, std::string> const * dict,
+    nnvm::NodePtr const * fwd_node) {
+  return MakeNode(op_name, name, &inputs, dict, fwd_node);
+}
+
+
+// quick helper to make node
+inline std::vector<nnvm::NodeEntry> MakeGradNode(
+    const char* op_name, const nnvm::NodePtr& n,
+    const std::vector<nnvm::NodeEntry>& inputs,
+    const std::unordered_map<std::string, std::string>& dict) {
+  auto p = MakeNode(op_name, n->attrs.name + "_backward",
+                    &inputs, &dict, &n);
   std::vector<nnvm::NodeEntry> ret;
   for (index_t i = 0; i < p->num_outputs(); ++i) {
     ret.emplace_back(nnvm::NodeEntry{p, i, 0});
@@ -199,22 +220,50 @@ inline std::vector<nnvm::NodeEntry> MakeZeroGradNodes(
     const std::vector<nnvm::NodeEntry>& ograds) {
   std::vector<nnvm::NodeEntry> ret;
   for (index_t i = 0; i < n->num_inputs(); ++i) {
-    nnvm::NodePtr p = nnvm::Node::Create();
-    p->attrs.op = nnvm::Op::Get("_zeros");
     std::ostringstream os;
     if (1 == n->num_inputs()) {
       os << n->attrs.name << "_backward";
     } else {
       os << n->attrs.name << "_in" << i << "_backward";
     }
-    p->attrs.name = os.str();
-    p->attrs.dict = std::unordered_map<std::string, std::string>();
-    p->control_deps.emplace_back(n);
+    auto p = MakeNode("zeros_like", os.str(), {n->inputs[i]}, nullptr, &n);
     ret.emplace_back(nnvm::NodeEntry{p, 0, 0});
   }
   return ret;
 }
 
+
+// check whether all output grads are zero.
+inline bool CheckGradAllZero(const std::vector<nnvm::NodeEntry>& ograds) {
+  const auto zero_op = nnvm::Op::Get("_zeros");
+  const auto zero_like_op = nnvm::Op::Get("zeros_like");
+  if (!ograds.size()) return false;
+  for (const auto& grad : ograds) {
+    if (!grad.node) return false;
+    if (grad.node->op() != zero_op && grad.node->op() != zero_like_op ) return false;
+  }
+  return true;
+}
+
+// make gradient node that doesn't add to objective.
+// i.e. igrads are always zero when ograds are zero.
+inline std::vector<nnvm::NodeEntry> MakeNonlossGradNode(
+    const char* op_name, const nnvm::NodePtr& n,
+    const std::vector<nnvm::NodeEntry>& ograds,
+    const std::vector<nnvm::NodeEntry>& inputs,
+    const std::unordered_map<std::string, std::string> dict) {
+  if (CheckGradAllZero(ograds)) return MakeZeroGradNodes(n, ograds);
+  auto p = MakeNode(op_name, n->attrs.name + "_backward",
+                    nullptr, &dict, &n);
+  p->inputs.insert(p->inputs.end(), ograds.begin(), ograds.end());
+  p->inputs.insert(p->inputs.end(), inputs.begin(), inputs.end());
+  std::vector<nnvm::NodeEntry> ret;
+  for (index_t i = 0; i < p->num_outputs(); ++i) {
+    ret.emplace_back(nnvm::NodeEntry{p, i, 0});
+  }
+  return ret;
+}
+
 /*! \brief Parse keyword arguments as PType arguments and save to parsed */
 template<typename PType>
 inline void ParamParser(nnvm::NodeAttrs* attrs) {
diff --git a/src/operator/tensor/broadcast_reduce_op.h b/src/operator/tensor/broadcast_reduce_op.h
index be592a96cacc..d40776369845 100644
--- a/src/operator/tensor/broadcast_reduce_op.h
+++ b/src/operator/tensor/broadcast_reduce_op.h
@@ -458,9 +458,9 @@ struct ReduceGrad {
   const char *op_name;
   std::vector<nnvm::NodeEntry> operator()(const nnvm::NodePtr& n,
                                           const std::vector<nnvm::NodeEntry>& ograds) {
-    return MakeGradNode(
+    return MakeNonlossGradNode(
         op_name, n,
-        {ograds[0], n->inputs[0], nnvm::NodeEntry{n, 0, 0}},
+        ograds, {n->inputs[0], nnvm::NodeEntry{n, 0, 0}},
         n->attrs.dict);
   }
 };
@@ -646,8 +646,8 @@ void PickOpBackward(const nnvm::NodeAttrs& attrs,
   .set_attr<nnvm::FGradient>("FGradient",                       \
     [](const nnvm::NodePtr& n,                                  \
        const std::vector<nnvm::NodeEntry>& ograds) {            \
-      return MakeGradNode("_broadcast_backward", n, ograds,     \
-                          {{"keepdims", "true"}});              \
+      return MakeNonlossGradNode("_broadcast_backward", n, ograds, {},    \
+                                 {{"keepdims", "true"}});              \
     })                                                          \
   .add_argument("data", "ndarray-or-symbol", "The input")
 
diff --git a/src/operator/tensor/broadcast_reduce_op_index.cc b/src/operator/tensor/broadcast_reduce_op_index.cc
index ffa335322f48..3cb5ed64a05c 100644
--- a/src/operator/tensor/broadcast_reduce_op_index.cc
+++ b/src/operator/tensor/broadcast_reduce_op_index.cc
@@ -45,16 +45,11 @@ NNVM_REGISTER_OP(pick)
 .set_attr<FCompute>("FCompute<cpu>", PickOpForward<cpu>)
 .set_attr<nnvm::FGradient>("FGradient",
   [](const nnvm::NodePtr& n, const std::vector<nnvm::NodeEntry>& ograds) {
-    std::vector<nnvm::NodeEntry> heads(ograds.begin(), ograds.end());
-    heads.push_back(n->inputs[1]);
-    auto ret = MakeGradNode("_backward_pick", n, heads, n->attrs.dict);
-
-    nnvm::NodePtr p = nnvm::Node::Create();
-    p->attrs.op = nnvm::Op::Get("_zeros");
-    p->attrs.name = n->attrs.name + "_index_backward";
-    p->control_deps.emplace_back(n);
+    auto ret = MakeNonlossGradNode("_backward_pick", n, ograds,
+                                   {n->inputs[1]}, n->attrs.dict);
+    auto p = MakeNode("zeros_like", n->attrs.name + "_index_backward",
+                      {n->inputs[1]}, nullptr, &n);
     ret.emplace_back(nnvm::NodeEntry{p, 0, 0});
-
     return ret;
   })
 .add_argument("data", "NDArray", "Source input")
diff --git a/src/operator/tensor/control_flow_op.cc b/src/operator/tensor/control_flow_op.cc
index 8332327de8c0..ab6f4a4c29ca 100644
--- a/src/operator/tensor/control_flow_op.cc
+++ b/src/operator/tensor/control_flow_op.cc
@@ -35,13 +35,8 @@ NNVM_REGISTER_OP(where)
   [](const nnvm::NodePtr& n, const std::vector<nnvm::NodeEntry>& ograds) {
     std::vector<nnvm::NodeEntry> ret;
     // make zero grad node for grad[condition]
-    nnvm::NodePtr p = nnvm::Node::Create();
-    p->attrs.op = nnvm::Op::Get("_zeros");
-    std::ostringstream os;
-    os << n->attrs.name << "_in" << 0 << "_backward";
-    p->attrs.name = os.str();
-    p->attrs.dict = std::unordered_map<std::string, std::string>();
-    p->control_deps.emplace_back(n);
+    auto p = MakeNode("zeros_like", n->attrs.name + "_cond_backward",
+                      {n->inputs[0]}, nullptr, &n);
     ret.emplace_back(nnvm::NodeEntry{p, 0, 0});
 
     // make grad nodes for grad[x] and grad[y]
diff --git a/src/operator/tensor/elemwise_unary_op.cc b/src/operator/tensor/elemwise_unary_op.cc
index 02d80d65d1c9..5389acb77893 100644
--- a/src/operator/tensor/elemwise_unary_op.cc
+++ b/src/operator/tensor/elemwise_unary_op.cc
@@ -41,13 +41,8 @@ MXNET_OPERATOR_REGISTER_UNARY(make_loss)
 .set_attr<FCompute>("FCompute<cpu>", IdentityCompute<cpu>)
 .set_attr<nnvm::FGradient>("FGradient",
   [](const nnvm::NodePtr& n, const std::vector<nnvm::NodeEntry>& ograds) {
-    nnvm::NodePtr p = nnvm::Node::Create();
-    p->attrs.op = nnvm::Op::Get("_ones");
-    p->attrs.name = n->attrs.name + "_backward";
-    p->control_deps.emplace_back(n);
-    if (p->op()->attr_parser != nullptr) {
-      p->op()->attr_parser(&(p->attrs));
-    }
+    auto p = MakeNode("ones_like", n->attrs.name + "_backward",
+                      &(n->inputs), nullptr, &n);
     std::vector<nnvm::NodeEntry> ret;
     ret.emplace_back(nnvm::NodeEntry{p, 0, 0});
     return ret;
@@ -60,16 +55,18 @@ NNVM_REGISTER_OP(_identity_with_attr_like_rhs)
     "FInplaceOption", [](const NodeAttrs& attrs) {
       return std::vector<std::pair<int, int> >{{0, 0}};
     })
+.set_attr<nnvm::FIgnoreInputs>("FIgnoreInputs",
+    [](const NodeAttrs& attrs) { return std::vector<uint32_t>(1, 1); })
 .set_attr<FCompute>("FCompute<cpu>", IdentityCompute<cpu>)
 .set_attr<nnvm::FInferShape>("FInferShape", ElemwiseShape<2, 1>)
 .set_attr<nnvm::FGradient>(
     "FGradient",  [](const nnvm::NodePtr& n,
                      const std::vector<nnvm::NodeEntry>& ograds) {
-      auto lhs = MakeGradNode("_backward_copy", n, ograds,
-                              std::unordered_map<std::string, std::string>());
-      nnvm::NodePtr ng = nnvm::Node::Create();
-      ng->attrs.op = nnvm::Op::Get("_zeros");
-      ng->attrs.name = "zeros";
+      auto lhs = MakeNonlossGradNode(
+          "_backward_copy", n, ograds, {},
+          std::unordered_map<std::string, std::string>());
+      auto ng = MakeNode("zeros_like", n->attrs.name + "rhs_backward",
+                         {n->inputs[1]}, nullptr, &n);
       lhs.push_back(nnvm::NodeEntry{ng, 0, 0});
       return lhs;
     });
diff --git a/src/operator/tensor/indexing_op.cc b/src/operator/tensor/indexing_op.cc
index 922578d9d27a..c65457e85e64 100644
--- a/src/operator/tensor/indexing_op.cc
+++ b/src/operator/tensor/indexing_op.cc
@@ -33,9 +33,8 @@ NNVM_REGISTER_OP(Embedding)
 .set_attr<FCompute>("FCompute<cpu>", EmbeddingOpForward<cpu>)
 .set_attr<nnvm::FGradient>("FGradient",
   [](const nnvm::NodePtr& n, const std::vector<nnvm::NodeEntry>& ograds) {
-    std::vector<nnvm::NodeEntry> heads(ograds.begin(), ograds.end());
-    heads.push_back(n->inputs[0]);
-    return MakeGradNode("_backward_Embedding", n, heads, n->attrs.dict);
+    return MakeNonlossGradNode("_backward_Embedding", n, ograds,
+                               {n->inputs[0]}, n->attrs.dict);
   })
 .add_argument("data", "Symbol", "Input data to the EmbeddingOp.")
 .add_argument("weight", "Symbol", "Embedding weight matrix.")
@@ -93,9 +92,8 @@ Examples::
 .set_attr<FCompute>("FCompute<cpu>", TakeOpForward<cpu>)
 .set_attr<nnvm::FGradient>("FGradient",
   [](const nnvm::NodePtr& n,  const std::vector<nnvm::NodeEntry>& ograds) {
-    std::vector<nnvm::NodeEntry> heads(ograds.begin(), ograds.end());
-    heads.push_back(n->inputs[1]);
-    return MakeGradNode("_backward_take", n, heads, n->attrs.dict);
+    return MakeNonlossGradNode("_backward_take", n, ograds,
+                               {n->inputs[1]}, n->attrs.dict);
   })
 .add_argument("a", "ndarray-or-symbol", "The source array.")
 .add_argument("indices", "ndarray-or-symbol", "The indices of the values to extract.")
diff --git a/src/operator/tensor/init_op.cc b/src/operator/tensor/init_op.cc
index 88937da2effb..8ee9a5b72f7d 100644
--- a/src/operator/tensor/init_op.cc
+++ b/src/operator/tensor/init_op.cc
@@ -4,6 +4,7 @@
  * \brief CPU Implementation of init op
  */
 #include "./init_op.h"
+#include "./elemwise_unary_op.h"
 
 namespace mxnet {
 namespace op {
@@ -42,5 +43,29 @@ NNVM_REGISTER_OP(_arange)
 .set_attr<FCompute>("FCompute<cpu>", RangeCompute<cpu>)
 .add_arguments(RangeParam::__FIELDS__());
 
+NNVM_REGISTER_OP(zeros_like)
+.MXNET_DESCRIBE("Return an array of zeros with the same shape and type as the input array.")
+.set_num_inputs(1)
+.set_num_outputs(1)
+.set_attr<nnvm::FInferShape>("FInferShape", ElemwiseShape<1, 1>)
+.set_attr<nnvm::FInferType>("FInferType", ElemwiseType<1, 1>)
+.set_attr<nnvm::FIgnoreInputs>("FIgnoreInputs",
+    [](const NodeAttrs& attrs) { return std::vector<uint32_t>(1, 0); })
+.set_attr<FCompute>("FCompute<cpu>", FillCompute<cpu, 0>)
+.set_attr<nnvm::FGradient>("FGradient", MakeZeroGradNodes)
+.add_argument("data", "ndarray-or-symbol", "The input");
+
+NNVM_REGISTER_OP(ones_like)
+.MXNET_DESCRIBE("Return an array of ones with the same shape and type as the input array.")
+.set_num_inputs(1)
+.set_num_outputs(1)
+.set_attr<nnvm::FInferShape>("FInferShape", ElemwiseShape<1, 1>)
+.set_attr<nnvm::FInferType>("FInferType", ElemwiseType<1, 1>)
+.set_attr<nnvm::FIgnoreInputs>("FIgnoreInputs",
+    [](const NodeAttrs& attrs) { return std::vector<uint32_t>(1, 0); })
+.set_attr<FCompute>("FCompute<cpu>", FillCompute<cpu, 1>)
+.set_attr<nnvm::FGradient>("FGradient", MakeZeroGradNodes)
+.add_argument("data", "ndarray-or-symbol", "The input");
+
 }  // namespace op
 }  // namespace mxnet
diff --git a/src/operator/tensor/init_op.cu b/src/operator/tensor/init_op.cu
index 9fa116ac51fb..a798f26db60d 100644
--- a/src/operator/tensor/init_op.cu
+++ b/src/operator/tensor/init_op.cu
@@ -17,5 +17,11 @@ NNVM_REGISTER_OP(_ones)
 NNVM_REGISTER_OP(_arange)
 .set_attr<FCompute>("FCompute<gpu>", RangeCompute<gpu>);
 
+NNVM_REGISTER_OP(zeros_like)
+.set_attr<FCompute>("FCompute<gpu>", FillCompute<gpu, 0>);
+
+NNVM_REGISTER_OP(ones_like)
+.set_attr<FCompute>("FCompute<gpu>", FillCompute<gpu, 1>);
+
 }  // namespace op
 }  // namespace mxnet
diff --git a/src/operator/tensor/matrix_op.cc b/src/operator/tensor/matrix_op.cc
index fdb6e932cfc9..738fd3a39c6f 100644
--- a/src/operator/tensor/matrix_op.cc
+++ b/src/operator/tensor/matrix_op.cc
@@ -149,8 +149,9 @@ Examples::
   [](const nnvm::NodePtr& n, const std::vector<nnvm::NodeEntry>& ograds) {
     const TransposeParam& param = nnvm::get<TransposeParam>(n->attrs.parsed);
     if (param.axes.ndim() == 0) {
-      return MakeGradNode("transpose", n, ograds,
-                          std::unordered_map<std::string, std::string>());
+      return MakeNonlossGradNode(
+          "transpose", n, ograds, {},
+          std::unordered_map<std::string, std::string>());
     } else {
       TShape axes = TShape(param.axes.ndim());
       for (index_t i = 0; i < axes.ndim(); ++i) {
@@ -158,7 +159,9 @@ Examples::
       }
       std::ostringstream os;
       os << axes;
-      return MakeGradNode("transpose", n, ograds, {{"axes", os.str()}});
+      return MakeNonlossGradNode(
+          "transpose", n, ograds,
+          {}, {{"axes", os.str()}});
     }
   })
 .set_attr<FCompute>("FCompute<cpu>", Transpose<cpu>)
diff --git a/src/operator/tensor/ordering_op.cc b/src/operator/tensor/ordering_op.cc
index a35627c6be79..24c18cbf2b0b 100644
--- a/src/operator/tensor/ordering_op.cc
+++ b/src/operator/tensor/ordering_op.cc
@@ -45,12 +45,12 @@ Examples::
   [](const nnvm::NodePtr& n, const std::vector<nnvm::NodeEntry>& ograds) {
     const TopKParam& param = nnvm::get<TopKParam>(n->attrs.parsed);
     if (param.ret_typ == topk_enum::kReturnValue || param.ret_typ == topk_enum::kReturnBoth) {
-      std::vector<nnvm::NodeEntry> heads(ograds.begin(), ograds.begin() + 1);
+      std::vector<nnvm::NodeEntry> inputs;
       index_t n_out = n->num_outputs();
       for (index_t i = 0; i < n_out; ++i) {
-        heads.emplace_back(nnvm::NodeEntry{ n, i, 0 });
+        inputs.emplace_back(nnvm::NodeEntry{ n, i, 0 });
       }
-      return MakeGradNode("_backward_topk", n, heads, n->attrs.dict);
+      return MakeNonlossGradNode("_backward_topk", n, {ograds[0]}, inputs, n->attrs.dict);
     } else {
       return MakeZeroGradNodes(n, ograds);
     }
@@ -107,16 +107,16 @@ Examples::
 .set_attr<nnvm::FGradient>("FGradient",
   [](const nnvm::NodePtr& n, const std::vector<nnvm::NodeEntry>& ograds) {
     const SortParam& param = nnvm::get<SortParam>(n->attrs.parsed);
-    std::vector<nnvm::NodeEntry> heads(ograds.begin(), ograds.begin() + 1);
+    std::vector<nnvm::NodeEntry> inputs;
     index_t n_out = n->num_outputs();
     for (index_t i = 0; i < n_out; ++i) {
-      heads.emplace_back(nnvm::NodeEntry{ n, i, 0 });
+      inputs.emplace_back(nnvm::NodeEntry{ n, i, 0 });
     }
-    return MakeGradNode("_backward_topk", n, heads,
-                         {{"axis", n->attrs.dict["axis"]},
-                          {"k", "0"},
-                          {"ret_typ", "value"},
-                          {"is_ascend", std::to_string(param.is_ascend)}});
+    return MakeNonlossGradNode("_backward_topk", n, {ograds[0]}, inputs,
+                               {{"axis", n->attrs.dict["axis"]},
+                                {"k", "0"},
+                                {"ret_typ", "value"},
+                                {"is_ascend", std::to_string(param.is_ascend)}});
   })
 .set_attr<FResourceRequest>("FResourceRequest",
   [](const NodeAttrs& attrs) {
diff --git a/tests/python/unittest/test_symbol.py b/tests/python/unittest/test_symbol.py
index 2ae11fa4a87f..ab25f48eeb52 100644
--- a/tests/python/unittest/test_symbol.py
+++ b/tests/python/unittest/test_symbol.py
@@ -1,5 +1,6 @@
 import copy
 import os
+import re
 import mxnet as mx
 import numpy as np
 from common import models
@@ -181,7 +182,52 @@ def test_load_000800():
         {'ctx': mx.cpu(0), 'group2ctx': {'stage1' : mx.cpu(1), 'stage2' : mx.cpu(2)}, 'data': (1,200)})
 
 
+def test_blockgrad():
+    a = mx.sym.Variable('a')
+    b = mx.sym.BlockGrad(2*a)
+    exe = b.simple_bind(ctx=mx.cpu(), a=(10,10))
+
+
+def test_zero_prop():
+    data = mx.symbol.Variable('data')
+    for i in range(10):
+        data = data * data
+
+    exe = data.simple_bind(ctx=mx.cpu(), data=(10, 3, 256, 256))
+    big = int(re.search('Total (\d+) MB allocated', exe.debug_str()).group(1))
+
+    exe = data.simple_bind(ctx=mx.cpu(), data=(10, 3, 256, 256), grad_req='null')
+    small1 = int(re.search('Total (\d+) MB allocated', exe.debug_str()).group(1))
+
+    data = mx.sym.stop_gradient(data)
+    exe = data.simple_bind(ctx=mx.cpu(), data=(10, 3, 256, 256))
+    small2 = int(re.search('Total (\d+) MB allocated', exe.debug_str()).group(1))
+
+    assert big > small2
+    assert small1 == small2
+
+def test_zero_prop2():
+    x = mx.sym.Variable('x')
+    idx = mx.sym.Variable('idx')
+    y = mx.sym.batch_take(x, idx)
+    z = mx.sym.stop_gradient(y)
+    exe = z.simple_bind(ctx=mx.cpu(), x=(10, 10), idx=(10,),
+                        type_dict={'x': np.float32, 'idx': np.int32})
+    exe.forward()
+    exe.backward()
+
+    try:
+        y.simple_bind(ctx=mx.cpu(), x=(10, 10), idx=(10,),
+                      type_dict={'x': np.float32, 'idx': np.int32})
+    except:
+        return
+
+    assert False
+
 if __name__ == '__main__':
+    test_zero_prop2()
+    test_zero_prop()
+    test_blockgrad()
     test_symbol_children()
     test_load_000800()
     test_symbol_infer_shape_var()