diff --git a/mshadow/tensor_container.h b/mshadow/tensor_container.h
index 3e51b4a761de..b4df68e8e3a5 100644
--- a/mshadow/tensor_container.h
+++ b/mshadow/tensor_container.h
@@ -171,12 +171,17 @@ class TensorContainer: public Tensor<Device, dimension, DType> {
    */
   inline void Release(void) {
     if (data_.dptr_ != NULL) {
-      mshadow::FreeSpace(&data_);
-      this->dptr_ = data_.dptr_ = NULL;
       this->shape_[0] = 0;
       this->stride_ = 0;
       this->data_.stride_ = 0;
       this->data_.shape_[0] = 0;
+      try {
+        mshadow::FreeSpace(&data_);
+      } catch (const dmlc::Error &e) {
+        this->dptr_ = data_.dptr_ = NULL;
+        throw e;
+      }
+      this->dptr_ = data_.dptr_ = NULL;
     }
   }