From 46d3128836b36ccc4473a3ac009fe742ee303da1 Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Tue, 4 Sep 2018 21:59:54 +0000
Subject: [PATCH 01/20] add multi-threading inference.

---
 include/mxnet/c_predict_api.h |  12 ++++
 src/c_api/c_predict_api.cc    | 132 ++++++++++++++++++++++++----------
 2 files changed, 107 insertions(+), 37 deletions(-)

diff --git a/include/mxnet/c_predict_api.h b/include/mxnet/c_predict_api.h
index cc1c2966bd75..a42ad70feed7 100644
--- a/include/mxnet/c_predict_api.h
+++ b/include/mxnet/c_predict_api.h
@@ -119,6 +119,18 @@ MXNET_DLL int MXPredCreatePartialOut(const char* symbol_json_str,
                                      mx_uint num_output_nodes,
                                      const char** output_keys,
                                      PredictorHandle* out);
+
+int MXPredCreateMultithread(const char* symbol_json_str,
+                            const void* param_bytes,
+							int param_size,
+							int dev_type, int dev_id,
+							mx_uint num_input_nodes,
+							const char** input_keys,
+							const mx_uint* input_shape_indptr,
+							const mx_uint* input_shape_data,
+							int num_threads,
+							PredictorHandle* out);
+
 /*!
  * \brief Change the input shape of an existing predictor.
  * \param num_input_nodes Number of input nodes to the net,
diff --git a/src/c_api/c_predict_api.cc b/src/c_api/c_predict_api.cc
index d84a89ab2133..a7753b1d6177 100644
--- a/src/c_api/c_predict_api.cc
+++ b/src/c_api/c_predict_api.cc
@@ -67,34 +67,7 @@ struct MXAPINDList {
   std::vector<mx_float> data;
 };
 
-int MXPredCreate(const char* symbol_json_str,
-                 const void* param_bytes,
-                 int param_size,
-                 int dev_type, int dev_id,
-                 mx_uint num_input_nodes,
-                 const char** input_keys,
-                 const mx_uint* input_shape_indptr,
-                 const mx_uint* input_shape_data,
-                PredictorHandle* out) {
-  return MXPredCreatePartialOut(
-      symbol_json_str,
-      param_bytes,
-      param_size,
-      dev_type,
-      dev_id,
-      num_input_nodes,
-      input_keys,
-      input_shape_indptr,
-      input_shape_data,
-      0,
-      NULL,
-      out);
-}
-namespace mxnet {
-
-}  // namespace mxnet
-
-int MXPredCreatePartialOut(const char* symbol_json_str,
+int MXPredCreatePartialOut1(const char* symbol_json_str,
                            const void* param_bytes,
                            int param_size,
                            int dev_type, int dev_id,
@@ -104,10 +77,11 @@ int MXPredCreatePartialOut(const char* symbol_json_str,
                            const mx_uint* input_shape_data,
                            mx_uint num_output_nodes,
                            const char** output_keys,
+                           // This is used for paralle inference.
+                           int num_threads,
                            PredictorHandle* out) {
   using nnvm::Symbol;
 
-  MXAPIPredictor* ret = new MXAPIPredictor();
   API_BEGIN();
   Symbol sym;
   // make sure symbols are registered
@@ -140,7 +114,6 @@ int MXPredCreatePartialOut(const char* symbol_json_str,
     }
     sym = nnvm::Symbol::CreateGroup(out_syms);
   }
-  ret->sym = sym;
 
   // load the parameters
   std::unordered_map<std::string, NDArray> arg_params, aux_params;
@@ -188,9 +161,10 @@ int MXPredCreatePartialOut(const char* symbol_json_str,
   std::vector<TShape> out_shapes(sym.ListOutputNames().size());
   std::vector<TShape> aux_shapes(aux_names.size());
   std::vector<TShape> arg_shapes;
+  std::unordered_map<std::string, size_t> key2arg;
   for (size_t i = 0; i < arg_names.size(); ++i) {
     std::string key = arg_names[i];
-    ret->key2arg[key] = i;
+    key2arg[key] = i;
   }
 
   try {
@@ -215,7 +189,6 @@ int MXPredCreatePartialOut(const char* symbol_json_str,
   }
 
   Context ctx = Context::Create(static_cast<Context::DeviceType>(dev_type), dev_id);
-  ret->ctx = ctx;
 
   std::vector<NDArray> arg_arrays, aux_arrays;
   for (size_t i = 0; i < arg_shapes.size(); ++i) {
@@ -232,10 +205,15 @@ int MXPredCreatePartialOut(const char* symbol_json_str,
     }
     aux_arrays.push_back(nd);
   }
-  ret->arg_arrays = arg_arrays;
-  ret->aux_arrays = aux_arrays;
   // bind
-  {
+  for (int i = 0; i < num_threads; i++) {
+    std::unique_ptr<MXAPIPredictor> ret(new MXAPIPredictor());
+	ret->sym = sym;
+    ret->ctx = ctx;
+	ret->key2arg = key2arg;
+    ret->arg_arrays = arg_arrays;
+    ret->aux_arrays = aux_arrays;
+
     std::map<std::string, Context> ctx_map;
     std::vector<NDArray> grad_store(arg_arrays.size());
     std::vector<OpReqType> grad_req(arg_arrays.size(), kNullOp);
@@ -247,9 +225,89 @@ int MXPredCreatePartialOut(const char* symbol_json_str,
                                    aux_arrays));
     ret->out_shapes = out_shapes;
     ret->out_arrays = ret->exec->outputs();
+    out[i] = ret.release();
   }
-  *out = ret;
-  API_END_HANDLE_ERROR(delete ret);
+  API_END_HANDLE_ERROR();
+}
+
+int MXPredCreatePartialOut(const char* symbol_json_str,
+                           const void* param_bytes,
+                           int param_size,
+                           int dev_type, int dev_id,
+                           mx_uint num_input_nodes,
+                           const char** input_keys,
+                           const mx_uint* input_shape_indptr,
+                           const mx_uint* input_shape_data,
+                           mx_uint num_output_nodes,
+                           const char** output_keys,
+                           // This is used for paralle inference.
+                           int num_threads,
+                           PredictorHandle* out) {
+  return MXPredCreatePartialOut1(
+		  symbol_json_str,
+		  param_bytes,
+		  param_size,
+		  dev_type, dev_id,
+		  num_input_nodes,
+		  input_keys,
+		  input_shape_indptr,
+		  input_shape_data,
+		  num_output_nodes,
+		  output_keys,
+		  1,
+		  out);
+}
+
+int MXPredCreate(const char* symbol_json_str,
+                 const void* param_bytes,
+                 int param_size,
+                 int dev_type, int dev_id,
+                 mx_uint num_input_nodes,
+                 const char** input_keys,
+                 const mx_uint* input_shape_indptr,
+                 const mx_uint* input_shape_data,
+                 PredictorHandle* out) {
+  return MXPredCreatePartialOut1(
+      symbol_json_str,
+      param_bytes,
+      param_size,
+      dev_type,
+      dev_id,
+      num_input_nodes,
+      input_keys,
+      input_shape_indptr,
+      input_shape_data,
+      0,
+      NULL,
+      1,
+      out);
+}
+
+int MXPredCreateMultithread(const char* symbol_json_str,
+                            const void* param_bytes,
+                            int param_size,
+                            int dev_type, int dev_id,
+                            mx_uint num_input_nodes,
+                            const char** input_keys,
+                            const mx_uint* input_shape_indptr,
+                            const mx_uint* input_shape_data,
+                            // This is used for paralle inference.
+                            int num_threads,
+                            PredictorHandle* out) {
+  return MXPredCreatePartialOut1(
+      symbol_json_str,
+      param_bytes,
+      param_size,
+      dev_type,
+      dev_id,
+      num_input_nodes,
+      input_keys,
+      input_shape_indptr,
+      input_shape_data,
+      0,
+      NULL,
+	  num_threads,
+      out);
 }
 
 int MXPredReshape(mx_uint num_input_nodes,

From b6850c735cd56ef5dac79f4adb0bde5c29249887 Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Tue, 4 Sep 2018 22:00:41 +0000
Subject: [PATCH 02/20] demo multi-threading inference.

---
 .../image-classification/predict-cpp/Makefile |   2 +-
 .../image-classification-predict.cc           | 109 +++++++++++-------
 2 files changed, 68 insertions(+), 43 deletions(-)

diff --git a/example/image-classification/predict-cpp/Makefile b/example/image-classification/predict-cpp/Makefile
index e0c0bc657297..5c084119b966 100644
--- a/example/image-classification/predict-cpp/Makefile
+++ b/example/image-classification/predict-cpp/Makefile
@@ -15,7 +15,7 @@ LDFLAGS+=`pkg-config --libs opencv`
 export MXNET_ROOT=`pwd`/../../..
 
 CFLAGS+=-Wall -I$(MXNET_ROOT)/include
-LDFLAGS+=$(MXNET_ROOT)/lib/libmxnet.so
+LDFLAGS+=$(MXNET_ROOT)/lib/libmxnet.so -lpthread
 
 image-classification-predict: image-classification-predict.o
 	g++ -O3 -o image-classification-predict image-classification-predict.o $(LDFLAGS)
diff --git a/example/image-classification/predict-cpp/image-classification-predict.cc b/example/image-classification/predict-cpp/image-classification-predict.cc
index 186107bd530f..4c6b1e46cb7e 100644
--- a/example/image-classification/predict-cpp/image-classification-predict.cc
+++ b/example/image-classification/predict-cpp/image-classification-predict.cc
@@ -37,6 +37,7 @@
 #include <fstream>
 #include <vector>
 #include <memory>
+#include <thread>
 #include <iomanip>
 #include <opencv2/opencv.hpp>
 // Path for c_predict_api
@@ -179,6 +180,46 @@ void PrintOutputResult(const std::vector<float>& data, const std::vector<std::st
             "accuracy=" << std::setprecision(8) << best_accuracy << ")" << std::endl;
 }
 
+void predict(PredictorHandle pred_hnd, const std::vector<mx_float> &image_data,
+		NDListHandle nd_hnd, int i) {
+  std::string synset_file = "synset.txt";
+  auto image_size = image_data.size();
+  // Set Input Image
+  MXPredSetInput(pred_hnd, "data", image_data.data(), static_cast<mx_uint>(image_size));
+
+  // Do Predict Forward
+  MXPredForward(pred_hnd);
+
+  mx_uint output_index = 0;
+
+  mx_uint* shape = nullptr;
+  mx_uint shape_len;
+
+  // Get Output Result
+  MXPredGetOutputShape(pred_hnd, output_index, &shape, &shape_len);
+
+  std::size_t size = 1;
+  for (mx_uint i = 0; i < shape_len; ++i) { size *= shape[i]; }
+
+  std::vector<float> data(size);
+
+  MXPredGetOutput(pred_hnd, output_index, &(data[0]), static_cast<mx_uint>(size));
+
+  // Release NDList
+  if (nd_hnd) {
+    MXNDListFree(nd_hnd);
+  }
+
+  // Release Predictor
+  MXPredFree(pred_hnd);
+
+  // Synset path for your model, you have to modify it
+  auto synset = LoadSynset(synset_file);
+
+  // Print Output Data
+  PrintOutputResult(data, synset);
+}
+
 int main(int argc, char* argv[]) {
   if (argc < 2) {
     std::cout << "No test image here." << std::endl
@@ -189,10 +230,10 @@ int main(int argc, char* argv[]) {
   std::string test_file(argv[1]);
 
   // Models path for your model, you have to modify it
-  std::string json_file = "model/Inception/Inception-BN-symbol.json";
-  std::string param_file = "model/Inception/Inception-BN-0126.params";
-  std::string synset_file = "model/Inception/synset.txt";
-  std::string nd_file = "model/Inception/mean_224.nd";
+  std::string json_file = "Inception-BN-symbol.json";
+  std::string param_file = "Inception-BN-0126.params";
+  std::string synset_file = "synset.txt";
+  std::string nd_file = "mean_224.nd";
 
   BufferFile json_data(json_file);
   BufferFile param_data(param_file);
@@ -214,14 +255,14 @@ int main(int argc, char* argv[]) {
                                         static_cast<mx_uint>(channels),
                                         static_cast<mx_uint>(height),
                                         static_cast<mx_uint>(width) };
-  PredictorHandle pred_hnd = nullptr;
+  std::vector<PredictorHandle> pred_hnds(8, nullptr);
 
   if (json_data.GetLength() == 0 || param_data.GetLength() == 0) {
     return EXIT_FAILURE;
   }
 
   // Create Predictor
-  MXPredCreate(static_cast<const char*>(json_data.GetBuffer()),
+  MXPredCreateMultithread(static_cast<const char*>(json_data.GetBuffer()),
                static_cast<const char*>(param_data.GetBuffer()),
                static_cast<int>(param_data.GetLength()),
                dev_type,
@@ -230,8 +271,10 @@ int main(int argc, char* argv[]) {
                input_keys,
                input_shape_indptr,
                input_shape_data,
-               &pred_hnd);
-  assert(pred_hnd);
+			   pred_hnds.size(),
+               pred_hnds.data());
+  for (auto hnd : pred_hnds)
+	  assert(hnd);
 
   auto image_size = static_cast<std::size_t>(width * height * channels);
 
@@ -259,40 +302,22 @@ int main(int argc, char* argv[]) {
 
   GetImageFile(test_file, image_data.data(), channels, cv::Size(width, height), nd_data);
 
-  // Set Input Image
-  MXPredSetInput(pred_hnd, "data", image_data.data(), static_cast<mx_uint>(image_size));
-
-  // Do Predict Forward
-  MXPredForward(pred_hnd);
-
-  mx_uint output_index = 0;
-
-  mx_uint* shape = nullptr;
-  mx_uint shape_len;
-
-  // Get Output Result
-  MXPredGetOutputShape(pred_hnd, output_index, &shape, &shape_len);
-
-  std::size_t size = 1;
-  for (mx_uint i = 0; i < shape_len; ++i) { size *= shape[i]; }
-
-  std::vector<float> data(size);
-
-  MXPredGetOutput(pred_hnd, output_index, &(data[0]), static_cast<mx_uint>(size));
-
-  // Release NDList
-  if (nd_hnd) {
-    MXNDListFree(nd_hnd);
-  }
-
-  // Release Predictor
-  MXPredFree(pred_hnd);
-
-  // Synset path for your model, you have to modify it
-  auto synset = LoadSynset(synset_file);
-
-  // Print Output Data
-  PrintOutputResult(data, synset);
+  std::thread t0(predict, pred_hnds[0], image_data, nd_hnd, 0);
+  std::thread t1(predict, pred_hnds[1], image_data, nd_hnd, 1);
+  std::thread t2(predict, pred_hnds[2], image_data, nd_hnd, 2);
+  std::thread t3(predict, pred_hnds[3], image_data, nd_hnd, 3);
+  std::thread t4(predict, pred_hnds[4], image_data, nd_hnd, 4);
+  std::thread t5(predict, pred_hnds[5], image_data, nd_hnd, 5);
+  std::thread t6(predict, pred_hnds[6], image_data, nd_hnd, 6);
+  std::thread t7(predict, pred_hnds[7], image_data, nd_hnd, 7);
+  t0.join();
+  t1.join();
+  t2.join();
+  t3.join();
+  t4.join();
+  t5.join();
+  t6.join();
+  t7.join();
 
   return EXIT_SUCCESS;
 }

From 7da425427fa3369b305e95d70794fd53e19dda65 Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Wed, 5 Sep 2018 22:16:59 +0000
Subject: [PATCH 03/20] add new capi.

---
 include/mxnet/c_predict_api.h |  2 ++
 src/c_api/c_predict_api.cc    | 15 +++++++++++++++
 2 files changed, 17 insertions(+)

diff --git a/include/mxnet/c_predict_api.h b/include/mxnet/c_predict_api.h
index a42ad70feed7..ecc3d6c3f806 100644
--- a/include/mxnet/c_predict_api.h
+++ b/include/mxnet/c_predict_api.h
@@ -131,6 +131,8 @@ int MXPredCreateMultithread(const char* symbol_json_str,
 							int num_threads,
 							PredictorHandle* out);
 
+void MXPredCreateNewExecutor(PredictorHandle pred_hnd);
+
 /*!
  * \brief Change the input shape of an existing predictor.
  * \param num_input_nodes Number of input nodes to the net,
diff --git a/src/c_api/c_predict_api.cc b/src/c_api/c_predict_api.cc
index a7753b1d6177..1267d67a2829 100644
--- a/src/c_api/c_predict_api.cc
+++ b/src/c_api/c_predict_api.cc
@@ -67,6 +67,21 @@ struct MXAPINDList {
   std::vector<mx_float> data;
 };
 
+void MXPredCreateNewExecutor(PredictorHandle pred_hnd) {
+	MXAPIPredictor *pred = (MXAPIPredictor *) pred_hnd;
+  auto sym = pred->sym;
+  auto ctx = pred->ctx;
+  auto key2arg = pred->key2arg;
+  auto arg_arrays = pred->arg_arrays;
+  auto aux_arrays = pred->aux_arrays;
+  std::map<std::string, Context> ctx_map;
+  std::vector<NDArray> grad_store(arg_arrays.size());
+  std::vector<OpReqType> grad_req(arg_arrays.size(), kNullOp);
+  pred->exec.reset(Executor::Bind(sym, ctx, ctx_map, arg_arrays,
+                                  grad_store, grad_req, aux_arrays));
+  pred->out_arrays = pred->exec->outputs();
+}
+
 int MXPredCreatePartialOut1(const char* symbol_json_str,
                            const void* param_bytes,
                            int param_size,

From 8cce6f345b956cefaaec9589636d55f554406c76 Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Wed, 5 Sep 2018 22:33:32 +0000
Subject: [PATCH 04/20] make naive engine thread local.

---
 src/engine/engine.cc | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/engine/engine.cc b/src/engine/engine.cc
index 1c72f33d24e1..e5f63f6f69a6 100644
--- a/src/engine/engine.cc
+++ b/src/engine/engine.cc
@@ -29,6 +29,17 @@
 
 namespace mxnet {
 namespace engine {
+
+inline bool IsNaiveEngine() {
+  const char *type = getenv("MXNET_ENGINE_TYPE");
+  if (type == nullptr) {
+    return false;
+  } else {
+    std::string stype = type;
+    return stype == "NaiveEngine";
+  }
+}
+
 inline Engine* CreateEngine() {
   const char *type = getenv("MXNET_ENGINE_TYPE");
   const bool default_engine = (type == nullptr);
@@ -59,12 +70,18 @@ inline Engine* CreateEngine() {
 }  // namespace engine
 
 std::shared_ptr<Engine> Engine::_GetSharedRef() {
-  static std::shared_ptr<Engine> sptr(engine::CreateEngine());
-  return sptr;
+  static bool is_naive = engine::IsNaiveEngine();
+  if (is_naive) {
+    static thread_local std::shared_ptr<Engine> sptr(engine::CreateEngine());
+    return sptr;
+  } else {
+    static std::shared_ptr<Engine> sptr(engine::CreateEngine());
+    return sptr;
+  }
 }
 
 Engine* Engine::Get() {
-  static Engine *inst = _GetSharedRef().get();
+  Engine *inst = _GetSharedRef().get();
   return inst;
 }
 }  // namespace mxnet

From 7e6ddfc1d23e0b98076b877ef58e102e5db38918 Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Wed, 5 Sep 2018 22:34:09 +0000
Subject: [PATCH 05/20] create an executor inside each thread.

---
 .../predict-cpp/image-classification-predict.cc              | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/example/image-classification/predict-cpp/image-classification-predict.cc b/example/image-classification/predict-cpp/image-classification-predict.cc
index 4c6b1e46cb7e..386e23e3ec9b 100644
--- a/example/image-classification/predict-cpp/image-classification-predict.cc
+++ b/example/image-classification/predict-cpp/image-classification-predict.cc
@@ -182,6 +182,8 @@ void PrintOutputResult(const std::vector<float>& data, const std::vector<std::st
 
 void predict(PredictorHandle pred_hnd, const std::vector<mx_float> &image_data,
 		NDListHandle nd_hnd, int i) {
+  MXPredCreateNewExecutor(pred_hnd);
+
   std::string synset_file = "synset.txt";
   auto image_size = image_data.size();
   // Set Input Image
@@ -274,7 +276,7 @@ int main(int argc, char* argv[]) {
 			   pred_hnds.size(),
                pred_hnds.data());
   for (auto hnd : pred_hnds)
-	  assert(hnd);
+    assert(hnd);
 
   auto image_size = static_cast<std::size_t>(width * height * channels);
 
@@ -318,6 +320,7 @@ int main(int argc, char* argv[]) {
   t5.join();
   t6.join();
   t7.join();
+  printf("run successfully\n");
 
   return EXIT_SUCCESS;
 }

From e93fb7252fbb7d43e9f8c94e056fab3b16574a77 Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Wed, 5 Sep 2018 22:38:59 +0000
Subject: [PATCH 06/20] fix format.

---
 .../predict-cpp/image-classification-predict.cc               | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/example/image-classification/predict-cpp/image-classification-predict.cc b/example/image-classification/predict-cpp/image-classification-predict.cc
index 386e23e3ec9b..b1eb8b41a1d1 100644
--- a/example/image-classification/predict-cpp/image-classification-predict.cc
+++ b/example/image-classification/predict-cpp/image-classification-predict.cc
@@ -181,7 +181,7 @@ void PrintOutputResult(const std::vector<float>& data, const std::vector<std::st
 }
 
 void predict(PredictorHandle pred_hnd, const std::vector<mx_float> &image_data,
-		NDListHandle nd_hnd, int i) {
+             NDListHandle nd_hnd, int i) {
   MXPredCreateNewExecutor(pred_hnd);
 
   std::string synset_file = "synset.txt";
@@ -273,7 +273,7 @@ int main(int argc, char* argv[]) {
                input_keys,
                input_shape_indptr,
                input_shape_data,
-			   pred_hnds.size(),
+               pred_hnds.size(),
                pred_hnds.data());
   for (auto hnd : pred_hnds)
     assert(hnd);

From acdd78e231e9f9191882abb2fe1fddc1f18c97ab Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Wed, 5 Sep 2018 22:39:13 +0000
Subject: [PATCH 07/20] fix format.

---
 include/mxnet/c_predict_api.h | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/include/mxnet/c_predict_api.h b/include/mxnet/c_predict_api.h
index ecc3d6c3f806..8a791b41ff85 100644
--- a/include/mxnet/c_predict_api.h
+++ b/include/mxnet/c_predict_api.h
@@ -120,18 +120,18 @@ MXNET_DLL int MXPredCreatePartialOut(const char* symbol_json_str,
                                      const char** output_keys,
                                      PredictorHandle* out);
 
-int MXPredCreateMultithread(const char* symbol_json_str,
-                            const void* param_bytes,
-							int param_size,
-							int dev_type, int dev_id,
-							mx_uint num_input_nodes,
-							const char** input_keys,
-							const mx_uint* input_shape_indptr,
-							const mx_uint* input_shape_data,
-							int num_threads,
-							PredictorHandle* out);
+MXNET_DLL int MXPredCreateMultithread(const char* symbol_json_str,
+                                      const void* param_bytes,
+                                      int param_size,
+                                      int dev_type, int dev_id,
+                                      mx_uint num_input_nodes,
+                                      const char** input_keys,
+                                      const mx_uint* input_shape_indptr,
+                                      const mx_uint* input_shape_data,
+                                      int num_threads,
+                                      PredictorHandle* out);
 
-void MXPredCreateNewExecutor(PredictorHandle pred_hnd);
+MXNET_DLL void MXPredCreateNewExecutor(PredictorHandle pred_hnd);
 
 /*!
  * \brief Change the input shape of an existing predictor.

From d5ef9cd3b559f7349f32a7769b247138766486b2 Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Wed, 5 Sep 2018 22:43:30 +0000
Subject: [PATCH 08/20] fix format.

---
 src/c_api/c_predict_api.cc | 56 +++++++++++++++++++-------------------
 1 file changed, 28 insertions(+), 28 deletions(-)

diff --git a/src/c_api/c_predict_api.cc b/src/c_api/c_predict_api.cc
index 1267d67a2829..e9166e09d4b7 100644
--- a/src/c_api/c_predict_api.cc
+++ b/src/c_api/c_predict_api.cc
@@ -68,7 +68,7 @@ struct MXAPINDList {
 };
 
 void MXPredCreateNewExecutor(PredictorHandle pred_hnd) {
-	MXAPIPredictor *pred = (MXAPIPredictor *) pred_hnd;
+  MXAPIPredictor *pred = (MXAPIPredictor *) pred_hnd;
   auto sym = pred->sym;
   auto ctx = pred->ctx;
   auto key2arg = pred->key2arg;
@@ -83,18 +83,18 @@ void MXPredCreateNewExecutor(PredictorHandle pred_hnd) {
 }
 
 int MXPredCreatePartialOut1(const char* symbol_json_str,
-                           const void* param_bytes,
-                           int param_size,
-                           int dev_type, int dev_id,
-                           mx_uint num_input_nodes,
-                           const char** input_keys,
-                           const mx_uint* input_shape_indptr,
-                           const mx_uint* input_shape_data,
-                           mx_uint num_output_nodes,
-                           const char** output_keys,
-                           // This is used for paralle inference.
-                           int num_threads,
-                           PredictorHandle* out) {
+                            const void* param_bytes,
+                            int param_size,
+                            int dev_type, int dev_id,
+                            mx_uint num_input_nodes,
+                            const char** input_keys,
+                            const mx_uint* input_shape_indptr,
+                            const mx_uint* input_shape_data,
+                            mx_uint num_output_nodes,
+                            const char** output_keys,
+                            // This is used for paralle inference.
+                            int num_threads,
+                            PredictorHandle* out) {
   using nnvm::Symbol;
 
   API_BEGIN();
@@ -223,9 +223,9 @@ int MXPredCreatePartialOut1(const char* symbol_json_str,
   // bind
   for (int i = 0; i < num_threads; i++) {
     std::unique_ptr<MXAPIPredictor> ret(new MXAPIPredictor());
-	ret->sym = sym;
+    ret->sym = sym;
     ret->ctx = ctx;
-	ret->key2arg = key2arg;
+    ret->key2arg = key2arg;
     ret->arg_arrays = arg_arrays;
     ret->aux_arrays = aux_arrays;
 
@@ -259,18 +259,18 @@ int MXPredCreatePartialOut(const char* symbol_json_str,
                            int num_threads,
                            PredictorHandle* out) {
   return MXPredCreatePartialOut1(
-		  symbol_json_str,
-		  param_bytes,
-		  param_size,
-		  dev_type, dev_id,
-		  num_input_nodes,
-		  input_keys,
-		  input_shape_indptr,
-		  input_shape_data,
-		  num_output_nodes,
-		  output_keys,
-		  1,
-		  out);
+      symbol_json_str,
+      param_bytes,
+      param_size,
+      dev_type, dev_id,
+      num_input_nodes,
+      input_keys,
+      input_shape_indptr,
+      input_shape_data,
+      num_output_nodes,
+      output_keys,
+      1,
+      out);
 }
 
 int MXPredCreate(const char* symbol_json_str,
@@ -321,7 +321,7 @@ int MXPredCreateMultithread(const char* symbol_json_str,
       input_shape_data,
       0,
       NULL,
-	  num_threads,
+      num_threads,
       out);
 }
 

From 00b47bf900fcddcf3b87251c0a8ce44695cc4f42 Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Wed, 5 Sep 2018 22:44:48 +0000
Subject: [PATCH 09/20] Revert "make naive engine thread local."

This reverts commit b9d844e46d33f11ad409feb099194e183da9bbda.
---
 src/engine/engine.cc | 23 +++--------------------
 1 file changed, 3 insertions(+), 20 deletions(-)

diff --git a/src/engine/engine.cc b/src/engine/engine.cc
index e5f63f6f69a6..1c72f33d24e1 100644
--- a/src/engine/engine.cc
+++ b/src/engine/engine.cc
@@ -29,17 +29,6 @@
 
 namespace mxnet {
 namespace engine {
-
-inline bool IsNaiveEngine() {
-  const char *type = getenv("MXNET_ENGINE_TYPE");
-  if (type == nullptr) {
-    return false;
-  } else {
-    std::string stype = type;
-    return stype == "NaiveEngine";
-  }
-}
-
 inline Engine* CreateEngine() {
   const char *type = getenv("MXNET_ENGINE_TYPE");
   const bool default_engine = (type == nullptr);
@@ -70,18 +59,12 @@ inline Engine* CreateEngine() {
 }  // namespace engine
 
 std::shared_ptr<Engine> Engine::_GetSharedRef() {
-  static bool is_naive = engine::IsNaiveEngine();
-  if (is_naive) {
-    static thread_local std::shared_ptr<Engine> sptr(engine::CreateEngine());
-    return sptr;
-  } else {
-    static std::shared_ptr<Engine> sptr(engine::CreateEngine());
-    return sptr;
-  }
+  static std::shared_ptr<Engine> sptr(engine::CreateEngine());
+  return sptr;
 }
 
 Engine* Engine::Get() {
-  Engine *inst = _GetSharedRef().get();
+  static Engine *inst = _GetSharedRef().get();
   return inst;
 }
 }  // namespace mxnet

From 20b739214295922eaeb699a8785e8dc9a3da3673 Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Thu, 6 Sep 2018 18:36:52 +0000
Subject: [PATCH 10/20] Update CAPI.

---
 .../image-classification-predict.cc           |  2 -
 include/mxnet/c_predict_api.h                 |  2 -
 src/c_api/c_predict_api.cc                    | 55 +++++++++++--------
 3 files changed, 32 insertions(+), 27 deletions(-)

diff --git a/example/image-classification/predict-cpp/image-classification-predict.cc b/example/image-classification/predict-cpp/image-classification-predict.cc
index b1eb8b41a1d1..931710a0ca57 100644
--- a/example/image-classification/predict-cpp/image-classification-predict.cc
+++ b/example/image-classification/predict-cpp/image-classification-predict.cc
@@ -182,8 +182,6 @@ void PrintOutputResult(const std::vector<float>& data, const std::vector<std::st
 
 void predict(PredictorHandle pred_hnd, const std::vector<mx_float> &image_data,
              NDListHandle nd_hnd, int i) {
-  MXPredCreateNewExecutor(pred_hnd);
-
   std::string synset_file = "synset.txt";
   auto image_size = image_data.size();
   // Set Input Image
diff --git a/include/mxnet/c_predict_api.h b/include/mxnet/c_predict_api.h
index 8a791b41ff85..27173762f2dd 100644
--- a/include/mxnet/c_predict_api.h
+++ b/include/mxnet/c_predict_api.h
@@ -131,8 +131,6 @@ MXNET_DLL int MXPredCreateMultithread(const char* symbol_json_str,
                                       int num_threads,
                                       PredictorHandle* out);
 
-MXNET_DLL void MXPredCreateNewExecutor(PredictorHandle pred_hnd);
-
 /*!
  * \brief Change the input shape of an existing predictor.
  * \param num_input_nodes Number of input nodes to the net,
diff --git a/src/c_api/c_predict_api.cc b/src/c_api/c_predict_api.cc
index e9166e09d4b7..93a00fc8387f 100644
--- a/src/c_api/c_predict_api.cc
+++ b/src/c_api/c_predict_api.cc
@@ -67,19 +67,21 @@ struct MXAPINDList {
   std::vector<mx_float> data;
 };
 
-void MXPredCreateNewExecutor(PredictorHandle pred_hnd) {
+inline void _CreateExecutor(PredictorHandle pred_hnd) {
   MXAPIPredictor *pred = (MXAPIPredictor *) pred_hnd;
-  auto sym = pred->sym;
-  auto ctx = pred->ctx;
-  auto key2arg = pred->key2arg;
-  auto arg_arrays = pred->arg_arrays;
-  auto aux_arrays = pred->aux_arrays;
-  std::map<std::string, Context> ctx_map;
-  std::vector<NDArray> grad_store(arg_arrays.size());
-  std::vector<OpReqType> grad_req(arg_arrays.size(), kNullOp);
-  pred->exec.reset(Executor::Bind(sym, ctx, ctx_map, arg_arrays,
-                                  grad_store, grad_req, aux_arrays));
-  pred->out_arrays = pred->exec->outputs();
+  if (pred->exec == nullptr) {
+    auto sym = pred->sym;
+    auto ctx = pred->ctx;
+    auto key2arg = pred->key2arg;
+    auto arg_arrays = pred->arg_arrays;
+    auto aux_arrays = pred->aux_arrays;
+    std::map<std::string, Context> ctx_map;
+    std::vector<NDArray> grad_store(arg_arrays.size());
+    std::vector<OpReqType> grad_req(arg_arrays.size(), kNullOp);
+    pred->exec.reset(Executor::Bind(sym, ctx, ctx_map, arg_arrays,
+                                    grad_store, grad_req, aux_arrays));
+    pred->out_arrays = pred->exec->outputs();
+  }
 }
 
 int MXPredCreatePartialOut1(const char* symbol_json_str,
@@ -94,6 +96,7 @@ int MXPredCreatePartialOut1(const char* symbol_json_str,
                             const char** output_keys,
                             // This is used for paralle inference.
                             int num_threads,
+                            bool lazy,
                             PredictorHandle* out) {
   using nnvm::Symbol;
 
@@ -228,18 +231,18 @@ int MXPredCreatePartialOut1(const char* symbol_json_str,
     ret->key2arg = key2arg;
     ret->arg_arrays = arg_arrays;
     ret->aux_arrays = aux_arrays;
-
-    std::map<std::string, Context> ctx_map;
-    std::vector<NDArray> grad_store(arg_arrays.size());
-    std::vector<OpReqType> grad_req(arg_arrays.size(), kNullOp);
-
-
-    ret->exec.reset(Executor::Bind(sym, ctx, ctx_map,
-                                   arg_arrays,
-                                   grad_store, grad_req,
-                                   aux_arrays));
     ret->out_shapes = out_shapes;
-    ret->out_arrays = ret->exec->outputs();
+
+    if (!lazy) {
+      std::map<std::string, Context> ctx_map;
+      std::vector<NDArray> grad_store(arg_arrays.size());
+      std::vector<OpReqType> grad_req(arg_arrays.size(), kNullOp);
+      ret->exec.reset(Executor::Bind(sym, ctx, ctx_map,
+                                     arg_arrays,
+                                     grad_store, grad_req,
+                                     aux_arrays));
+      ret->out_arrays = ret->exec->outputs();
+    }
     out[i] = ret.release();
   }
   API_END_HANDLE_ERROR();
@@ -270,6 +273,7 @@ int MXPredCreatePartialOut(const char* symbol_json_str,
       num_output_nodes,
       output_keys,
       1,
+      false,
       out);
 }
 
@@ -295,6 +299,7 @@ int MXPredCreate(const char* symbol_json_str,
       0,
       NULL,
       1,
+      false,
       out);
 }
 
@@ -322,6 +327,7 @@ int MXPredCreateMultithread(const char* symbol_json_str,
       0,
       NULL,
       num_threads,
+      true,
       out);
 }
 
@@ -331,6 +337,7 @@ int MXPredReshape(mx_uint num_input_nodes,
                   const mx_uint* input_shape_data,
                   PredictorHandle handle,
                   PredictorHandle* out) {
+  _CreateExecutor(handle);
   MXAPIPredictor* p = static_cast<MXAPIPredictor*>(handle);
   std::unique_ptr<MXAPIPredictor> ret(new MXAPIPredictor());
 
@@ -447,6 +454,7 @@ int MXPredSetInput(PredictorHandle handle,
 }
 
 int MXPredForward(PredictorHandle handle) {
+  _CreateExecutor(handle);
   MXAPIPredictor* p = static_cast<MXAPIPredictor*>(handle);
   API_BEGIN();
   p->exec->Forward(false);
@@ -454,6 +462,7 @@ int MXPredForward(PredictorHandle handle) {
 }
 
 int MXPredPartialForward(PredictorHandle handle, int step, int* step_left) {
+  _CreateExecutor(handle);
   MXAPIPredictor* p = static_cast<MXAPIPredictor*>(handle);
   API_BEGIN();
   p->exec->PartialForward(false, step, step_left);

From 899d32427f4de89a02223f81978a8daf78406889 Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Thu, 6 Sep 2018 18:41:18 +0000
Subject: [PATCH 11/20] add doc.

---
 include/mxnet/c_predict_api.h | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/include/mxnet/c_predict_api.h b/include/mxnet/c_predict_api.h
index 27173762f2dd..2e0023f8869a 100644
--- a/include/mxnet/c_predict_api.h
+++ b/include/mxnet/c_predict_api.h
@@ -120,6 +120,27 @@ MXNET_DLL int MXPredCreatePartialOut(const char* symbol_json_str,
                                      const char** output_keys,
                                      PredictorHandle* out);
 
+/*!
+ * \brief create predictors for multiple threads. One predictor for a thread.
+ * \param symbol_json_str The JSON string of the symbol.
+ * \param param_bytes The in-memory raw bytes of parameter ndarray file.
+ * \param param_size The size of parameter ndarray file.
+ * \param dev_type The device type, 1: cpu, 2:gpu
+ * \param dev_id The device id of the predictor.
+ * \param num_input_nodes Number of input nodes to the net,
+ *    For feedforward net, this is 1.
+ * \param input_keys The name of input argument.
+ *    For feedforward net, this is {"data"}
+ * \param input_shape_indptr Index pointer of shapes of each input node.
+ *    The length of this array = num_input_nodes + 1.
+ *    For feedforward net that takes 4 dimensional input, this is {0, 4}.
+ * \param input_shape_data A flatted data of shapes of each input node.
+ *    For feedforward net that takes 4 dimensional input, this is the shape data.
+ * \param num_threads The number of threads that we'll run the predictors.
+ * \param out An array of created predictor handles. The array has to be large
+ *   enough to keep `num_threads` predictors.
+ * \return 0 when success, -1 when failure.
+ */
 MXNET_DLL int MXPredCreateMultithread(const char* symbol_json_str,
                                       const void* param_bytes,
                                       int param_size,

From da579e2c38cb109c5e90bcb8e08916811c4b7f4b Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Thu, 6 Sep 2018 18:45:08 +0000
Subject: [PATCH 12/20] fix lint.

---
 src/c_api/c_predict_api.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/c_api/c_predict_api.cc b/src/c_api/c_predict_api.cc
index 93a00fc8387f..35e655b6647b 100644
--- a/src/c_api/c_predict_api.cc
+++ b/src/c_api/c_predict_api.cc
@@ -68,7 +68,7 @@ struct MXAPINDList {
 };
 
 inline void _CreateExecutor(PredictorHandle pred_hnd) {
-  MXAPIPredictor *pred = (MXAPIPredictor *) pred_hnd;
+  MXAPIPredictor *pred = static_cast<MXAPIPredictor*>(pred_hnd);
   if (pred->exec == nullptr) {
     auto sym = pred->sym;
     auto ctx = pred->ctx;

From d1e49121d018301f40c6f908367e6f5997e23778 Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Thu, 6 Sep 2018 18:55:16 +0000
Subject: [PATCH 13/20] update example.

---
 .../image-classification-predict.cc           | 76 ++++++++++---------
 1 file changed, 42 insertions(+), 34 deletions(-)

diff --git a/example/image-classification/predict-cpp/image-classification-predict.cc b/example/image-classification/predict-cpp/image-classification-predict.cc
index 931710a0ca57..463c98c086d5 100644
--- a/example/image-classification/predict-cpp/image-classification-predict.cc
+++ b/example/image-classification/predict-cpp/image-classification-predict.cc
@@ -221,13 +221,14 @@ void predict(PredictorHandle pred_hnd, const std::vector<mx_float> &image_data,
 }
 
 int main(int argc, char* argv[]) {
-  if (argc < 2) {
+  if (argc < 3) {
     std::cout << "No test image here." << std::endl
-              << "Usage: ./image-classification-predict apple.jpg" << std::endl;
+              << "Usage: ./image-classification-predict apple.jpg num_threads" << std::endl;
     return EXIT_FAILURE;
   }
 
   std::string test_file(argv[1]);
+  int num_threads = std::atoi(argv[2]);
 
   // Models path for your model, you have to modify it
   std::string json_file = "Inception-BN-symbol.json";
@@ -255,27 +256,11 @@ int main(int argc, char* argv[]) {
                                         static_cast<mx_uint>(channels),
                                         static_cast<mx_uint>(height),
                                         static_cast<mx_uint>(width) };
-  std::vector<PredictorHandle> pred_hnds(8, nullptr);
 
   if (json_data.GetLength() == 0 || param_data.GetLength() == 0) {
     return EXIT_FAILURE;
   }
 
-  // Create Predictor
-  MXPredCreateMultithread(static_cast<const char*>(json_data.GetBuffer()),
-               static_cast<const char*>(param_data.GetBuffer()),
-               static_cast<int>(param_data.GetLength()),
-               dev_type,
-               dev_id,
-               num_input_nodes,
-               input_keys,
-               input_shape_indptr,
-               input_shape_data,
-               pred_hnds.size(),
-               pred_hnds.data());
-  for (auto hnd : pred_hnds)
-    assert(hnd);
-
   auto image_size = static_cast<std::size_t>(width * height * channels);
 
   // Read Mean Data
@@ -302,22 +287,45 @@ int main(int argc, char* argv[]) {
 
   GetImageFile(test_file, image_data.data(), channels, cv::Size(width, height), nd_data);
 
-  std::thread t0(predict, pred_hnds[0], image_data, nd_hnd, 0);
-  std::thread t1(predict, pred_hnds[1], image_data, nd_hnd, 1);
-  std::thread t2(predict, pred_hnds[2], image_data, nd_hnd, 2);
-  std::thread t3(predict, pred_hnds[3], image_data, nd_hnd, 3);
-  std::thread t4(predict, pred_hnds[4], image_data, nd_hnd, 4);
-  std::thread t5(predict, pred_hnds[5], image_data, nd_hnd, 5);
-  std::thread t6(predict, pred_hnds[6], image_data, nd_hnd, 6);
-  std::thread t7(predict, pred_hnds[7], image_data, nd_hnd, 7);
-  t0.join();
-  t1.join();
-  t2.join();
-  t3.join();
-  t4.join();
-  t5.join();
-  t6.join();
-  t7.join();
+  if (num_threads == 1) {
+    // Create Predictor
+    PredictorHandle pred_hnd;
+    MXPredCreate(static_cast<const char*>(json_data.GetBuffer()),
+                 static_cast<const char*>(param_data.GetBuffer()),
+                 static_cast<int>(param_data.GetLength()),
+                 dev_type,
+                 dev_id,
+                 num_input_nodes,
+                 input_keys,
+                 input_shape_indptr,
+                 input_shape_data,
+                 &pred_hnd);
+    assert(pred_hnd);
+
+    predict(pred_hnd, image_data, nd_hnd, 0);
+  } else {
+    // Create Predictor
+    std::vector<PredictorHandle> pred_hnds(num_threads, nullptr);
+    MXPredCreateMultithread(static_cast<const char*>(json_data.GetBuffer()),
+                            static_cast<const char*>(param_data.GetBuffer()),
+                            static_cast<int>(param_data.GetLength()),
+                            dev_type,
+                            dev_id,
+                            num_input_nodes,
+                            input_keys,
+                            input_shape_indptr,
+                            input_shape_data,
+                            pred_hnds.size(),
+                            pred_hnds.data());
+    for (auto hnd : pred_hnds)
+      assert(hnd);
+
+    std::vector<std::thread> threads;
+    for (int i = 0; i < num_threads; i++)
+      threads.emplace_back(predict, pred_hnds[i], image_data, nd_hnd, i);
+    for (int i = 0; i < num_threads; i++)
+      threads[i].join();
+  }
   printf("run successfully\n");
 
   return EXIT_SUCCESS;

From c64041eceadc62f539b77c3c4722b57d06c30831 Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Thu, 6 Sep 2018 18:56:52 +0000
Subject: [PATCH 14/20] update.

---
 .../predict-cpp/image-classification-predict.cc           | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/example/image-classification/predict-cpp/image-classification-predict.cc b/example/image-classification/predict-cpp/image-classification-predict.cc
index 463c98c086d5..823038779ab3 100644
--- a/example/image-classification/predict-cpp/image-classification-predict.cc
+++ b/example/image-classification/predict-cpp/image-classification-predict.cc
@@ -231,10 +231,10 @@ int main(int argc, char* argv[]) {
   int num_threads = std::atoi(argv[2]);
 
   // Models path for your model, you have to modify it
-  std::string json_file = "Inception-BN-symbol.json";
-  std::string param_file = "Inception-BN-0126.params";
-  std::string synset_file = "synset.txt";
-  std::string nd_file = "mean_224.nd";
+  std::string json_file = "model/Inception/Inception-BN-symbol.json";
+  std::string param_file = "model/Inception/Inception-BN-0126.params";
+  std::string synset_file = "model/Inception/synset.txt";
+  std::string nd_file = "model/Inception/mean_224.nd";
 
   BufferFile json_data(json_file);
   BufferFile param_data(param_file);

From d5560fefd73736d2fd03d65afe50d3e74052a81b Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Fri, 7 Sep 2018 14:50:13 -0700
Subject: [PATCH 15/20] fix.

---
 src/c_api/c_predict_api.cc | 36 +++++++++++++++++-------------------
 1 file changed, 17 insertions(+), 19 deletions(-)

diff --git a/src/c_api/c_predict_api.cc b/src/c_api/c_predict_api.cc
index 35e655b6647b..cd4bfb342df1 100644
--- a/src/c_api/c_predict_api.cc
+++ b/src/c_api/c_predict_api.cc
@@ -84,20 +84,20 @@ inline void _CreateExecutor(PredictorHandle pred_hnd) {
   }
 }
 
-int MXPredCreatePartialOut1(const char* symbol_json_str,
-                            const void* param_bytes,
-                            int param_size,
-                            int dev_type, int dev_id,
-                            mx_uint num_input_nodes,
-                            const char** input_keys,
-                            const mx_uint* input_shape_indptr,
-                            const mx_uint* input_shape_data,
-                            mx_uint num_output_nodes,
-                            const char** output_keys,
-                            // This is used for paralle inference.
-                            int num_threads,
-                            bool lazy,
-                            PredictorHandle* out) {
+int _CreatePartialOut(const char* symbol_json_str,
+                      const void* param_bytes,
+                      int param_size,
+                      int dev_type, int dev_id,
+                      mx_uint num_input_nodes,
+                      const char** input_keys,
+                      const mx_uint* input_shape_indptr,
+                      const mx_uint* input_shape_data,
+                      mx_uint num_output_nodes,
+                      const char** output_keys,
+                      // This is used for paralle inference.
+                      int num_threads,
+                      bool lazy,
+                      PredictorHandle* out) {
   using nnvm::Symbol;
 
   API_BEGIN();
@@ -258,10 +258,8 @@ int MXPredCreatePartialOut(const char* symbol_json_str,
                            const mx_uint* input_shape_data,
                            mx_uint num_output_nodes,
                            const char** output_keys,
-                           // This is used for paralle inference.
-                           int num_threads,
                            PredictorHandle* out) {
-  return MXPredCreatePartialOut1(
+  return _CreatePartialOut(
       symbol_json_str,
       param_bytes,
       param_size,
@@ -286,7 +284,7 @@ int MXPredCreate(const char* symbol_json_str,
                  const mx_uint* input_shape_indptr,
                  const mx_uint* input_shape_data,
                  PredictorHandle* out) {
-  return MXPredCreatePartialOut1(
+  return _CreatePartialOut(
       symbol_json_str,
       param_bytes,
       param_size,
@@ -314,7 +312,7 @@ int MXPredCreateMultithread(const char* symbol_json_str,
                             // This is used for paralle inference.
                             int num_threads,
                             PredictorHandle* out) {
-  return MXPredCreatePartialOut1(
+  return _CreatePartialOut(
       symbol_json_str,
       param_bytes,
       param_size,

From 811e99858694af24744148b2722a197ff64adc92 Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Fri, 7 Sep 2018 17:14:50 -0700
Subject: [PATCH 16/20] add check.

---
 src/c_api/c_predict_api.cc | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/c_api/c_predict_api.cc b/src/c_api/c_predict_api.cc
index cd4bfb342df1..dd8f0952ba13 100644
--- a/src/c_api/c_predict_api.cc
+++ b/src/c_api/c_predict_api.cc
@@ -312,6 +312,13 @@ int MXPredCreateMultithread(const char* symbol_json_str,
                             // This is used for paralle inference.
                             int num_threads,
                             PredictorHandle* out) {
+  const char *type = getenv("MXNET_ENGINE_TYPE");
+  std::string stype;
+  if (type)
+    stype = type;
+  CHECK(type == "NaiveEngine") << "Multithread inference only works with NaiveEngine.\n"
+                              << "Please set MXNET_ENGINE_TYPE to NaiveEngine"
+                              << std::endl;
   return _CreatePartialOut(
       symbol_json_str,
       param_bytes,

From 9376c8505c749e17fd33fc25b93372a1432efb4a Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Sat, 8 Sep 2018 00:21:14 +0000
Subject: [PATCH 17/20] fix.

---
 src/c_api/c_predict_api.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/c_api/c_predict_api.cc b/src/c_api/c_predict_api.cc
index dd8f0952ba13..7c0705145998 100644
--- a/src/c_api/c_predict_api.cc
+++ b/src/c_api/c_predict_api.cc
@@ -316,9 +316,9 @@ int MXPredCreateMultithread(const char* symbol_json_str,
   std::string stype;
   if (type)
     stype = type;
-  CHECK(type == "NaiveEngine") << "Multithread inference only works with NaiveEngine.\n"
-                              << "Please set MXNET_ENGINE_TYPE to NaiveEngine"
-                              << std::endl;
+  CHECK(stype == "NaiveEngine") << "Multithread inference only works with NaiveEngine.\n"
+      << "Please set MXNET_ENGINE_TYPE to NaiveEngine"
+      << std::endl;
   return _CreatePartialOut(
       symbol_json_str,
       param_bytes,

From 1a013428addf695134d9816d690d53379fc63c1d Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Sat, 8 Sep 2018 00:33:45 +0000
Subject: [PATCH 18/20] fix example.

---
 .../predict-cpp/image-classification-predict.cc   | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/example/image-classification/predict-cpp/image-classification-predict.cc b/example/image-classification/predict-cpp/image-classification-predict.cc
index 823038779ab3..bde4d244432e 100644
--- a/example/image-classification/predict-cpp/image-classification-predict.cc
+++ b/example/image-classification/predict-cpp/image-classification-predict.cc
@@ -181,8 +181,7 @@ void PrintOutputResult(const std::vector<float>& data, const std::vector<std::st
 }
 
 void predict(PredictorHandle pred_hnd, const std::vector<mx_float> &image_data,
-             NDListHandle nd_hnd, int i) {
-  std::string synset_file = "synset.txt";
+             NDListHandle nd_hnd, const std::string &synset_file, int i) {
   auto image_size = image_data.size();
   // Set Input Image
   MXPredSetInput(pred_hnd, "data", image_data.data(), static_cast<mx_uint>(image_size));
@@ -221,14 +220,16 @@ void predict(PredictorHandle pred_hnd, const std::vector<mx_float> &image_data,
 }
 
 int main(int argc, char* argv[]) {
-  if (argc < 3) {
+  if (argc < 2) {
     std::cout << "No test image here." << std::endl
-              << "Usage: ./image-classification-predict apple.jpg num_threads" << std::endl;
+              << "Usage: ./image-classification-predict apple.jpg [num_threads]" << std::endl;
     return EXIT_FAILURE;
   }
 
   std::string test_file(argv[1]);
-  int num_threads = std::atoi(argv[2]);
+  int num_threads = 1;
+  if (argc == 3)
+    num_threads = std::atoi(argv[2]);
 
   // Models path for your model, you have to modify it
   std::string json_file = "model/Inception/Inception-BN-symbol.json";
@@ -302,7 +303,7 @@ int main(int argc, char* argv[]) {
                  &pred_hnd);
     assert(pred_hnd);
 
-    predict(pred_hnd, image_data, nd_hnd, 0);
+    predict(pred_hnd, image_data, nd_hnd, synset_file, 0);
   } else {
     // Create Predictor
     std::vector<PredictorHandle> pred_hnds(num_threads, nullptr);
@@ -322,7 +323,7 @@ int main(int argc, char* argv[]) {
 
     std::vector<std::thread> threads;
     for (int i = 0; i < num_threads; i++)
-      threads.emplace_back(predict, pred_hnds[i], image_data, nd_hnd, i);
+      threads.emplace_back(predict, pred_hnds[i], image_data, nd_hnd, synset_file, i);
     for (int i = 0; i < num_threads; i++)
       threads[i].join();
   }

From 4c9c4772c0517a6275e2cbd55377b36c1578f923 Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Sat, 8 Sep 2018 00:36:43 +0000
Subject: [PATCH 19/20] update name.

---
 .../predict-cpp/image-classification-predict.cc                 | 2 +-
 include/mxnet/c_predict_api.h                                   | 2 +-
 src/c_api/c_predict_api.cc                                      | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/example/image-classification/predict-cpp/image-classification-predict.cc b/example/image-classification/predict-cpp/image-classification-predict.cc
index bde4d244432e..2a605b8b2674 100644
--- a/example/image-classification/predict-cpp/image-classification-predict.cc
+++ b/example/image-classification/predict-cpp/image-classification-predict.cc
@@ -307,7 +307,7 @@ int main(int argc, char* argv[]) {
   } else {
     // Create Predictor
     std::vector<PredictorHandle> pred_hnds(num_threads, nullptr);
-    MXPredCreateMultithread(static_cast<const char*>(json_data.GetBuffer()),
+    MXPredCreateMultiThread(static_cast<const char*>(json_data.GetBuffer()),
                             static_cast<const char*>(param_data.GetBuffer()),
                             static_cast<int>(param_data.GetLength()),
                             dev_type,
diff --git a/include/mxnet/c_predict_api.h b/include/mxnet/c_predict_api.h
index 2e0023f8869a..16addff6345f 100644
--- a/include/mxnet/c_predict_api.h
+++ b/include/mxnet/c_predict_api.h
@@ -141,7 +141,7 @@ MXNET_DLL int MXPredCreatePartialOut(const char* symbol_json_str,
  *   enough to keep `num_threads` predictors.
  * \return 0 when success, -1 when failure.
  */
-MXNET_DLL int MXPredCreateMultithread(const char* symbol_json_str,
+MXNET_DLL int MXPredCreateMultiThread(const char* symbol_json_str,
                                       const void* param_bytes,
                                       int param_size,
                                       int dev_type, int dev_id,
diff --git a/src/c_api/c_predict_api.cc b/src/c_api/c_predict_api.cc
index 7c0705145998..c2576cc8e0af 100644
--- a/src/c_api/c_predict_api.cc
+++ b/src/c_api/c_predict_api.cc
@@ -301,7 +301,7 @@ int MXPredCreate(const char* symbol_json_str,
       out);
 }
 
-int MXPredCreateMultithread(const char* symbol_json_str,
+int MXPredCreateMultiThread(const char* symbol_json_str,
                             const void* param_bytes,
                             int param_size,
                             int dev_type, int dev_id,

From c21ba35970f7ad00168940df6be11338a180d78c Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Tue, 11 Sep 2018 15:54:32 -0700
Subject: [PATCH 20/20] update README.

---
 example/image-classification/predict-cpp/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/example/image-classification/predict-cpp/README.md b/example/image-classification/predict-cpp/README.md
index 69f63d706006..2a5e350e4afb 100644
--- a/example/image-classification/predict-cpp/README.md
+++ b/example/image-classification/predict-cpp/README.md
@@ -1,5 +1,5 @@
 # Image Classification Example Using the C Predict API
-This is a simple predictor which shows how to use the MXNet C Predict API for image classification with a pre-trained ImageNet model.
+This is a simple predictor which shows how to use the MXNet C Predict API for image classification with a pre-trained ImageNet model in a single thread and multiple threads.
 
 ## Prerequisites
 
@@ -45,10 +45,10 @@ Run the example by passing it an image that you want to classify. If you don't h
   wget https://upload.wikimedia.org/wikipedia/commons/thumb/f/f4/Honeycrisp.jpg/1920px-Honeycrisp.jpg
   ```
 
-Then run the `image-classification-predict` program, passing the image as the argument.
+Then run the `image-classification-predict` program, passing the image as the first argument and the number of threads as the second parameter.
 
   ```bash
-  ./image-classification-predict 1920px-Honeycrisp.jpg
+  ./image-classification-predict 1920px-Honeycrisp.jpg 1
   ```
 
 ## Tips