From 1e17a516c121b49b7dbaa458a83f050dc8984396 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Tue, 18 Sep 2018 11:53:27 -0700
Subject: [PATCH 01/96] mkldnn is default makefile and explicitly turn off for
 buidls

---
 Makefile                       | 3 +++
 ci/docker/runtime_functions.sh | 7 +++++++
 2 files changed, 10 insertions(+)

diff --git a/Makefile b/Makefile
index 620679a44f80..bfe3e97d6f13 100644
--- a/Makefile
+++ b/Makefile
@@ -60,6 +60,9 @@ endif
 # use customized config file
 include $(config)
 
+ifneq ($(USE_MKLDNN), 0)
+    USE_MKLDNN = 1
+
 ifeq ($(USE_MKL2017), 1)
 $(warning "USE_MKL2017 is deprecated. We will switch to USE_MKLDNN.")
 	USE_MKLDNN=1
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 43006f23974d..55b8bfd6ac14 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -251,6 +251,7 @@ build_centos7_cpu() {
         USE_LAPACK_PATH=/usr/lib64/liblapack.so \
         USE_BLAS=openblas \
         USE_DIST_KVSTORE=1 \
+        USE_MKLDNN=0 \
         -j$(nproc)
 }
 
@@ -303,6 +304,7 @@ build_centos7_gpu() {
         USE_CUDA=1                                \
         USE_CUDA_PATH=/usr/local/cuda             \
         USE_CUDNN=1                               \
+        USE_MKLDNN=0                              \
         USE_DIST_KVSTORE=1                        \
         CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
         -j$(nproc)
@@ -318,6 +320,7 @@ build_ubuntu_cpu_openblas() {
     export CXX="ccache g++"
     make \
         DEV=1                         \
+        USE_MKLDNN=0                  \
         ENABLE_TESTCOVERAGE=1         \
         USE_CPP_PACKAGE=1             \
         USE_BLAS=openblas             \
@@ -385,6 +388,7 @@ build_ubuntu_cpu_clang39() {
         USE_BLAS=openblas             \
         USE_OPENMP=0                  \
         USE_DIST_KVSTORE=1            \
+        USE_MKLDNN=0                  \
         -j$(nproc)
 }
 
@@ -402,6 +406,7 @@ build_ubuntu_cpu_clang60() {
         USE_BLAS=openblas             \
         USE_OPENMP=1                  \
         USE_DIST_KVSTORE=1            \
+        USE_MKLDNN=0                  \
         -j$(nproc)
 }
 
@@ -534,6 +539,7 @@ build_ubuntu_gpu_tensorrt() {
         USE_TENSORRT=1                                       \
         USE_JEMALLOC=0                                       \
         USE_GPERFTOOLS=0                                     \
+        USE_MKLDNN=0                                         \
         ONNX_NAMESPACE=onnx                                  \
         CUDA_ARCH="-gencode arch=compute_70,code=compute_70" \
         -j$(nproc)
@@ -588,6 +594,7 @@ build_ubuntu_gpu_cuda91_cudnn7() {
         USE_CPP_PACKAGE=1                         \
         USE_DIST_KVSTORE=1                        \
         CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
+        USE_MKLDNN=0                              \
         -j$(nproc)
 }
 

From abbc3ad526e5ae09334a29d72c87e597a1554993 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Tue, 18 Sep 2018 13:44:24 -0700
Subject: [PATCH 02/96] add endif

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index bfe3e97d6f13..18543fdc917f 100644
--- a/Makefile
+++ b/Makefile
@@ -62,6 +62,7 @@ include $(config)
 
 ifneq ($(USE_MKLDNN), 0)
     USE_MKLDNN = 1
+endif
 
 ifeq ($(USE_MKL2017), 1)
 $(warning "USE_MKL2017 is deprecated. We will switch to USE_MKLDNN.")

From 92b91f8a19069255d26fa841e06a9ffb2739b949 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 29 Oct 2018 18:06:55 -0700
Subject: [PATCH 03/96] retrigger


From ce5336cb94a912fe6693c3a7c8603fbb406c5c46 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 31 Oct 2018 13:28:44 -0700
Subject: [PATCH 04/96] retrigger


From b8a0203a1f296c12cf9659c015f829d817f15ec2 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 8 Nov 2018 17:58:33 -0800
Subject: [PATCH 05/96] build mkldnn as static lib

---
 CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5cea683e8f56..abe52ca4d6a3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -224,6 +224,7 @@ if(ENABLE_TESTCOVERAGE)
 endif()
 
 if(USE_MKLDNN)
+  SET(MKLDNN_LIBRARY_TYPE STATIC)
   include(cmake/MklDnn.cmake)
   # CPU architecture (e.g., C5) can't run on another architecture (e.g., g3).
   if(NOT MSVC)

From bc6c482132856e0acd3060ed17fdad04d2f418c6 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 8 Nov 2018 18:19:33 -0800
Subject: [PATCH 06/96] update makefile to statically build mkldnn

---
 mkldnn.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mkldnn.mk b/mkldnn.mk
index d79bbe7d2a0e..4364f14ba0bc 100644
--- a/mkldnn.mk
+++ b/mkldnn.mk
@@ -37,7 +37,7 @@ mkldnn_build: $(MKLDNN_LIBFILE)
 $(MKLDNN_LIBFILE):
 	mkdir -p $(MKLDNNROOT)
 	cd $(MKLDNN_SUBMODDIR) && rm -rf external && cd scripts && ./prepare_mkl.sh && cd .. && cp -a external/*/* $(MKLDNNROOT)/.
-	cmake $(MKLDNN_SUBMODDIR) -DCMAKE_INSTALL_PREFIX=$(MKLDNNROOT) -B$(MKLDNN_BUILDDIR) -DARCH_OPT_FLAGS="-mtune=generic" -DWITH_TEST=OFF -DWITH_EXAMPLE=OFF
+	cmake $(MKLDNN_SUBMODDIR) -DCMAKE_INSTALL_PREFIX=$(MKLDNNROOT) -B$(MKLDNN_BUILDDIR) -DARCH_OPT_FLAGS="-mtune=generic" -DWITH_TEST=OFF -DWITH_EXAMPLE=OFF -DMKLDNN_LIBRARY_TYPE=STATIC
 	$(MAKE) -C $(MKLDNN_BUILDDIR) VERBOSE=1
 	$(MAKE) -C $(MKLDNN_BUILDDIR) install
 	mkdir -p $(MXNET_LIBDIR)

From 15a41fc816fe68ecd8d6e07154f5775003458dde Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 8 Nov 2018 18:43:53 -0800
Subject: [PATCH 07/96] build static mkldnn

---
 Makefile  | 3 +--
 mkldnn.mk | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index b12311736647..f3d3360096f8 100644
--- a/Makefile
+++ b/Makefile
@@ -131,8 +131,7 @@ ifeq ($(USE_MKLDNN), 1)
 		CFLAGS += -I$(MKLROOT)/include
 		LDFLAGS += -L$(MKLROOT)/lib
 	endif
-	CFLAGS += -I$(MKLDNNROOT)/include
-	LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,'$${ORIGIN}'
+	LIB_DEP += $(MKLDNNROOT)/lib/libmkldnn.a.0
 endif
 
 # setup opencv
diff --git a/mkldnn.mk b/mkldnn.mk
index 4364f14ba0bc..f84d4cd743e4 100644
--- a/mkldnn.mk
+++ b/mkldnn.mk
@@ -26,7 +26,7 @@ ifeq ($(UNAME_S), Darwin)
 else
 	OMP_LIBFILE = $(MKLDNNROOT)/lib/libiomp5.so
 	MKLML_LIBFILE = $(MKLDNNROOT)/lib/libmklml_intel.so
-	MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.so.0
+	MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.a.0
 endif
 endif
 

From 42b3353ea1a968df004482cb0b188839f575286d Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 8 Nov 2018 18:49:20 -0800
Subject: [PATCH 08/96] fix static name

---
 mkldnn.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mkldnn.mk b/mkldnn.mk
index f84d4cd743e4..3cb6062cd221 100644
--- a/mkldnn.mk
+++ b/mkldnn.mk
@@ -26,7 +26,7 @@ ifeq ($(UNAME_S), Darwin)
 else
 	OMP_LIBFILE = $(MKLDNNROOT)/lib/libiomp5.so
 	MKLML_LIBFILE = $(MKLDNNROOT)/lib/libmklml_intel.so
-	MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.a.0
+	MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.a
 endif
 endif
 

From 5af258ae73d24a8d1103f915ee7ab0db26ff5f2e Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 8 Nov 2018 18:52:30 -0800
Subject: [PATCH 09/96] fix static name

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index f3d3360096f8..a92b02e08394 100644
--- a/Makefile
+++ b/Makefile
@@ -131,7 +131,7 @@ ifeq ($(USE_MKLDNN), 1)
 		CFLAGS += -I$(MKLROOT)/include
 		LDFLAGS += -L$(MKLROOT)/lib
 	endif
-	LIB_DEP += $(MKLDNNROOT)/lib/libmkldnn.a.0
+	LIB_DEP += $(MKLDNNROOT)/lib/libmkldnn.a
 endif
 
 # setup opencv

From 32ab9ce9649044541460a30f2d172d6622f36e27 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 8 Nov 2018 18:57:42 -0800
Subject: [PATCH 10/96] update static for mac

---
 mkldnn.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mkldnn.mk b/mkldnn.mk
index 3cb6062cd221..cd5bd8556dfe 100644
--- a/mkldnn.mk
+++ b/mkldnn.mk
@@ -22,7 +22,7 @@ ifeq ($(USE_MKLDNN), 1)
 ifeq ($(UNAME_S), Darwin)
 	OMP_LIBFILE = $(MKLDNNROOT)/lib/libiomp5.dylib
 	MKLML_LIBFILE = $(MKLDNNROOT)/lib/libmklml.dylib
-	MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.0.dylib
+	MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.a
 else
 	OMP_LIBFILE = $(MKLDNNROOT)/lib/libiomp5.so
 	MKLML_LIBFILE = $(MKLDNNROOT)/lib/libmklml_intel.so

From e2422d63cd450274ba3236bc6e5e5069ece82967 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 12 Nov 2018 14:03:33 -0800
Subject: [PATCH 11/96] rename mkldnn dep in ci

---
 Jenkinsfile                    | 4 ++--
 ci/docker/runtime_functions.sh | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3f72843596e7..600e8448c13b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -32,8 +32,8 @@ mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3r
 mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
 mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests'
-mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
-mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
+mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.a'
+mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.a, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
 mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so'
 mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/lenet, build/cpp-package/example/alexnet, build/cpp-package/example/googlenet, build/cpp-package/example/lenet_with_mxdataiter, build/cpp-package/example/resnet, build/cpp-package/example/mlp, build/cpp-package/example/mlp_cpu, build/cpp-package/example/mlp_gpu, build/cpp-package/example/test_score, build/cpp-package/example/test_optimizer'
 mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/mlp_cpu'
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 27c93b5049ef..f571a54e4c29 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -629,8 +629,8 @@ build_ubuntu_gpu_cmake_mkldnn() {
 
     ninja -v
     # libmkldnn.so.0 is a link file. We need an actual binary file named libmkldnn.so.0.
-    cp 3rdparty/mkldnn/src/libmkldnn.so.0 3rdparty/mkldnn/src/libmkldnn.so.0.tmp
-    mv 3rdparty/mkldnn/src/libmkldnn.so.0.tmp 3rdparty/mkldnn/src/libmkldnn.so.0
+#    cp 3rdparty/mkldnn/src/libmkldnn.so.0 3rdparty/mkldnn/src/libmkldnn.so.0.tmp
+#    mv 3rdparty/mkldnn/src/libmkldnn.so.0.tmp 3rdparty/mkldnn/src/libmkldnn.so.0
 }
 
 build_ubuntu_gpu_cmake() {

From 372f6978283903092d65101f0efb6dead2fd37b1 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 12 Nov 2018 15:09:20 -0800
Subject: [PATCH 12/96] remove moving mkldnn dynamic lib

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 600e8448c13b..655135f5ad59 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -32,8 +32,8 @@ mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3r
 mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
 mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests'
-mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.a'
-mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.a, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
+mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
+mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
 mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so'
 mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/lenet, build/cpp-package/example/alexnet, build/cpp-package/example/googlenet, build/cpp-package/example/lenet_with_mxdataiter, build/cpp-package/example/resnet, build/cpp-package/example/mlp, build/cpp-package/example/mlp_cpu, build/cpp-package/example/mlp_gpu, build/cpp-package/example/test_score, build/cpp-package/example/test_optimizer'
 mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/mlp_cpu'

From 67e4dff4c718401d1552d8bfb54f83654ea4aaf7 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 12 Nov 2018 16:06:03 -0800
Subject: [PATCH 13/96] retrigger


From 150b324572fd445bfcbf6863c60e727b4f42bd4f Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 12 Nov 2018 16:50:50 -0800
Subject: [PATCH 14/96] remove commented code

---
 ci/docker/runtime_functions.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index f571a54e4c29..7ac8b0297990 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -628,9 +628,6 @@ build_ubuntu_gpu_cmake_mkldnn() {
         /work/mxnet
 
     ninja -v
-    # libmkldnn.so.0 is a link file. We need an actual binary file named libmkldnn.so.0.
-#    cp 3rdparty/mkldnn/src/libmkldnn.so.0 3rdparty/mkldnn/src/libmkldnn.so.0.tmp
-#    mv 3rdparty/mkldnn/src/libmkldnn.so.0.tmp 3rdparty/mkldnn/src/libmkldnn.so.0
 }
 
 build_ubuntu_gpu_cmake() {

From 89b11c6e8094ccaba3a441a3e433c0d03307992b Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Tue, 13 Nov 2018 13:25:49 -0800
Subject: [PATCH 15/96] retrigger


From 40fd0ac94e00d91b05f041da1c2ddd9236b7f7a4 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 14 Nov 2018 12:25:48 -0800
Subject: [PATCH 16/96] remove mkldnn dnaymic for unitest

---
 tests/cpp/unittest.mk | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/cpp/unittest.mk b/tests/cpp/unittest.mk
index 746ee2f096f1..665ce6982874 100644
--- a/tests/cpp/unittest.mk
+++ b/tests/cpp/unittest.mk
@@ -41,22 +41,22 @@ gtest-all.o : $(GTEST_SRCS_)
 gtest.a : gtest-all.o
 	$(AR) $(ARFLAGS) $@ $^
 
-build/tests/cpp/%.o : tests/cpp/%.cc | mkldnn
+build/tests/cpp/%.o : tests/cpp/%.cc
 	@mkdir -p $(@D)
 	$(CXX) -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/$* $< > build/tests/cpp/$*.d
 	$(CXX) -c -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/$*.o $(filter %.cc %.a, $^)
 
-build/tests/cpp/operator/%.o : tests/cpp/operator/%.cc | mkldnn
+build/tests/cpp/operator/%.o : tests/cpp/operator/%.cc
 	@mkdir -p $(@D)
 	$(CXX) -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/operator/$* $< > build/tests/cpp/operator/$*.d
 	$(CXX) -c -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/operator/$*.o $(filter %.cc %.a, $^)
 
-build/tests/cpp/storage/%.o : tests/cpp/storage/%.cc | mkldnn
+build/tests/cpp/storage/%.o : tests/cpp/storage/%.cc
 	@mkdir -p $(@D)
 	$(CXX) -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/storage/$* $< > build/tests/cpp/storage/$*.d
 	$(CXX) -c -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/storage/$*.o $(filter %.cc %.a, $^)
 
-build/tests/cpp/engine/%.o : tests/cpp/engine/%.cc | mkldnn
+build/tests/cpp/engine/%.o : tests/cpp/engine/%.cc
 	@mkdir -p $(@D)
 	$(CXX) -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/engine/$* $< > build/tests/cpp/engine/$*.d
 	$(CXX) -c -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/engine/$*.o $(filter %.cc %.a, $^)

From cb095c6594148a650c4c689592d71c1af251580b Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 21 Nov 2018 14:25:02 -0800
Subject: [PATCH 17/96] retrigger


From c08f6face171389b51a7ac55cb7a930a8623b503 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 21 Nov 2018 16:35:19 -0800
Subject: [PATCH 18/96] retrigger


From 0302290659bb7da3fbd46e4ac85f0c266b3aa014 Mon Sep 17 00:00:00 2001
From: Alexander Zai <azai91@gmail.com>
Date: Thu, 22 Nov 2018 11:52:12 -0800
Subject: [PATCH 19/96] force static for mkldnn lib

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ba76569c7101..6b1c858326fc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -225,7 +225,7 @@ endif()
 
 if(USE_MKLDNN)
   include(cmake/DownloadMKLML.cmake)
-  SET(MKLDNN_LIBRARY_TYPE STATIC)
+  set(MKLDNN_LIBRARY_TYPE "STATIC" CACHE INTERNAL "" FORCE)
   # CPU architecture (e.g., C5) can't run on another architecture (e.g., g3).
   if(NOT MSVC)
     set(ARCH_OPT_FLAGS "-mtune=generic")

From bf78666c88e27624fa4d971afbdcf6ca5175fc8f Mon Sep 17 00:00:00 2001
From: Alexander Zai <azai91@gmail.com>
Date: Thu, 22 Nov 2018 11:55:27 -0800
Subject: [PATCH 20/96] turn of mkldnn on arm builds

---
 ci/docker/runtime_functions.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 45eb09f59d10..6b5397c7ff69 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -139,6 +139,7 @@ build_armv6() {
         -DCMAKE_BUILD_TYPE=Release \
         -DUSE_MKL_IF_AVAILABLE=OFF \
         -DUSE_LAPACK=OFF \
+        -DUSE_MKLDNN=0FF \
         -DBUILD_CPP_EXAMPLES=OFF \
         -Dmxnet_LINKER_LIBS=-lgfortran \
         -G Ninja /work/mxnet
@@ -170,6 +171,7 @@ build_armv7() {
         -DCMAKE_BUILD_TYPE=Release \
         -DUSE_MKL_IF_AVAILABLE=OFF \
         -DUSE_LAPACK=OFF \
+        -DUSE_MKLDNN=0FF \
         -DBUILD_CPP_EXAMPLES=OFF \
         -Dmxnet_LINKER_LIBS=-lgfortran \
         -G Ninja /work/mxnet
@@ -190,6 +192,7 @@ build_armv8() {
         -DUSE_LAPACK=OFF\
         -DUSE_SIGNAL_HANDLER=ON\
         -DCMAKE_BUILD_TYPE=Release\
+        -DUSE_MKLDNN=0FF \
         -DUSE_MKL_IF_AVAILABLE=OFF\
         -G Ninja /work/mxnet
     ninja -v
@@ -215,6 +218,7 @@ build_android_armv7() {
         -DUSE_OPENCV=OFF\
         -DUSE_OPENMP=OFF\
         -DUSE_SIGNAL_HANDLER=ON\
+        -DUSE_MKLDNN=0FF \
         -DCMAKE_BUILD_TYPE=RelWithDebInfo\
         -DUSE_MKL_IF_AVAILABLE=OFF\
         -G Ninja /work/mxnet
@@ -234,6 +238,7 @@ build_android_armv8() {
         -DUSE_SIGNAL_HANDLER=ON\
         -DCMAKE_BUILD_TYPE=RelWithDebInfo\
         -DUSE_MKL_IF_AVAILABLE=OFF\
+        -DUSE_MKLDNN=0FF \
         -G Ninja /work/mxnet
     ninja -v
 }

From d103ec85e84e3a4eb0538b88dcb36788289777a0 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Tue, 27 Nov 2018 17:38:03 -0800
Subject: [PATCH 21/96] remove dynamic mkldnn bind

---
 ci/jenkins/Jenkins_steps.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index f48a26737308..e12b4ed04110 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -34,7 +34,7 @@ mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3r
 mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
 mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests'
-mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
+mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so
 mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
 mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so'
 mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*'

From 4b07dcf120419cd90b176a0a0a6cc0badee865ef Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 29 Nov 2018 12:07:50 -0800
Subject: [PATCH 22/96] update jenkins to use only mkldnn

---
 ci/docker/runtime_functions.sh | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 4f2f0f0d8994..babb73158b1f 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -140,7 +140,6 @@ build_armv6() {
         -DCMAKE_BUILD_TYPE=Release \
         -DUSE_MKL_IF_AVAILABLE=OFF \
         -DUSE_LAPACK=OFF \
-        -DUSE_MKLDNN=0FF \
         -DBUILD_CPP_EXAMPLES=OFF \
         -Dmxnet_LINKER_LIBS=-lgfortran \
         -G Ninja /work/mxnet
@@ -172,7 +171,6 @@ build_armv7() {
         -DCMAKE_BUILD_TYPE=Release \
         -DUSE_MKL_IF_AVAILABLE=OFF \
         -DUSE_LAPACK=OFF \
-        -DUSE_MKLDNN=0FF \
         -DBUILD_CPP_EXAMPLES=OFF \
         -Dmxnet_LINKER_LIBS=-lgfortran \
         -G Ninja /work/mxnet
@@ -219,7 +217,6 @@ build_android_armv7() {
         -DUSE_OPENCV=OFF\
         -DUSE_OPENMP=OFF\
         -DUSE_SIGNAL_HANDLER=ON\
-        -DUSE_MKLDNN=0FF \
         -DCMAKE_BUILD_TYPE=RelWithDebInfo\
         -DUSE_MKL_IF_AVAILABLE=OFF\
         -G Ninja /work/mxnet
@@ -239,7 +236,6 @@ build_android_armv8() {
         -DUSE_SIGNAL_HANDLER=ON\
         -DCMAKE_BUILD_TYPE=RelWithDebInfo\
         -DUSE_MKL_IF_AVAILABLE=OFF\
-        -DUSE_MKLDNN=0FF \
         -G Ninja /work/mxnet
     ninja -v
 }
@@ -257,7 +253,6 @@ build_centos7_cpu() {
         USE_LAPACK_PATH=/usr/lib64/liblapack.so \
         USE_BLAS=openblas \
         USE_DIST_KVSTORE=1 \
-        USE_MKLDNN=0 \
         -j$(nproc)
 }
 
@@ -291,7 +286,6 @@ build_centos7_mkldnn() {
         ENABLE_TESTCOVERAGE=1 \
         USE_LAPACK=1 \
         USE_LAPACK_PATH=/usr/lib64/liblapack.so \
-        USE_MKLDNN=1 \
         USE_BLAS=openblas \
         -j$(nproc)
 }
@@ -310,7 +304,6 @@ build_centos7_gpu() {
         USE_CUDA=1                                \
         USE_CUDA_PATH=/usr/local/cuda             \
         USE_CUDNN=1                               \
-        USE_MKLDNN=0                              \
         USE_DIST_KVSTORE=1                        \
         CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
         -j$(nproc)
@@ -326,7 +319,6 @@ build_ubuntu_cpu_openblas() {
     export CXX="ccache g++"
     make \
         DEV=1                         \
-        USE_MKLDNN=0                  \
         ENABLE_TESTCOVERAGE=1         \
         USE_CPP_PACKAGE=1             \
         USE_BLAS=openblas             \
@@ -394,7 +386,6 @@ build_ubuntu_cpu_clang39() {
         USE_BLAS=openblas             \
         USE_OPENMP=0                  \
         USE_DIST_KVSTORE=1            \
-        USE_MKLDNN=0                  \
         -j$(nproc)
 }
 
@@ -412,7 +403,6 @@ build_ubuntu_cpu_clang60() {
         USE_BLAS=openblas             \
         USE_OPENMP=1                  \
         USE_DIST_KVSTORE=1            \
-        USE_MKLDNN=0                  \
         -j$(nproc)
 }
 
@@ -454,7 +444,6 @@ build_ubuntu_cpu_clang39_mkldnn() {
         ENABLE_TESTCOVERAGE=1         \
         USE_CPP_PACKAGE=1             \
         USE_BLAS=openblas             \
-        USE_MKLDNN=1                  \
         USE_OPENMP=0                  \
         -j$(nproc)
 }
@@ -471,7 +460,6 @@ build_ubuntu_cpu_clang60_mkldnn() {
         ENABLE_TESTCOVERAGE=1         \
         USE_CPP_PACKAGE=1             \
         USE_BLAS=openblas             \
-        USE_MKLDNN=1                  \
         USE_OPENMP=1                  \
         -j$(nproc)
 }
@@ -486,7 +474,6 @@ build_ubuntu_cpu_mkldnn() {
         ENABLE_TESTCOVERAGE=1         \
         USE_CPP_PACKAGE=1             \
         USE_BLAS=openblas             \
-        USE_MKLDNN=1                  \
         -j$(nproc)
 }
 
@@ -545,7 +532,6 @@ build_ubuntu_gpu_tensorrt() {
         USE_TENSORRT=1                                       \
         USE_JEMALLOC=0                                       \
         USE_GPERFTOOLS=0                                     \
-        USE_MKLDNN=0                                         \
         ONNX_NAMESPACE=onnx                                  \
         CUDA_ARCH="-gencode arch=compute_70,code=compute_70" \
         -j$(nproc)
@@ -561,7 +547,6 @@ build_ubuntu_gpu_mkldnn() {
         ENABLE_TESTCOVERAGE=1                     \
         USE_CPP_PACKAGE=1                         \
         USE_BLAS=openblas                         \
-        USE_MKLDNN=1                              \
         USE_CUDA=1                                \
         USE_CUDA_PATH=/usr/local/cuda             \
         USE_CUDNN=1                               \
@@ -578,7 +563,6 @@ build_ubuntu_gpu_mkldnn_nocudnn() {
         DEV=1                                     \
         ENABLE_TESTCOVERAGE=1                     \
         USE_BLAS=openblas                         \
-        USE_MKLDNN=1                              \
         USE_CUDA=1                                \
         USE_CUDA_PATH=/usr/local/cuda             \
         USE_CUDNN=0                               \
@@ -600,7 +584,6 @@ build_ubuntu_gpu_cuda91_cudnn7() {
         USE_CPP_PACKAGE=1                         \
         USE_DIST_KVSTORE=1                        \
         CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
-        USE_MKLDNN=0                              \
         -j$(nproc)
 }
 
@@ -656,7 +639,6 @@ build_ubuntu_gpu_cmake() {
         -DUSE_CUDA=1                            \
         -DUSE_CUDNN=1                           \
         -DUSE_MKLML_MKL=0                       \
-        -DUSE_MKLDNN=0                          \
         -DUSE_DIST_KVSTORE=1                    \
         -DCMAKE_BUILD_TYPE=Release              \
         -DCUDA_ARCH_NAME=Manual                 \

From 6536cda6a87ab2ef804c0e320dfcbe2a8e56eb8d Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 29 Nov 2018 12:30:25 -0800
Subject: [PATCH 23/96] remove last flag

---
 ci/docker/runtime_functions.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index babb73158b1f..081a1475b5c4 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -191,7 +191,6 @@ build_armv8() {
         -DUSE_LAPACK=OFF\
         -DUSE_SIGNAL_HANDLER=ON\
         -DCMAKE_BUILD_TYPE=Release\
-        -DUSE_MKLDNN=0FF \
         -DUSE_MKL_IF_AVAILABLE=OFF\
         -G Ninja /work/mxnet
     ninja -v

From de4ff3112739012fac2fcc617665699b36d4b82b Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 29 Nov 2018 12:41:06 -0800
Subject: [PATCH 24/96] turn mkldnn by default on mac

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3b8bbd2e0272..f09a9c29e9d8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -20,7 +20,7 @@ mxnet_option(USE_F16C             "Build with x86 F16C instruction support" ON)
 mxnet_option(USE_LAPACK           "Build with lapack support" ON)
 mxnet_option(USE_MKL_IF_AVAILABLE "Use MKL if found" ON)
 mxnet_option(USE_MKLML_MKL        "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND (NOT APPLE))
-mxnet_option(USE_MKLDNN           "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND (NOT APPLE))
+mxnet_option(USE_MKLDNN           "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE)
 mxnet_option(USE_OPERATOR_TUNING  "Enable auto-tuning of operators" ON IF NOT MSVC)
 mxnet_option(USE_GPERFTOOLS       "Build with GPerfTools support (if found)" ON)
 mxnet_option(USE_JEMALLOC         "Build with Jemalloc support"   ON)

From 0e5a36286220d4917b0daae8c7431ea290aadf58 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 29 Nov 2018 15:11:25 -0800
Subject: [PATCH 25/96] move mkldnn files for GPU MKLDNN build

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 015ca81bad76..a05cfa98fc6a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -372,7 +372,7 @@ core_logic: {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.init_git()
             utils.docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake', false)
-            utils.pack_lib('cmake_gpu', mx_cmake_lib, true)
+            utils.pack_lib('cmake_gpu', mx_cmake_mkldnn_lib, true)
           }
         }
       }
@@ -772,7 +772,7 @@ core_logic: {
       node(NODE_LINUX_GPU) {
         ws('workspace/ut-cpp-gpu') {
           timeout(time: max_time, unit: 'MINUTES') {
-            utils.unpack_and_init('cmake_gpu', mx_cmake_lib, true)
+            utils.unpack_and_init('cmake_gpu', mx_cmake_mkldnn_lib, true)
             utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true)
             utils.publish_test_coverage()
           }

From 6eadfae419422fb631965958041ecec14f2e514e Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 29 Nov 2018 17:10:28 -0800
Subject: [PATCH 26/96] copy lib mxnet in gpu build

---
 ci/docker/runtime_functions.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 081a1475b5c4..e7f2f62d120c 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -646,6 +646,9 @@ build_ubuntu_gpu_cmake() {
         /work/mxnet
 
     ninja -v
+    # libmkldnn.so.0 is a link file. We need an actual binary file named libmkldnn.so.0.
+    cp 3rdparty/mkldnn/src/libmkldnn.so.0 3rdparty/mkldnn/src/libmkldnn.so.0.tmp
+    mv 3rdparty/mkldnn/src/libmkldnn.so.0.tmp 3rdparty/mkldnn/src/libmkldnn.so.0
 }
 
 build_ubuntu_blc() {

From 0d0f40723388211b50357f97abe1458fc5887649 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 29 Nov 2018 17:36:53 -0800
Subject: [PATCH 27/96] only link windows

---
 CMakeLists.txt | 3 ++-
 Makefile       | 8 +++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 205262b85125..5f35cd8934bc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -225,11 +225,12 @@ endif()
 
 if(USE_MKLDNN)
   include(cmake/DownloadMKLML.cmake)
-  set(MKLDNN_LIBRARY_TYPE "STATIC" CACHE INTERNAL "" FORCE)
   # CPU architecture (e.g., C5) can't run on another architecture (e.g., g3).
   if(NOT MSVC)
+    set(MKLDNN_LIBRARY_TYPE "STATIC")
     set(ARCH_OPT_FLAGS "-mtune=generic")
   else()
+    set(MKLDNN_LIBRARY_TYPE "SHARED")
     set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /EHsc")
     set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /EHsc /Gy")
   endif()
diff --git a/Makefile b/Makefile
index 30c21636a6ea..dd05e2157822 100644
--- a/Makefile
+++ b/Makefile
@@ -131,7 +131,13 @@ ifeq ($(USE_MKLDNN), 1)
 		CFLAGS += -I$(MKLROOT)/include
 		LDFLAGS += -L$(MKLROOT)/lib
 	endif
-	LIB_DEP += $(MKLDNNROOT)/lib/libmkldnn.a
+
+	ifneq ($(UNAME_S), Windows)
+	    LIB_DEP += $(MKLDNNROOT)/lib/libmkldnn.a
+    else
+        CFLAGS += -I$(MKLDNNROOT)/include
+        LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,'$${ORIGIN}'
+    endif
 endif
 
 # setup opencv

From b300b88bea2fef4a4a7d2030c3ff2b3384886c87 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 29 Nov 2018 17:43:36 -0800
Subject: [PATCH 28/96] add mkldnn.mk

---
 mkldnn.mk | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/mkldnn.mk b/mkldnn.mk
index cd5bd8556dfe..c9ca55c975ae 100644
--- a/mkldnn.mk
+++ b/mkldnn.mk
@@ -19,10 +19,16 @@ ifeq ($(USE_MKLDNN), 1)
 	MKLDNN_SUBMODDIR = $(ROOTDIR)/3rdparty/mkldnn
 	MKLDNN_BUILDDIR = $(MKLDNN_SUBMODDIR)/build
 	MXNET_LIBDIR = $(ROOTDIR)/lib
+	MKLDNN_LIBRARY_TYPE=STATIC
 ifeq ($(UNAME_S), Darwin)
 	OMP_LIBFILE = $(MKLDNNROOT)/lib/libiomp5.dylib
 	MKLML_LIBFILE = $(MKLDNNROOT)/lib/libmklml.dylib
 	MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.a
+else ifeq ($(UNAME_S), Windows)
+	OMP_LIBFILE = $(MKLDNNROOT)/lib/libiomp5.so
+    MKLML_LIBFILE = $(MKLDNNROOT)/lib/libmklml_intel.so
+    MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.so
+    MKLDNN_LIBRARY_TYPE=SHARED
 else
 	OMP_LIBFILE = $(MKLDNNROOT)/lib/libiomp5.so
 	MKLML_LIBFILE = $(MKLDNNROOT)/lib/libmklml_intel.so
@@ -37,7 +43,7 @@ mkldnn_build: $(MKLDNN_LIBFILE)
 $(MKLDNN_LIBFILE):
 	mkdir -p $(MKLDNNROOT)
 	cd $(MKLDNN_SUBMODDIR) && rm -rf external && cd scripts && ./prepare_mkl.sh && cd .. && cp -a external/*/* $(MKLDNNROOT)/.
-	cmake $(MKLDNN_SUBMODDIR) -DCMAKE_INSTALL_PREFIX=$(MKLDNNROOT) -B$(MKLDNN_BUILDDIR) -DARCH_OPT_FLAGS="-mtune=generic" -DWITH_TEST=OFF -DWITH_EXAMPLE=OFF -DMKLDNN_LIBRARY_TYPE=STATIC
+	cmake $(MKLDNN_SUBMODDIR) -DCMAKE_INSTALL_PREFIX=$(MKLDNNROOT) -B$(MKLDNN_BUILDDIR) -DARCH_OPT_FLAGS="-mtune=generic" -DWITH_TEST=OFF -DWITH_EXAMPLE=OFF -DMKLDNN_LIBRARY_TYPE=$(MKLDNN_LIBRARY_TYPE)
 	$(MAKE) -C $(MKLDNN_BUILDDIR) VERBOSE=1
 	$(MAKE) -C $(MKLDNN_BUILDDIR) install
 	mkdir -p $(MXNET_LIBDIR)

From b336ef029db0a730819c5b1e20409c0f01a2d669 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 29 Nov 2018 18:23:59 -0800
Subject: [PATCH 29/96] try force linking

---
 CMakeLists.txt | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5f35cd8934bc..161705643194 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -227,10 +227,9 @@ if(USE_MKLDNN)
   include(cmake/DownloadMKLML.cmake)
   # CPU architecture (e.g., C5) can't run on another architecture (e.g., g3).
   if(NOT MSVC)
-    set(MKLDNN_LIBRARY_TYPE "STATIC")
+    set(MKLDNN_LIBRARY_TYPE "STATIC" CACHE INTERNAL "" FORCE)
     set(ARCH_OPT_FLAGS "-mtune=generic")
   else()
-    set(MKLDNN_LIBRARY_TYPE "SHARED")
     set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /EHsc")
     set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /EHsc /Gy")
   endif()

From b9be8236739801ac08033bc6fe5d58242e12081b Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 29 Nov 2018 18:56:49 -0800
Subject: [PATCH 30/96] retrigger


From fdcee0d4bec144cf4b80acec3a24a79286b4d87c Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 29 Nov 2018 19:35:24 -0800
Subject: [PATCH 31/96] retrigger


From 46ee0bd935dc51c7e3e018db263f3b5f642d141e Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 29 Nov 2018 20:26:16 -0800
Subject: [PATCH 32/96] remove mkldnn dynanmic check

---
 tests/python/mkl/test_mkldnn.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/tests/python/mkl/test_mkldnn.py b/tests/python/mkl/test_mkldnn.py
index c6c0a0832f1f..d9d3abfc3ced 100644
--- a/tests/python/mkl/test_mkldnn.py
+++ b/tests/python/mkl/test_mkldnn.py
@@ -27,7 +27,6 @@
 from mxnet import gluon
 from mxnet.gluon import nn
 from mxnet.test_utils import *
-import test_mkldnn_install as install
 curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
 sys.path.append(os.path.join(curr_path, '../unittest/'))
 from common import with_seed
@@ -441,7 +440,4 @@ def backward(self, req, out_grad, in_data, out_data, in_grad, aux):
     custom = mx.symbol.Custom(name='custom', data=conv, op_type='custom')
     exec1 = custom.bind(mx.cpu(), args={'data': mx.nd.ones([10,3,96,96]), 'conv_weight': mx.nd.ones([8,3,5,5])})
     exec1.forward()[0].wait_to_read()
-
-
-if __name__ == '__main__':
-    install.test_mkldnn_install()
+    

From 7772cdd34a773ef27ce792c7970659ea337e7e1f Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 29 Nov 2018 21:05:00 -0800
Subject: [PATCH 33/96] use ifndef

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index e9d1f7942a70..3c7ca0c3f25c 100644
--- a/Makefile
+++ b/Makefile
@@ -60,7 +60,7 @@ endif
 # use customized config file
 include $(config)
 
-ifneq ($(USE_MKLDNN), 0)
+ifndef $(USE_MKLDNN)
     USE_MKLDNN = 1
 endif
 

From 45e8cd89ad11fde9f56d33a04534b99841a4f9d1 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 29 Nov 2018 21:27:44 -0800
Subject: [PATCH 34/96] remove test mkldnn install

---
 tests/python/mkl/test_mkldnn_install.py | 56 -------------------------
 1 file changed, 56 deletions(-)
 delete mode 100644 tests/python/mkl/test_mkldnn_install.py

diff --git a/tests/python/mkl/test_mkldnn_install.py b/tests/python/mkl/test_mkldnn_install.py
deleted file mode 100644
index c2f26df72f2e..000000000000
--- a/tests/python/mkl/test_mkldnn_install.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""
-MKL-DNN related test cases
-"""
-
-import sys
-import os
-import logging
-
-
-def test_mkldnn_install():
-    """
-    This test will verify that MXNet is built/installed correctly when
-    compiled with Intel MKL-DNN library. The method will try to import
-    the mxnet module and see if the mkldnn library is mapped to this
-    process's address space.
-    """
-    logging.basicConfig(level=logging.INFO)
-
-    if not sys.platform.startswith('linux'):
-        logging.info("Bypass mkldnn install test for non-Linux OS")
-        return
-
-    try:
-        #pylint: disable=unused-variable
-        import mxnet as mx
-    except (ImportError, OSError) as e:
-        assert 0, "Import mxnet error: %s. Please double check your build/" \
-            "install steps or environment variable settings" % str(e)
-
-    pid = os.getpid()
-    rc = os.system("cat /proc/" + str(pid) +
-                   "/maps | grep libmkldnn > /dev/null")
-
-    if rc == 0:
-        logging.info("MXNet is built/installed correctly with MKL-DNN")
-    else:
-        assert 0, "MXNet is built/installed incorrectly with MKL-DNN, please " \
-            "double check your build/install steps or environment " \
-            "variable settings"

From ed31e12cb6b8e9f13a7d957589705a1c4f4aea81 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Fri, 30 Nov 2018 11:07:23 -0800
Subject: [PATCH 35/96] fix spacing

---
 Makefile  | 8 ++++----
 mkldnn.mk | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/Makefile b/Makefile
index dd05e2157822..88a5459f0b2a 100644
--- a/Makefile
+++ b/Makefile
@@ -133,10 +133,10 @@ ifeq ($(USE_MKLDNN), 1)
 	endif
 
 	ifneq ($(UNAME_S), Windows)
-	    LIB_DEP += $(MKLDNNROOT)/lib/libmkldnn.a
-    else
-        CFLAGS += -I$(MKLDNNROOT)/include
-        LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,'$${ORIGIN}'
+		LIB_DEP += $(MKLDNNROOT)/lib/libmkldnn.a
+	else
+		CFLAGS += -I$(MKLDNNROOT)/include
+		LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,'$${ORIGIN}'
     endif
 endif
 
diff --git a/mkldnn.mk b/mkldnn.mk
index c9ca55c975ae..5af3e9b1d741 100644
--- a/mkldnn.mk
+++ b/mkldnn.mk
@@ -26,9 +26,9 @@ ifeq ($(UNAME_S), Darwin)
 	MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.a
 else ifeq ($(UNAME_S), Windows)
 	OMP_LIBFILE = $(MKLDNNROOT)/lib/libiomp5.so
-    MKLML_LIBFILE = $(MKLDNNROOT)/lib/libmklml_intel.so
-    MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.so
-    MKLDNN_LIBRARY_TYPE=SHARED
+	MKLML_LIBFILE = $(MKLDNNROOT)/lib/libmklml_intel.so
+	MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.so
+	MKLDNN_LIBRARY_TYPE=SHARED
 else
 	OMP_LIBFILE = $(MKLDNNROOT)/lib/libiomp5.so
 	MKLML_LIBFILE = $(MKLDNNROOT)/lib/libmklml_intel.so

From 6a029493c4df9d473b1f56a3b418760f709f0cb9 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Fri, 30 Nov 2018 13:23:04 -0800
Subject: [PATCH 36/96] fix index

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 88a5459f0b2a..2cf3526abc24 100644
--- a/Makefile
+++ b/Makefile
@@ -137,7 +137,7 @@ ifeq ($(USE_MKLDNN), 1)
 	else
 		CFLAGS += -I$(MKLDNNROOT)/include
 		LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,'$${ORIGIN}'
-    endif
+	endif
 endif
 
 # setup opencv

From 3cc21f37aaa96c11ea249d69182f95669536f8f9 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Fri, 30 Nov 2018 13:29:04 -0800
Subject: [PATCH 37/96] remove cp of mkldnn since statically linked

---
 ci/docker/runtime_functions.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 0036ce798e24..94143ed0521a 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -643,9 +643,6 @@ build_ubuntu_gpu_cmake() {
         /work/mxnet
 
     ninja -v
-    # libmkldnn.so.0 is a link file. We need an actual binary file named libmkldnn.so.0.
-    cp 3rdparty/mkldnn/src/libmkldnn.so.0 3rdparty/mkldnn/src/libmkldnn.so.0.tmp
-    mv 3rdparty/mkldnn/src/libmkldnn.so.0.tmp 3rdparty/mkldnn/src/libmkldnn.so.0
 }
 
 build_ubuntu_blc() {

From 0b894a0e8b896924cbc8bcaf203d944abbcaf345 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Fri, 30 Nov 2018 14:22:34 -0800
Subject: [PATCH 38/96] add libmkldnn.a to list of files to pack

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f329e48dc761..63bf6d25fc42 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -53,7 +53,7 @@ mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdpart
 mx_pip = 'build/*.whl'
 
 // for scala build, need to pass extra libs when run with dist_kvstore
-mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a'
+mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, lib/libmkldnn.a'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
 mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.

From 6b4db54d48aada5168b772d66a84fe13e4b2094b Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Fri, 30 Nov 2018 14:25:57 -0800
Subject: [PATCH 39/96] include mkl_ml

---
 Jenkinsfile                     | 2 +-
 ci/jenkins/Jenkins_steps.groovy | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 63bf6d25fc42..70a899b6112e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -47,7 +47,7 @@
 
 
 // mxnet libraries
-mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
 
 // Python wheels
 mx_pip = 'build/*.whl'
diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index e12b4ed04110..e9456f4c05bc 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -23,13 +23,13 @@
 utils = load('ci/Jenkinsfile_utils.groovy')
 
 // mxnet libraries
-mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
 
 // Python wheels
 mx_pip = 'build/*.whl'
 
 // for scala build, need to pass extra libs when run with dist_kvstore
-mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a'
+mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, lib/libmkldnn.a'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
 mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.

From ccf9855651605f7bc6a3eed705e690f4b0c32df3 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 10 Dec 2018 15:49:16 -0800
Subject: [PATCH 40/96] add mkldnn to pack

---
 ci/jenkins/Jenkins_steps.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index f48a26737308..8e50ca5b81c7 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -29,7 +29,7 @@ mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdpart
 mx_pip = 'build/*.whl'
 
 // for scala build, need to pass extra libs when run with dist_kvstore
-mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a'
+mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, lib/libmklml_intel.so, lib/libmkldnn.so.0'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
 mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.

From 225b446cdcf97642740fddcbc665c8037648546f Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 10 Dec 2018 16:01:28 -0800
Subject: [PATCH 41/96] add libiomp to ci pack

---
 ci/jenkins/Jenkins_steps.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index 8e50ca5b81c7..f2026cc0e74e 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -29,7 +29,7 @@ mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdpart
 mx_pip = 'build/*.whl'
 
 // for scala build, need to pass extra libs when run with dist_kvstore
-mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, lib/libmklml_intel.so, lib/libmkldnn.so.0'
+mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
 mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.

From 54231a9fb8c6909a0750947b3db0a4093001ca75 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 10 Dec 2018 16:54:03 -0800
Subject: [PATCH 42/96] move static libs

---
 ci/jenkins/Jenkins_steps.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index f2026cc0e74e..3ff0f0865481 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -23,7 +23,7 @@
 utils = load('ci/Jenkinsfile_utils.groovy')
 
 // mxnet libraries
-mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a. lib/libmklml_intel.so, lib/libmkldnn.so.0 lib/libiomp5.so'
 
 // Python wheels
 mx_pip = 'build/*.whl'

From 2d2a0f9686b99e56a6a89da2032fe404de728d2d Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 10 Dec 2018 17:31:48 -0800
Subject: [PATCH 43/96] fix typo

---
 ci/jenkins/Jenkins_steps.groovy | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index 3ff0f0865481..e39d1e366dfe 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -23,7 +23,7 @@
 utils = load('ci/Jenkinsfile_utils.groovy')
 
 // mxnet libraries
-mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a. lib/libmklml_intel.so, lib/libmkldnn.so.0 lib/libiomp5.so'
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, lib/libmklml_intel.so, lib/libmkldnn.so.0 lib/libiomp5.so'
 
 // Python wheels
 mx_pip = 'build/*.whl'
@@ -248,7 +248,7 @@ def compile_centos7_cpu_mkldnn() {
 }
 
 def compile_centos7_gpu() {
-    return ['GPU: CentOS 7': {
+    return ['vGPU: CentOS 7': {
       node(NODE_LINUX_CPU) {
         ws('workspace/build-centos7-gpu') {
           timeout(time: max_time, unit: 'MINUTES') {

From 884d955aa57c1c0d3857736c69f195eba0eb5eaa Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 10 Dec 2018 17:55:32 -0800
Subject: [PATCH 44/96] pack mkldnn

---
 ci/jenkins/Jenkins_steps.groovy | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index e39d1e366dfe..e46b17f02032 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -37,8 +37,8 @@ mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-c
 mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
 mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
 mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so'
-mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*'
-mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*'
+mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libmkldnn.so.0 lib/libiomp5.so'
+mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libmkldnn.so.0 lib/libiomp5.so'
 
 // Python unittest for CPU
 // Python 2

From 7dfa87e5a83f4506c2dbb2c33fca069f339c0aef Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 10 Dec 2018 18:23:17 -0800
Subject: [PATCH 45/96] retrigger


From 409acd0fe52ecbe888b20f8ca3a563c65e979ae0 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 10 Dec 2018 19:38:16 -0800
Subject: [PATCH 46/96] add linux artifacts

---
 ci/jenkins/Jenkins_steps.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index e46b17f02032..2925518c454f 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -31,7 +31,7 @@ mx_pip = 'build/*.whl'
 // for scala build, need to pass extra libs when run with dist_kvstore
 mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
-mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
+mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
 mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests'
 mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'

From 842228787ba30dc792c3c911fe15bb02fb1c01fc Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 10 Dec 2018 21:42:42 -0800
Subject: [PATCH 47/96] move libmkldnn in gpu cmake build

---
 ci/docker/runtime_functions.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 081a1475b5c4..e7f2f62d120c 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -646,6 +646,9 @@ build_ubuntu_gpu_cmake() {
         /work/mxnet
 
     ninja -v
+    # libmkldnn.so.0 is a link file. We need an actual binary file named libmkldnn.so.0.
+    cp 3rdparty/mkldnn/src/libmkldnn.so.0 3rdparty/mkldnn/src/libmkldnn.so.0.tmp
+    mv 3rdparty/mkldnn/src/libmkldnn.so.0.tmp 3rdparty/mkldnn/src/libmkldnn.so.0
 }
 
 build_ubuntu_blc() {

From 346a602d105599fc7467614a37ffe3cac4fbd944 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Mon, 10 Dec 2018 21:51:22 -0800
Subject: [PATCH 48/96] move libmkldnn and libiomp5 on gpu workspace

---
 ci/docker/runtime_functions.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index e7f2f62d120c..1ddec377242b 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -584,6 +584,10 @@ build_ubuntu_gpu_cuda91_cudnn7() {
         USE_DIST_KVSTORE=1                        \
         CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
         -j$(nproc)
+    cp lib/libmkldnn.so.0 lib/libmkldnn.so.0.tmp
+    mv lib/libmkldnn.so.0.tmp lib/libmkldnn.so.0
+    cp lib/libiomp5.so.0 lib/libiomp5.so.0.tmp
+    mv lib/libiomp5.so.0.tmp lib/libiomp5.so.0
 }
 
 build_ubuntu_amalgamation() {

From 20741c4b2bad3c7447dee6995ebcd8ff920bfdb2 Mon Sep 17 00:00:00 2001
From: Alexander Zai <azai91@gmail.com>
Date: Mon, 10 Dec 2018 22:08:38 -0800
Subject: [PATCH 49/96] move linked files

---
 ci/docker/runtime_functions.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 1ddec377242b..8c381169e8b0 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -306,6 +306,10 @@ build_centos7_gpu() {
         USE_DIST_KVSTORE=1                        \
         CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
         -j$(nproc)
+    cp lib/libmkldnn.so.0 lib/libmkldnn.so.0.tmp
+    mv lib/libmkldnn.so.0.tmp lib/libmkldnn.so.0
+    cp lib/libiomp5.so lib/libiomp5.so.tmp
+    mv lib/libiomp5.so.tmp lib/libiomp5.so
 }
 
 build_ubuntu_cpu() {
@@ -586,8 +590,8 @@ build_ubuntu_gpu_cuda91_cudnn7() {
         -j$(nproc)
     cp lib/libmkldnn.so.0 lib/libmkldnn.so.0.tmp
     mv lib/libmkldnn.so.0.tmp lib/libmkldnn.so.0
-    cp lib/libiomp5.so.0 lib/libiomp5.so.0.tmp
-    mv lib/libiomp5.so.0.tmp lib/libiomp5.so.0
+    cp lib/libiomp5.so lib/libiomp5.so.tmp
+    mv lib/libiomp5.so.tmp lib/libiomp5.so
 }
 
 build_ubuntu_amalgamation() {

From 4e6b2caa4765a2c2f58b6f156e95461f58b6c231 Mon Sep 17 00:00:00 2001
From: Alexander Zai <azai91@gmail.com>
Date: Mon, 10 Dec 2018 22:45:09 -0800
Subject: [PATCH 50/96] fix typo

---
 ci/jenkins/Jenkins_steps.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index 2925518c454f..06183a01c01b 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -23,7 +23,7 @@
 utils = load('ci/Jenkinsfile_utils.groovy')
 
 // mxnet libraries
-mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, lib/libmklml_intel.so, lib/libmkldnn.so.0 lib/libiomp5.so'
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
 
 // Python wheels
 mx_pip = 'build/*.whl'

From 992a2a0c2a9ad6d0c3e1e0ece22aba84809dd002 Mon Sep 17 00:00:00 2001
From: Alexander Zai <azai91@gmail.com>
Date: Mon, 10 Dec 2018 23:31:06 -0800
Subject: [PATCH 51/96] fix typo

---
 ci/jenkins/Jenkins_steps.groovy | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index 06183a01c01b..c3e5cb54e450 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -37,8 +37,8 @@ mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-c
 mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
 mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
 mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so'
-mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libmkldnn.so.0 lib/libiomp5.so'
-mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libmkldnn.so.0 lib/libiomp5.so'
+mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
+mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
 
 // Python unittest for CPU
 // Python 2

From 45fd008af65d2986d6ad8844125b2ab37187ff72 Mon Sep 17 00:00:00 2001
From: Alexander Zai <azai91@gmail.com>
Date: Tue, 11 Dec 2018 00:11:47 -0800
Subject: [PATCH 52/96] add artifacts for tensorrt

---
 ci/jenkins/Jenkins_steps.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index c3e5cb54e450..28f757dac81c 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -36,7 +36,7 @@ mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/li
 mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests'
 mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
 mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
-mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so'
+mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
 mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
 mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
 

From 7934cf16a190c58708e49b7161f47c7b755fce13 Mon Sep 17 00:00:00 2001
From: Alexander Zai <azai91@gmail.com>
Date: Tue, 11 Dec 2018 02:51:46 -0800
Subject: [PATCH 53/96] move mkldnn lib in scala build

---
 ci/docker/runtime_functions.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 8c381169e8b0..1e0dae52d540 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -920,6 +920,7 @@ integrationtest_ubuntu_cpu_dist_kvstore() {
 
 integrationtest_ubuntu_gpu_scala() {
     set -ex
+    cp lib/lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so.0
     make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1
     make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
 }

From b40e996800f06d71bef76de1da55047109cb7024 Mon Sep 17 00:00:00 2001
From: Alexander Zai <azai91@gmail.com>
Date: Tue, 11 Dec 2018 02:55:20 -0800
Subject: [PATCH 54/96] move mkldnn lib on cpu scala

---
 ci/docker/runtime_functions.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 1e0dae52d540..c766f800285d 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -768,6 +768,7 @@ unittest_ubuntu_cpu_scala() {
 
 unittest_centos7_cpu_scala() {
     set -ex
+    cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so.0
     cd /work/mxnet
     make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
     make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
@@ -920,7 +921,7 @@ integrationtest_ubuntu_cpu_dist_kvstore() {
 
 integrationtest_ubuntu_gpu_scala() {
     set -ex
-    cp lib/lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so.0
+    cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so.0
     make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1
     make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
 }

From 74e86e6a1971e6d6cdaa36cd0d2a091b6cbb6628 Mon Sep 17 00:00:00 2001
From: Alexander Zai <azai91@gmail.com>
Date: Tue, 11 Dec 2018 09:51:46 -0800
Subject: [PATCH 55/96] create dir for binding

---
 ci/docker/runtime_functions.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index c766f800285d..7b170e1ed87a 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -768,6 +768,7 @@ unittest_ubuntu_cpu_scala() {
 
 unittest_centos7_cpu_scala() {
     set -ex
+    mkdir -p /work/mxnet/3rdparty/mkldnn/build/install/lib/
     cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so.0
     cd /work/mxnet
     make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
@@ -921,6 +922,7 @@ integrationtest_ubuntu_cpu_dist_kvstore() {
 
 integrationtest_ubuntu_gpu_scala() {
     set -ex
+    mkdir /work/mxnet/3rdparty/mkldnn/build/install/lib/
     cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so.0
     make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1
     make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1

From 5a18a8fcda6fa3724cfdcbdb42db7a8e4957d147 Mon Sep 17 00:00:00 2001
From: Alexander Zai <azai91@gmail.com>
Date: Tue, 11 Dec 2018 10:07:24 -0800
Subject: [PATCH 56/96] rename libmkldnn in scala

---
 ci/docker/runtime_functions.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 7b170e1ed87a..44754a9a501b 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -769,7 +769,7 @@ unittest_ubuntu_cpu_scala() {
 unittest_centos7_cpu_scala() {
     set -ex
     mkdir -p /work/mxnet/3rdparty/mkldnn/build/install/lib/
-    cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so.0
+    cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so
     cd /work/mxnet
     make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
     make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
@@ -923,7 +923,7 @@ integrationtest_ubuntu_cpu_dist_kvstore() {
 integrationtest_ubuntu_gpu_scala() {
     set -ex
     mkdir /work/mxnet/3rdparty/mkldnn/build/install/lib/
-    cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so.0
+    cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so
     make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1
     make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
 }

From c0bd964b10d55e1a72b520fa484736dc224bd460 Mon Sep 17 00:00:00 2001
From: Alexander Zai <azai91@gmail.com>
Date: Tue, 11 Dec 2018 10:25:34 -0800
Subject: [PATCH 57/96] move mklml dep in scala builds

---
 ci/docker/runtime_functions.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 44754a9a501b..26ee9409a368 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -769,7 +769,9 @@ unittest_ubuntu_cpu_scala() {
 unittest_centos7_cpu_scala() {
     set -ex
     mkdir -p /work/mxnet/3rdparty/mkldnn/build/install/lib/
+    mkdir -p /work/mxnet/deps/include
     cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so
+    cp lib/libmklml_intel.so /work/mxnet/deps/include/libmklml_intel.so
     cd /work/mxnet
     make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
     make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
@@ -922,8 +924,10 @@ integrationtest_ubuntu_cpu_dist_kvstore() {
 
 integrationtest_ubuntu_gpu_scala() {
     set -ex
-    mkdir /work/mxnet/3rdparty/mkldnn/build/install/lib/
+    mkdir -p /work/mxnet/3rdparty/mkldnn/build/install/lib/
+    mkdir -p /work/mxnet/deps/include
     cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so
+    cp lib/libmklml_intel.so /work/mxnet/deps/include/libmklml_intel.so
     make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1
     make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
 }

From 2dc1683316d27f30d6eb0473a34eebe5cd92abf7 Mon Sep 17 00:00:00 2001
From: Alexander Zai <azai91@gmail.com>
Date: Tue, 11 Dec 2018 10:44:40 -0800
Subject: [PATCH 58/96] move mkl to another linked folder

---
 ci/docker/runtime_functions.sh | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 26ee9409a368..faa22e1c21ba 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -769,9 +769,8 @@ unittest_ubuntu_cpu_scala() {
 unittest_centos7_cpu_scala() {
     set -ex
     mkdir -p /work/mxnet/3rdparty/mkldnn/build/install/lib/
-    mkdir -p /work/mxnet/deps/include
     cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so
-    cp lib/libmklml_intel.so /work/mxnet/deps/include/libmklml_intel.so
+    cp lib/libmklml_intel.so /work/mxnet/3rdparty/mkldnn/build/install/lib/libmklml_intel.so
     cd /work/mxnet
     make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
     make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
@@ -925,9 +924,8 @@ integrationtest_ubuntu_cpu_dist_kvstore() {
 integrationtest_ubuntu_gpu_scala() {
     set -ex
     mkdir -p /work/mxnet/3rdparty/mkldnn/build/install/lib/
-    mkdir -p /work/mxnet/deps/include
     cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so
-    cp lib/libmklml_intel.so /work/mxnet/deps/include/libmklml_intel.so
+    cp lib/libmklml_intel.so /work/mxnet/3rdparty/mkldnn/build/install/lib/libmklml_intel.so
     make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1
     make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
 }

From 4eb65df07b0f3c2d0ba6e2d1210fc8da836ecfc8 Mon Sep 17 00:00:00 2001
From: Alexander Zai <azai91@gmail.com>
Date: Tue, 11 Dec 2018 23:00:34 -0800
Subject: [PATCH 59/96] move libmkl to another dir

---
 ci/docker/runtime_functions.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index faa22e1c21ba..2938b20a6c6b 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -770,7 +770,7 @@ unittest_centos7_cpu_scala() {
     set -ex
     mkdir -p /work/mxnet/3rdparty/mkldnn/build/install/lib/
     cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so
-    cp lib/libmklml_intel.so /work/mxnet/3rdparty/mkldnn/build/install/lib/libmklml_intel.so
+    cp lib/libmklml_intel.so /work/mxnet/lib/libmklml_intel.so
     cd /work/mxnet
     make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
     make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
@@ -925,7 +925,7 @@ integrationtest_ubuntu_gpu_scala() {
     set -ex
     mkdir -p /work/mxnet/3rdparty/mkldnn/build/install/lib/
     cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so
-    cp lib/libmklml_intel.so /work/mxnet/3rdparty/mkldnn/build/install/lib/libmklml_intel.so
+    cp lib/libmklml_intel.so /work/mxnet/lib/libmklml_intel.so
     make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1
     make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
 }

From ec5421dd97ad4e9582955a1fab3998cb1c2459dc Mon Sep 17 00:00:00 2001
From: Alexander Zai <azai91@gmail.com>
Date: Wed, 12 Dec 2018 00:30:25 -0800
Subject: [PATCH 60/96] add libmklml

---
 ci/docker/runtime_functions.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 2938b20a6c6b..faa22e1c21ba 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -770,7 +770,7 @@ unittest_centos7_cpu_scala() {
     set -ex
     mkdir -p /work/mxnet/3rdparty/mkldnn/build/install/lib/
     cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so
-    cp lib/libmklml_intel.so /work/mxnet/lib/libmklml_intel.so
+    cp lib/libmklml_intel.so /work/mxnet/3rdparty/mkldnn/build/install/lib/libmklml_intel.so
     cd /work/mxnet
     make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
     make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
@@ -925,7 +925,7 @@ integrationtest_ubuntu_gpu_scala() {
     set -ex
     mkdir -p /work/mxnet/3rdparty/mkldnn/build/install/lib/
     cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so
-    cp lib/libmklml_intel.so /work/mxnet/lib/libmklml_intel.so
+    cp lib/libmklml_intel.so /work/mxnet/3rdparty/mkldnn/build/install/lib/libmklml_intel.so
     make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1
     make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
 }

From 75dd532697f9a94bf34fc30e481602215b300330 Mon Sep 17 00:00:00 2001
From: Alexander Zai <azai91@gmail.com>
Date: Wed, 12 Dec 2018 00:57:04 -0800
Subject: [PATCH 61/96] move mkldnn

---
 ci/docker/runtime_functions.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index faa22e1c21ba..847015b2196e 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -308,6 +308,8 @@ build_centos7_gpu() {
         -j$(nproc)
     cp lib/libmkldnn.so.0 lib/libmkldnn.so.0.tmp
     mv lib/libmkldnn.so.0.tmp lib/libmkldnn.so.0
+    cp lib/libmklml_intel.so lib/libmklml_intel.so.tmp
+    mv lib/libmklml_intel.so.tmp lib/libmklml_intel.so
     cp lib/libiomp5.so lib/libiomp5.so.tmp
     mv lib/libiomp5.so.tmp lib/libiomp5.so
 }

From 2af09a896641c0ea75bc2021a060d87e7c782853 Mon Sep 17 00:00:00 2001
From: Alexander Zai <azai91@gmail.com>
Date: Wed, 12 Dec 2018 00:59:47 -0800
Subject: [PATCH 62/96] move mkldnn on centos

---
 ci/docker/runtime_functions.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 847015b2196e..4d21fb31ab54 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -253,6 +253,12 @@ build_centos7_cpu() {
         USE_BLAS=openblas \
         USE_DIST_KVSTORE=1 \
         -j$(nproc)
+    cp lib/libmkldnn.so.0 lib/libmkldnn.so.0.tmp
+    mv lib/libmkldnn.so.0.tmp lib/libmkldnn.so.0
+    cp lib/libmklml_intel.so lib/libmklml_intel.so.tmp
+    mv lib/libmklml_intel.so.tmp lib/libmklml_intel.so
+    cp lib/libiomp5.so lib/libiomp5.so.tmp
+    mv lib/libiomp5.so.tmp lib/libiomp5.so
 }
 
 build_amzn_linux_cpu() {

From 76d842f9817ed6d3b8e994549deca2f049f81c07 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 12 Dec 2018 11:12:08 -0800
Subject: [PATCH 63/96] specify new dynamic path

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 3c7ca0c3f25c..4bd127b6c479 100644
--- a/Makefile
+++ b/Makefile
@@ -136,7 +136,7 @@ ifeq ($(USE_MKLDNN), 1)
 		LDFLAGS += -L$(MKLROOT)/lib
 	endif
 	CFLAGS += -I$(MKLDNNROOT)/include
-	LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,'$${ORIGIN}'
+	LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,$(ROOTDIR)/lib
 endif
 
 # setup opencv

From 103c9d1cfab0c818d3badbd05cc9000e9d0c0211 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Wed, 12 Dec 2018 17:42:03 -0800
Subject: [PATCH 64/96] retrigger


From d51043622d4ef7fcb95aff6a3e84d91ab71b48c9 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 13 Dec 2018 15:02:05 -0800
Subject: [PATCH 65/96] remove mkldnn dynamic lib

---
 ci/jenkins/Jenkins_steps.groovy | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index a187e0c9f259..77735d68da60 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -23,22 +23,22 @@
 utils = load('ci/Jenkinsfile_utils.groovy')
 
 // mxnet libraries
-mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, lib/libmklml_intel.so lib/libiomp5.so'
 
 // Python wheels
 mx_pip = 'build/*.whl'
 
 // for scala build, need to pass extra libs when run with dist_kvstore
-mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
+mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, lib/libmklml_intel.so, lib/libiomp5.so'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
-mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
+mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
 mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests'
-mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
-mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
-mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
-mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
-mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
+mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
+mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
+mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so, lib/libmklml_intel.so, lib/libiomp5.so'
+mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libiomp5.so'
+mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libiomp5.so'
 
 // Python unittest for CPU
 // Python 2

From 16cca196e9e1ad92db74f4e8a01b3b052076d268 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 13 Dec 2018 15:12:15 -0800
Subject: [PATCH 66/96] remove moving mkldnn artifact

---
 ci/docker/runtime_functions.sh | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 07f2484144dc..4d46f5af8001 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -253,8 +253,6 @@ build_centos7_cpu() {
         USE_BLAS=openblas \
         USE_DIST_KVSTORE=1 \
         -j$(nproc)
-    cp lib/libmkldnn.so.0 lib/libmkldnn.so.0.tmp
-    mv lib/libmkldnn.so.0.tmp lib/libmkldnn.so.0
     cp lib/libmklml_intel.so lib/libmklml_intel.so.tmp
     mv lib/libmklml_intel.so.tmp lib/libmklml_intel.so
     cp lib/libiomp5.so lib/libiomp5.so.tmp
@@ -312,8 +310,6 @@ build_centos7_gpu() {
         USE_DIST_KVSTORE=1                        \
         CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
         -j$(nproc)
-    cp lib/libmkldnn.so.0 lib/libmkldnn.so.0.tmp
-    mv lib/libmkldnn.so.0.tmp lib/libmkldnn.so.0
     cp lib/libmklml_intel.so lib/libmklml_intel.so.tmp
     mv lib/libmklml_intel.so.tmp lib/libmklml_intel.so
     cp lib/libiomp5.so lib/libiomp5.so.tmp
@@ -624,8 +620,6 @@ build_ubuntu_gpu_cuda91_cudnn7() {
         USE_DIST_KVSTORE=1                        \
         CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
         -j$(nproc)
-    cp lib/libmkldnn.so.0 lib/libmkldnn.so.0.tmp
-    mv lib/libmkldnn.so.0.tmp lib/libmkldnn.so.0
     cp lib/libiomp5.so lib/libiomp5.so.tmp
     mv lib/libiomp5.so.tmp lib/libiomp5.so
 }

From ab753738fb7978053681de1c900b0d7fc75baeb4 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 13 Dec 2018 16:19:57 -0800
Subject: [PATCH 67/96] add ld path

---
 ci/docker/runtime_functions.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 4d46f5af8001..90285a0355a1 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -867,6 +867,7 @@ unittest_ubuntu_cpu_julia06() {
 
     # FIXME
     export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so'
+    export LD_LIBRARY_PATH=/work/mxnet/lib:$LD_LIBRARY_PATH
 
     # use the prebuilt binary from $MXNET_HOME/lib
     julia -e 'Pkg.build("MXNet")'
@@ -1196,6 +1197,7 @@ deploy_jl_docs() {
 
     # FIXME
     export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so'
+    export LD_LIBRARY_PATH=/work/mxnet/lib:$LD_LIBRARY_PATH
 
     # use the prebuilt binary from $MXNET_HOME/lib
     julia -e 'Pkg.build("MXNet")'

From 63821d4c15ec50ed649205c3b0d6d1e58da3d2fb Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 13 Dec 2018 22:34:44 -0800
Subject: [PATCH 68/96] retrigger


From 13bf7ae0bac9e364a6498c47ce734eea9a87e583 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 13 Dec 2018 22:43:46 -0800
Subject: [PATCH 69/96] Revert "remove moving mkldnn artifact"

This reverts commit 16cca196e9e1ad92db74f4e8a01b3b052076d268.
---
 ci/docker/runtime_functions.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 90285a0355a1..4e6622c6e3a4 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -253,6 +253,8 @@ build_centos7_cpu() {
         USE_BLAS=openblas \
         USE_DIST_KVSTORE=1 \
         -j$(nproc)
+    cp lib/libmkldnn.so.0 lib/libmkldnn.so.0.tmp
+    mv lib/libmkldnn.so.0.tmp lib/libmkldnn.so.0
     cp lib/libmklml_intel.so lib/libmklml_intel.so.tmp
     mv lib/libmklml_intel.so.tmp lib/libmklml_intel.so
     cp lib/libiomp5.so lib/libiomp5.so.tmp
@@ -310,6 +312,8 @@ build_centos7_gpu() {
         USE_DIST_KVSTORE=1                        \
         CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
         -j$(nproc)
+    cp lib/libmkldnn.so.0 lib/libmkldnn.so.0.tmp
+    mv lib/libmkldnn.so.0.tmp lib/libmkldnn.so.0
     cp lib/libmklml_intel.so lib/libmklml_intel.so.tmp
     mv lib/libmklml_intel.so.tmp lib/libmklml_intel.so
     cp lib/libiomp5.so lib/libiomp5.so.tmp
@@ -620,6 +624,8 @@ build_ubuntu_gpu_cuda91_cudnn7() {
         USE_DIST_KVSTORE=1                        \
         CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
         -j$(nproc)
+    cp lib/libmkldnn.so.0 lib/libmkldnn.so.0.tmp
+    mv lib/libmkldnn.so.0.tmp lib/libmkldnn.so.0
     cp lib/libiomp5.so lib/libiomp5.so.tmp
     mv lib/libiomp5.so.tmp lib/libiomp5.so
 }

From 935b1dc89b1e56383aa7bc5ad35916dba13fb6da Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 13 Dec 2018 22:43:53 -0800
Subject: [PATCH 70/96] Revert "remove mkldnn dynamic lib"

This reverts commit d51043622d4ef7fcb95aff6a3e84d91ab71b48c9.
---
 ci/jenkins/Jenkins_steps.groovy | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index 77735d68da60..a187e0c9f259 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -23,22 +23,22 @@
 utils = load('ci/Jenkinsfile_utils.groovy')
 
 // mxnet libraries
-mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, lib/libmklml_intel.so lib/libiomp5.so'
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
 
 // Python wheels
 mx_pip = 'build/*.whl'
 
 // for scala build, need to pass extra libs when run with dist_kvstore
-mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, lib/libmklml_intel.so, lib/libiomp5.so'
+mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
-mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
+mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
 mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests'
-mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
-mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
-mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so, lib/libmklml_intel.so, lib/libiomp5.so'
-mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libiomp5.so'
-mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libiomp5.so'
+mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
+mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
+mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
+mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
+mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
 
 // Python unittest for CPU
 // Python 2

From 59bae57a038f147d95d740475423d0b69a2b3759 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Thu, 13 Dec 2018 23:02:15 -0800
Subject: [PATCH 71/96] update makefile

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 1faa9e039467..dc17fcae7bdf 100644
--- a/Makefile
+++ b/Makefile
@@ -136,7 +136,7 @@ ifeq ($(USE_MKLDNN), 1)
 		LDFLAGS += -L$(MKLROOT)/lib
 	endif
 	CFLAGS += -I$(MKLDNNROOT)/include
-	LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,'$${ORIGIN}'
+	LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,$(ROOTDIR)/lib
 endif
 
 # setup opencv

From 6d583dfc48ab304b83357e6ce00c2506748b099d Mon Sep 17 00:00:00 2001
From: Anirudh Subramanian <anirudh2290@ufl.edu>
Date: Tue, 18 Dec 2018 19:50:59 +0000
Subject: [PATCH 72/96] Revert RPATH change and trigger CI

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index dc17fcae7bdf..1faa9e039467 100644
--- a/Makefile
+++ b/Makefile
@@ -136,7 +136,7 @@ ifeq ($(USE_MKLDNN), 1)
 		LDFLAGS += -L$(MKLROOT)/lib
 	endif
 	CFLAGS += -I$(MKLDNNROOT)/include
-	LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,$(ROOTDIR)/lib
+	LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,'$${ORIGIN}'
 endif
 
 # setup opencv

From 83c36d5b5e6be1ee21c79383e9c10b88bc970074 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Wed, 19 Dec 2018 00:56:18 +0000
Subject: [PATCH 73/96] correcting use-mkldnn flags for two tests

---
 ci/docker/runtime_functions.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 377d18016ab6..de3ee5ff83b1 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -707,7 +707,6 @@ build_ubuntu_gpu_cmake_mkldnn() {
         -DUSE_CUDA=1                            \
         -DUSE_CUDNN=1                           \
         -DUSE_MKLML_MKL=1                       \
-        -DUSE_MKLDNN=1                          \
         -DCMAKE_BUILD_TYPE=Release              \
         -DCUDA_ARCH_NAME=Manual                 \
         -DCUDA_ARCH_BIN=$CI_CMAKE_CUDA_ARCH_BIN \
@@ -732,6 +731,7 @@ build_ubuntu_gpu_cmake() {
         -DUSE_CUDA=1                            \
         -DUSE_CUDNN=1                           \
         -DUSE_MKLML_MKL=0                       \
+        -DUSE_MKLDNN=0                          \
         -DUSE_DIST_KVSTORE=1                    \
         -DCMAKE_BUILD_TYPE=Release              \
         -DCUDA_ARCH_NAME=Manual                 \

From c94129a4265991b8e2fbddbc255aeaa5540797bf Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Wed, 19 Dec 2018 02:40:34 +0000
Subject: [PATCH 74/96] mkldnn default on linux for starters

---
 CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f09a9c29e9d8..4c3c165dc7ae 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -20,7 +20,7 @@ mxnet_option(USE_F16C             "Build with x86 F16C instruction support" ON)
 mxnet_option(USE_LAPACK           "Build with lapack support" ON)
 mxnet_option(USE_MKL_IF_AVAILABLE "Use MKL if found" ON)
 mxnet_option(USE_MKLML_MKL        "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND (NOT APPLE))
-mxnet_option(USE_MKLDNN           "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE)
+mxnet_option(USE_MKLDNN           "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND (NOT APPLE))
 mxnet_option(USE_OPERATOR_TUNING  "Enable auto-tuning of operators" ON IF NOT MSVC)
 mxnet_option(USE_GPERFTOOLS       "Build with GPerfTools support (if found)" ON)
 mxnet_option(USE_JEMALLOC         "Build with Jemalloc support"   ON)
@@ -215,7 +215,7 @@ if(ENABLE_TESTCOVERAGE)
   if(NOT GCOV_PATH)
     message(FATAL_ERROR "gcov not found! Aborting...")
   endif() # NOT GCOV_PATH
-  
+
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --coverage")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage")
   set(CMAKE_LINKER_FLAGS "${CMAKE_LINKER_FLAGS} --coverage")

From 02205727b94a2f3362a609253a60aa0e8ea978ac Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Wed, 19 Dec 2018 03:09:51 +0000
Subject: [PATCH 75/96] reverting naming rules of pack_lib

---
 ci/jenkins/Jenkins_steps.groovy | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index 94cc81a444a6..74bde1eee211 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -23,22 +23,22 @@
 utils = load('ci/Jenkinsfile_utils.groovy')
 
 // mxnet libraries
-mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
 
 // Python wheels
 mx_pip = 'build/*.whl'
 
 // for scala build, need to pass extra libs when run with dist_kvstore
-mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
+mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
-mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
+mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
 mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests'
 mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
 mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
-mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
-mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
-mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
+mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so'
+mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*'
+mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*'
 
 // Python unittest for CPU
 // Python 2

From 2384a906349e11e669f58e4fbd6dd0eb82c1193c Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Wed, 19 Dec 2018 19:28:23 +0000
Subject: [PATCH 76/96] adding mkldnn=0 flags to centos non mkldnn builds

---
 ci/docker/runtime_functions.sh | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index de3ee5ff83b1..6c9d1e5a0516 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -285,14 +285,9 @@ build_centos7_cpu() {
         ENABLE_TESTCOVERAGE=1 \
         USE_LAPACK_PATH=/usr/lib64/liblapack.so \
         USE_BLAS=openblas \
+        USE_MKLDNN=0 \
         USE_DIST_KVSTORE=1 \
         -j$(nproc)
-    cp lib/libmkldnn.so.0 lib/libmkldnn.so.0.tmp
-    mv lib/libmkldnn.so.0.tmp lib/libmkldnn.so.0
-    cp lib/libmklml_intel.so lib/libmklml_intel.so.tmp
-    mv lib/libmklml_intel.so.tmp lib/libmklml_intel.so
-    cp lib/libiomp5.so lib/libiomp5.so.tmp
-    mv lib/libiomp5.so.tmp lib/libiomp5.so
 }
 
 build_amzn_linux_cpu() {
@@ -341,18 +336,13 @@ build_centos7_gpu() {
         USE_LAPACK=1                              \
         USE_LAPACK_PATH=/usr/lib64/liblapack.so   \
         USE_BLAS=openblas                         \
+        USE_MKLDNN=0                              \
         USE_CUDA=1                                \
         USE_CUDA_PATH=/usr/local/cuda             \
         USE_CUDNN=1                               \
         USE_DIST_KVSTORE=1                        \
         CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
         -j$(nproc)
-    cp lib/libmkldnn.so.0 lib/libmkldnn.so.0.tmp
-    mv lib/libmkldnn.so.0.tmp lib/libmkldnn.so.0
-    cp lib/libmklml_intel.so lib/libmklml_intel.so.tmp
-    mv lib/libmklml_intel.so.tmp lib/libmklml_intel.so
-    cp lib/libiomp5.so lib/libiomp5.so.tmp
-    mv lib/libiomp5.so.tmp lib/libiomp5.so
 }
 
 build_ubuntu_cpu() {

From 377db812b9c15d78fd29a5f6d34b713667bd85a5 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Wed, 19 Dec 2018 19:42:31 +0000
Subject: [PATCH 77/96] adding mkldnn=0 flags to ubuntu gpu non mkldnn builds

---
 ci/docker/runtime_functions.sh | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 6c9d1e5a0516..3fc121ae4412 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -651,6 +651,7 @@ build_ubuntu_gpu_cuda91_cudnn7() {
         DEV=1                                     \
         ENABLE_TESTCOVERAGE=1                     \
         USE_BLAS=openblas                         \
+        USE_MKLDNN=0                              \
         USE_CUDA=1                                \
         USE_CUDA_PATH=/usr/local/cuda             \
         USE_CUDNN=1                               \
@@ -658,10 +659,6 @@ build_ubuntu_gpu_cuda91_cudnn7() {
         USE_DIST_KVSTORE=1                        \
         CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
         -j$(nproc)
-    cp lib/libmkldnn.so.0 lib/libmkldnn.so.0.tmp
-    mv lib/libmkldnn.so.0.tmp lib/libmkldnn.so.0
-    cp lib/libiomp5.so lib/libiomp5.so.tmp
-    mv lib/libiomp5.so.tmp lib/libiomp5.so
 }
 
 build_ubuntu_amalgamation() {

From 365838dbfc7476e89eaa574314bf760ded1ed6a4 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Wed, 19 Dec 2018 20:12:47 +0000
Subject: [PATCH 78/96] removing mkldnn binary operation for ubuntu gpu cmake
 non mkldnn build

---
 ci/docker/runtime_functions.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 3fc121ae4412..551c34413f56 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -727,9 +727,6 @@ build_ubuntu_gpu_cmake() {
         /work/mxnet
 
     ninja -v
-    # libmkldnn.so.0 is a link file. We need an actual binary file named libmkldnn.so.0.
-    cp 3rdparty/mkldnn/src/libmkldnn.so.0 3rdparty/mkldnn/src/libmkldnn.so.0.tmp
-    mv 3rdparty/mkldnn/src/libmkldnn.so.0.tmp 3rdparty/mkldnn/src/libmkldnn.so.0
 }
 
 build_ubuntu_blc() {

From dfaef511b33712532949f390f07495165f65b69b Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Wed, 19 Dec 2018 20:45:36 +0000
Subject: [PATCH 79/96] removing mkldnn binary operation for centos non-mkldnn
 unittest

---
 ci/docker/runtime_functions.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 551c34413f56..413347321306 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -838,9 +838,6 @@ unittest_ubuntu_cpu_scala() {
 
 unittest_centos7_cpu_scala() {
     set -ex
-    mkdir -p /work/mxnet/3rdparty/mkldnn/build/install/lib/
-    cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so
-    cp lib/libmklml_intel.so /work/mxnet/3rdparty/mkldnn/build/install/lib/libmklml_intel.so
     cd /work/mxnet
     make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
     make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1

From 9f7e869369a2050bf337f3b1810b516cd59055f3 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Wed, 19 Dec 2018 23:20:18 +0000
Subject: [PATCH 80/96] adding explicit USE_MKLDNN=0 flags for clang builds

---
 ci/docker/runtime_functions.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 413347321306..33e6b3d0e27b 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -437,6 +437,7 @@ build_ubuntu_cpu_clang39() {
         ENABLE_TESTCOVERAGE=1         \
         USE_CPP_PACKAGE=1             \
         USE_BLAS=openblas             \
+        USE_MKLDNN=0                  \
         USE_OPENMP=0                  \
         USE_DIST_KVSTORE=1            \
         -j$(nproc)
@@ -454,6 +455,7 @@ build_ubuntu_cpu_clang60() {
         ENABLE_TESTCOVERAGE=1         \
         USE_CPP_PACKAGE=1             \
         USE_BLAS=openblas             \
+        USE_MKLDNN=0                  \
         USE_OPENMP=1                  \
         USE_DIST_KVSTORE=1            \
         -j$(nproc)
@@ -473,6 +475,7 @@ build_ubuntu_cpu_clang_tidy() {
         -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
         -DCMAKE_C_COMPILER_LAUNCHER=ccache \
         -DUSE_CUDA=OFF \
+        -DUSE_MKLDNN=OFF \
         -DUSE_MKL_IF_AVAILABLE=OFF \
         -DUSE_OPENCV=ON \
         -DCMAKE_BUILD_TYPE=Debug \

From c1a5d2c18c8304eca5947a21f1f0ed5adfadd878 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Wed, 19 Dec 2018 23:32:01 +0000
Subject: [PATCH 81/96] adding explicit USE_MKLDNN=0 flags for cpu ubuntu
 builds

---
 ci/docker/runtime_functions.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 33e6b3d0e27b..5d52a84eaa56 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -359,6 +359,7 @@ build_ubuntu_cpu_openblas() {
         ENABLE_TESTCOVERAGE=1         \
         USE_CPP_PACKAGE=1             \
         USE_BLAS=openblas             \
+        USE_MKLDNN=0                  \
         USE_DIST_KVSTORE=1            \
         -j$(nproc)
 }
@@ -372,6 +373,7 @@ build_ubuntu_cpu_mkl() {
         ENABLE_TESTCOVERAGE=1         \
         USE_CPP_PACKAGE=1             \
         USE_BLAS=mkl                  \
+        USE_MKLDNN=0                  \
         USE_INTEL_PATH=/opt/intel     \
         USE_DIST_KVSTORE=1            \
         -j$(nproc)
@@ -544,7 +546,6 @@ build_ubuntu_cpu_mkldnn_mkl() {
         ENABLE_TESTCOVERAGE=1         \
         USE_CPP_PACKAGE=1             \
         USE_BLAS=mkl                  \
-        USE_MKLDNN=1                  \
         -j$(nproc)
 }
 

From 3ec81a11ad8a4c462dd494fd4dbc5f6ec98d37d6 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Wed, 19 Dec 2018 23:38:44 +0000
Subject: [PATCH 82/96] removing mkldnn binaries from non mkldnn builds scala
 gpu

---
 ci/docker/runtime_functions.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 5d52a84eaa56..a561fcd6794c 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -995,9 +995,6 @@ integrationtest_ubuntu_cpu_dist_kvstore() {
 
 integrationtest_ubuntu_gpu_scala() {
     set -ex
-    mkdir -p /work/mxnet/3rdparty/mkldnn/build/install/lib/
-    cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so
-    cp lib/libmklml_intel.so /work/mxnet/3rdparty/mkldnn/build/install/lib/libmklml_intel.so
     make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1
     make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
 }

From 2a3d362e452ac0b535bcc1892af7b25f6bef39c9 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Wed, 19 Dec 2018 23:42:49 +0000
Subject: [PATCH 83/96] adding explicit flag mkldnn=0 for tensorrt gpu build

---
 ci/docker/runtime_functions.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index a561fcd6794c..825188b7d414 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -605,6 +605,7 @@ build_ubuntu_gpu_tensorrt() {
         USE_CUDA_PATH=/usr/local/cuda                        \
         USE_CUDNN=1                                          \
         USE_OPENCV=0                                         \
+        USE_MKLDNN=0                                         \
         USE_DIST_KVSTORE=0                                   \
         USE_TENSORRT=1                                       \
         USE_JEMALLOC=0                                       \

From bd9e55b539e372f9287efdfbcfb1c2e33c09fa49 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Wed, 19 Dec 2018 23:48:23 +0000
Subject: [PATCH 84/96] adding explicit flag mkldnn=0 for ubuntu cmake asan

---
 ci/docker/runtime_functions.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 825188b7d414..be10cd9a7d2b 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -413,6 +413,7 @@ build_ubuntu_cpu_cmake_asan() {
         -DCMAKE_C_COMPILER_LAUNCHER=ccache \
         -DUSE_CUDA=OFF \
         -DUSE_MKL_IF_AVAILABLE=OFF \
+        -DUSE_MKLDNN=OFF \
         -DUSE_OPENMP=OFF \
         -DUSE_OPENCV=OFF \
         -DCMAKE_BUILD_TYPE=Debug \

From bd55a1ca2f59bd9a42a215aae60f7affa134ffe9 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Thu, 20 Dec 2018 01:45:41 +0000
Subject: [PATCH 85/96] adding centos cpu mkldnn tests to CI

---
 ci/jenkins/Jenkins_steps.groovy   | 36 ++++++++++++++++++++++++++++++-
 ci/jenkins/Jenkinsfile_centos_cpu |  4 +++-
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index 74bde1eee211..841ccbb9d94b 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -36,6 +36,7 @@ mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/li
 mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests'
 mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
 mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
+mx_dist_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a'
 mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so'
 mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*'
 mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*'
@@ -268,7 +269,7 @@ def compile_centos7_cpu_mkldnn() {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.init_git()
             utils.docker_run('centos7_cpu', 'build_centos7_mkldnn', false)
-            utils.pack_lib('centos7_mkldnn', mx_lib, true)
+            utils.pack_lib('centos7_mkldnn', mx_dist_mkldnn_lib, true)
           }
         }
       }
@@ -992,6 +993,25 @@ def test_centos7_python3_cpu() {
     }]
 }
 
+def test_centos7_python3_cpu_mkldnn() {
+    return ['Python3: CentOS 7 CPU MKLDNN': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/build-centos7-cpu-mkldnn') {
+          timeout(time: max_time, unit: 'MINUTES') {
+            try {
+              utils.unpack_and_init('centos7_mkldnn', mx_dist_mkldnn_lib, true)
+              utils.docker_run('centos7_cpu', 'unittest_centos7_cpu', false)
+              utils.publish_test_coverage()
+            } finally {
+              utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_centos7_cpu_mkldnn_unittest.xml')
+              utils.collect_test_results_unix('nosetests_train.xml', 'nosetests_python3_centos7_cpu_mkldnn_train.xml')
+            }
+          }
+        }
+      }
+    }]
+}
+
 def test_centos7_python3_gpu() {
     return ['Python3: CentOS 7 GPU': {
       node(NODE_LINUX_GPU) {
@@ -1024,6 +1044,20 @@ def test_centos7_scala_cpu() {
     }]
 }
 
+def test_centos7_scala_cpu_mkldnn() {
+    return ['Scala: CentOS CPU MKLDNN': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/ut-scala-centos7-cpu-mkldnn') {
+          timeout(time: max_time, unit: 'MINUTES') {
+            utils.unpack_and_init('centos7_mkldnn', mx_dist_mkldnn_lib, true)
+            utils.docker_run('centos7_cpu', 'unittest_centos7_cpu_scala', false)
+            utils.publish_test_coverage()
+          }
+        }
+      }
+    }]
+}
+
 def test_windows_python2_cpu() {
     return ['Python 2: CPU Win':{
       node(NODE_WINDOWS_CPU) {
diff --git a/ci/jenkins/Jenkinsfile_centos_cpu b/ci/jenkins/Jenkinsfile_centos_cpu
index 3b66f8100173..ef0d0a767ce1 100644
--- a/ci/jenkins/Jenkinsfile_centos_cpu
+++ b/ci/jenkins/Jenkinsfile_centos_cpu
@@ -40,7 +40,9 @@ core_logic: {
 
   utils.parallel_stage('Tests', [
     custom_steps.test_centos7_python3_cpu(),
-    custom_steps.test_centos7_scala_cpu()
+    custom_steps.test_centos7_scala_cpu(),
+    custom_steps.test_centos7_python3_cpu_mkldnn(),
+    custom_steps.test_centos7_scala_cpu_mkldnn()
   ]) 
 }
 ,

From c68ccd0228d61995bf6905caf65e55047e03eaa9 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Thu, 20 Dec 2018 20:43:49 +0000
Subject: [PATCH 86/96] adding CentOS GPU MKLDNN build and unittest

---
 ci/docker/runtime_functions.sh    | 19 ++++++++++++++++++
 ci/jenkins/Jenkins_steps.groovy   | 32 +++++++++++++++++++++++++++++++
 ci/jenkins/Jenkinsfile_centos_gpu |  6 ++++--
 3 files changed, 55 insertions(+), 2 deletions(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index be10cd9a7d2b..facb96033520 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -345,6 +345,25 @@ build_centos7_gpu() {
         -j$(nproc)
 }
 
+build_centos7_gpu_mkldnn() {
+    set -ex
+    cd /work/mxnet
+    # unfortunately this build has problems in 3rdparty dependencies with ccache and make
+    build_ccache_wrappers
+    make \
+        DEV=1                                     \
+        ENABLE_TESTCOVERAGE=1                     \
+        USE_LAPACK=1                              \
+        USE_LAPACK_PATH=/usr/lib64/liblapack.so   \
+        USE_BLAS=openblas                         \
+        USE_CUDA=1                                \
+        USE_CUDA_PATH=/usr/local/cuda             \
+        USE_CUDNN=1                               \
+        USE_DIST_KVSTORE=1                        \
+        CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
+        -j$(nproc)
+}
+
 build_ubuntu_cpu() {
     build_ubuntu_cpu_openblas
 }
diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index 841ccbb9d94b..da5f8bf45fa4 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -290,6 +290,20 @@ def compile_centos7_gpu() {
     }]
 }
 
+def compile_centos7_gpu_mkldnn() {
+    return ['GPU: CentOS 7 MKLDNN': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/build-centos7-gpu-mkldnn') {
+          timeout(time: max_time, unit: 'MINUTES') {
+            utils.init_git()
+            utils.docker_run('centos7_gpu', 'build_centos7_gpu_mkldnn', false)
+            utils.pack_lib('centos7_gpu_mkldnn', mx_mkldnn_lib, true)
+          }
+        }
+      }
+    }]
+}
+
 def compile_unix_clang_3_9_cpu() {
     return ['CPU: Clang 3.9': {
       node(NODE_LINUX_CPU) {
@@ -1030,6 +1044,24 @@ def test_centos7_python3_gpu() {
     }]
 }
 
+def test_centos7_python3_gpu_mkldnn() {
+    return ['Python3: CentOS 7 GPU MKLDNN': {
+      node(NODE_LINUX_GPU) {
+        ws('workspace/build-centos7-gpu-mkldnn') {
+          timeout(time: max_time, unit: 'MINUTES') {
+            try {
+              utils.unpack_and_init('centos7_gpu_mkldnn', mx_mkldnn_lib, true)
+              utils.docker_run('centos7_gpu', 'unittest_centos7_gpu', true)
+              utils.publish_test_coverage()
+            } finally {
+              utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_centos7_gpu_mkldnn.xml')
+            }
+          }
+        }
+      }
+    }]
+}
+
 def test_centos7_scala_cpu() {
     return ['Scala: CentOS CPU': {
       node(NODE_LINUX_CPU) {
diff --git a/ci/jenkins/Jenkinsfile_centos_gpu b/ci/jenkins/Jenkinsfile_centos_gpu
index aec3b9054f2f..f06fbb14d7d5 100644
--- a/ci/jenkins/Jenkinsfile_centos_gpu
+++ b/ci/jenkins/Jenkinsfile_centos_gpu
@@ -34,11 +34,13 @@ utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_
 utils.main_wrapper(
 core_logic: {
   utils.parallel_stage('Build', [
-    custom_steps.compile_centos7_gpu()
+    custom_steps.compile_centos7_gpu(),
+    custom_steps.compile_centos7_gpu_mkldnn()
   ]) 
   
   utils.parallel_stage('Tests', [
-    custom_steps.test_centos7_python3_gpu()
+    custom_steps.test_centos7_python3_gpu(),
+    custom_steps.test_centos7_python3_gpu_mkldnn()
   ]) 
 }
 ,

From 8c82763bb5c97c4823e3bb77bd04bc1623a8e284 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Thu, 20 Dec 2018 22:28:02 +0000
Subject: [PATCH 87/96] not keeping mkldnn default for mac os

---
 Makefile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 08539df7ea12..e86d53f9c3f5 100644
--- a/Makefile
+++ b/Makefile
@@ -61,7 +61,9 @@ endif
 include $(config)
 
 ifndef $(USE_MKLDNN)
-    USE_MKLDNN = 1
+ifneq ($(UNAME_S), Darwin)
+	USE_MKLDNN = 1
+endif
 endif
 
 ifeq ($(USE_MKL2017), 1)

From 670c3ae0a32d72ba385d1fc62fe6bb433cc2ad9b Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Fri, 21 Dec 2018 21:00:08 +0000
Subject: [PATCH 88/96] setting mkldnn default for x86_64 only

---
 Makefile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index e86d53f9c3f5..23b8a65f1bfa 100644
--- a/Makefile
+++ b/Makefile
@@ -24,6 +24,7 @@ ifeq ($(OS),Windows_NT)
 	UNAME_S := Windows
 else
 	UNAME_S := $(shell uname -s)
+	UNAME_P := $(shell uname -p)
 endif
 
 ifndef config
@@ -62,7 +63,9 @@ include $(config)
 
 ifndef $(USE_MKLDNN)
 ifneq ($(UNAME_S), Darwin)
-	USE_MKLDNN = 1
+ifeq ($(UNAME_P), x86_64)
+	USE_MKLDNN=1
+endif
 endif
 endif
 

From 130ec16633e8cdf49d5d4f7f7b285fcebbef3def Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Fri, 21 Dec 2018 21:44:09 +0000
Subject: [PATCH 89/96] running docs with mkldnn=0 flag

---
 docs/mxdoc.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/mxdoc.py b/docs/mxdoc.py
index 7be4b90e799b..d4f379b3e849 100644
--- a/docs/mxdoc.py
+++ b/docs/mxdoc.py
@@ -87,10 +87,10 @@ def generate_doxygen(app):
 def build_mxnet(app):
     """Build mxnet .so lib"""
     if not os.path.exists(os.path.join(app.builder.srcdir, '..', 'config.mk')):
-        _run_cmd("cd %s/.. && cp make/config.mk config.mk && make -j$(nproc) DEBUG=1" %
+        _run_cmd("cd %s/.. && cp make/config.mk config.mk && make -j$(nproc) DEBUG=1 USE_MKLDNN=0" %
                 app.builder.srcdir)
     else:
-        _run_cmd("cd %s/.. && make -j$(nproc) DEBUG=1" %
+        _run_cmd("cd %s/.. && make -j$(nproc) DEBUG=1 USE_MKLDNN=0" %
                 app.builder.srcdir)
 
 def build_r_docs(app):

From b7330ccdb6c45afb50d144d7c121dd569280f76c Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Fri, 21 Dec 2018 23:32:26 +0000
Subject: [PATCH 90/96] removing CentOS CPU Scala MKLDNN test

---
 ci/jenkins/Jenkins_steps.groovy   | 19 ++-----------------
 ci/jenkins/Jenkinsfile_centos_cpu |  3 +--
 2 files changed, 3 insertions(+), 19 deletions(-)

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index da5f8bf45fa4..b79ae7e5517c 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -36,7 +36,6 @@ mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/li
 mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests'
 mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
 mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
-mx_dist_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a'
 mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so'
 mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*'
 mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*'
@@ -269,7 +268,7 @@ def compile_centos7_cpu_mkldnn() {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.init_git()
             utils.docker_run('centos7_cpu', 'build_centos7_mkldnn', false)
-            utils.pack_lib('centos7_mkldnn', mx_dist_mkldnn_lib, true)
+            utils.pack_lib('centos7_mkldnn', mx_mkldnn_lib, true)
           }
         }
       }
@@ -1013,7 +1012,7 @@ def test_centos7_python3_cpu_mkldnn() {
         ws('workspace/build-centos7-cpu-mkldnn') {
           timeout(time: max_time, unit: 'MINUTES') {
             try {
-              utils.unpack_and_init('centos7_mkldnn', mx_dist_mkldnn_lib, true)
+              utils.unpack_and_init('centos7_mkldnn', mx_mkldnn_lib, true)
               utils.docker_run('centos7_cpu', 'unittest_centos7_cpu', false)
               utils.publish_test_coverage()
             } finally {
@@ -1076,20 +1075,6 @@ def test_centos7_scala_cpu() {
     }]
 }
 
-def test_centos7_scala_cpu_mkldnn() {
-    return ['Scala: CentOS CPU MKLDNN': {
-      node(NODE_LINUX_CPU) {
-        ws('workspace/ut-scala-centos7-cpu-mkldnn') {
-          timeout(time: max_time, unit: 'MINUTES') {
-            utils.unpack_and_init('centos7_mkldnn', mx_dist_mkldnn_lib, true)
-            utils.docker_run('centos7_cpu', 'unittest_centos7_cpu_scala', false)
-            utils.publish_test_coverage()
-          }
-        }
-      }
-    }]
-}
-
 def test_windows_python2_cpu() {
     return ['Python 2: CPU Win':{
       node(NODE_WINDOWS_CPU) {
diff --git a/ci/jenkins/Jenkinsfile_centos_cpu b/ci/jenkins/Jenkinsfile_centos_cpu
index ef0d0a767ce1..7f3f5f1e3b8c 100644
--- a/ci/jenkins/Jenkinsfile_centos_cpu
+++ b/ci/jenkins/Jenkinsfile_centos_cpu
@@ -41,8 +41,7 @@ core_logic: {
   utils.parallel_stage('Tests', [
     custom_steps.test_centos7_python3_cpu(),
     custom_steps.test_centos7_scala_cpu(),
-    custom_steps.test_centos7_python3_cpu_mkldnn(),
-    custom_steps.test_centos7_scala_cpu_mkldnn()
+    custom_steps.test_centos7_python3_cpu_mkldnn()
   ]) 
 }
 ,

From fe4b20ece1325e1b25ca582b7de2baffd940e842 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Sat, 22 Dec 2018 01:47:52 +0000
Subject: [PATCH 91/96] setting mkldnn default for x86_64 only

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4c3c165dc7ae..e11d5f81bffc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -20,7 +20,7 @@ mxnet_option(USE_F16C             "Build with x86 F16C instruction support" ON)
 mxnet_option(USE_LAPACK           "Build with lapack support" ON)
 mxnet_option(USE_MKL_IF_AVAILABLE "Use MKL if found" ON)
 mxnet_option(USE_MKLML_MKL        "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND (NOT APPLE))
-mxnet_option(USE_MKLDNN           "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND (NOT APPLE))
+mxnet_option(USE_MKLDNN           "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND (NOT APPLE) AND (CMAKE_SYSTEM_PROCESSOR MATCHES x86_64))
 mxnet_option(USE_OPERATOR_TUNING  "Enable auto-tuning of operators" ON IF NOT MSVC)
 mxnet_option(USE_GPERFTOOLS       "Build with GPerfTools support (if found)" ON)
 mxnet_option(USE_JEMALLOC         "Build with Jemalloc support"   ON)

From 38b3d66ace23abee876c5783b0d10f6e7f18e52a Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Sat, 22 Dec 2018 02:06:36 +0000
Subject: [PATCH 92/96] not making mkldn default on windows

---
 CMakeLists.txt | 2 +-
 Makefile       | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index e11d5f81bffc..e783e5262f8f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -20,7 +20,7 @@ mxnet_option(USE_F16C             "Build with x86 F16C instruction support" ON)
 mxnet_option(USE_LAPACK           "Build with lapack support" ON)
 mxnet_option(USE_MKL_IF_AVAILABLE "Use MKL if found" ON)
 mxnet_option(USE_MKLML_MKL        "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND (NOT APPLE))
-mxnet_option(USE_MKLDNN           "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND (NOT APPLE) AND (CMAKE_SYSTEM_PROCESSOR MATCHES x86_64))
+mxnet_option(USE_MKLDNN           "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND (NOT APPLE) AND (NOT MSVC) AND (CMAKE_SYSTEM_PROCESSOR MATCHES x86_64))
 mxnet_option(USE_OPERATOR_TUNING  "Enable auto-tuning of operators" ON IF NOT MSVC)
 mxnet_option(USE_GPERFTOOLS       "Build with GPerfTools support (if found)" ON)
 mxnet_option(USE_JEMALLOC         "Build with Jemalloc support"   ON)
diff --git a/Makefile b/Makefile
index 23b8a65f1bfa..56f2537d68ff 100644
--- a/Makefile
+++ b/Makefile
@@ -63,11 +63,13 @@ include $(config)
 
 ifndef $(USE_MKLDNN)
 ifneq ($(UNAME_S), Darwin)
+ifneq ($(UNAME_S), Windows)
 ifeq ($(UNAME_P), x86_64)
 	USE_MKLDNN=1
 endif
 endif
 endif
+endif
 
 ifeq ($(USE_MKL2017), 1)
 $(warning "USE_MKL2017 is deprecated. We will switch to USE_MKLDNN.")

From b554519269f71a381885095a94099e4036e46c80 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-40-106.us-east-2.compute.internal>
Date: Wed, 26 Dec 2018 19:24:31 +0000
Subject: [PATCH 93/96] removing Centos MKLDNN tests from CI

---
 ci/docker/runtime_functions.sh    | 19 ------------
 ci/jenkins/Jenkins_steps.groovy   | 51 -------------------------------
 ci/jenkins/Jenkinsfile_centos_cpu |  3 +-
 ci/jenkins/Jenkinsfile_centos_gpu |  6 ++--
 4 files changed, 3 insertions(+), 76 deletions(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index facb96033520..be10cd9a7d2b 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -345,25 +345,6 @@ build_centos7_gpu() {
         -j$(nproc)
 }
 
-build_centos7_gpu_mkldnn() {
-    set -ex
-    cd /work/mxnet
-    # unfortunately this build has problems in 3rdparty dependencies with ccache and make
-    build_ccache_wrappers
-    make \
-        DEV=1                                     \
-        ENABLE_TESTCOVERAGE=1                     \
-        USE_LAPACK=1                              \
-        USE_LAPACK_PATH=/usr/lib64/liblapack.so   \
-        USE_BLAS=openblas                         \
-        USE_CUDA=1                                \
-        USE_CUDA_PATH=/usr/local/cuda             \
-        USE_CUDNN=1                               \
-        USE_DIST_KVSTORE=1                        \
-        CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
-        -j$(nproc)
-}
-
 build_ubuntu_cpu() {
     build_ubuntu_cpu_openblas
 }
diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index b79ae7e5517c..3ca1f2ee0339 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -289,20 +289,6 @@ def compile_centos7_gpu() {
     }]
 }
 
-def compile_centos7_gpu_mkldnn() {
-    return ['GPU: CentOS 7 MKLDNN': {
-      node(NODE_LINUX_CPU) {
-        ws('workspace/build-centos7-gpu-mkldnn') {
-          timeout(time: max_time, unit: 'MINUTES') {
-            utils.init_git()
-            utils.docker_run('centos7_gpu', 'build_centos7_gpu_mkldnn', false)
-            utils.pack_lib('centos7_gpu_mkldnn', mx_mkldnn_lib, true)
-          }
-        }
-      }
-    }]
-}
-
 def compile_unix_clang_3_9_cpu() {
     return ['CPU: Clang 3.9': {
       node(NODE_LINUX_CPU) {
@@ -1006,25 +992,6 @@ def test_centos7_python3_cpu() {
     }]
 }
 
-def test_centos7_python3_cpu_mkldnn() {
-    return ['Python3: CentOS 7 CPU MKLDNN': {
-      node(NODE_LINUX_CPU) {
-        ws('workspace/build-centos7-cpu-mkldnn') {
-          timeout(time: max_time, unit: 'MINUTES') {
-            try {
-              utils.unpack_and_init('centos7_mkldnn', mx_mkldnn_lib, true)
-              utils.docker_run('centos7_cpu', 'unittest_centos7_cpu', false)
-              utils.publish_test_coverage()
-            } finally {
-              utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_centos7_cpu_mkldnn_unittest.xml')
-              utils.collect_test_results_unix('nosetests_train.xml', 'nosetests_python3_centos7_cpu_mkldnn_train.xml')
-            }
-          }
-        }
-      }
-    }]
-}
-
 def test_centos7_python3_gpu() {
     return ['Python3: CentOS 7 GPU': {
       node(NODE_LINUX_GPU) {
@@ -1043,24 +1010,6 @@ def test_centos7_python3_gpu() {
     }]
 }
 
-def test_centos7_python3_gpu_mkldnn() {
-    return ['Python3: CentOS 7 GPU MKLDNN': {
-      node(NODE_LINUX_GPU) {
-        ws('workspace/build-centos7-gpu-mkldnn') {
-          timeout(time: max_time, unit: 'MINUTES') {
-            try {
-              utils.unpack_and_init('centos7_gpu_mkldnn', mx_mkldnn_lib, true)
-              utils.docker_run('centos7_gpu', 'unittest_centos7_gpu', true)
-              utils.publish_test_coverage()
-            } finally {
-              utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_centos7_gpu_mkldnn.xml')
-            }
-          }
-        }
-      }
-    }]
-}
-
 def test_centos7_scala_cpu() {
     return ['Scala: CentOS CPU': {
       node(NODE_LINUX_CPU) {
diff --git a/ci/jenkins/Jenkinsfile_centos_cpu b/ci/jenkins/Jenkinsfile_centos_cpu
index 7f3f5f1e3b8c..3b66f8100173 100644
--- a/ci/jenkins/Jenkinsfile_centos_cpu
+++ b/ci/jenkins/Jenkinsfile_centos_cpu
@@ -40,8 +40,7 @@ core_logic: {
 
   utils.parallel_stage('Tests', [
     custom_steps.test_centos7_python3_cpu(),
-    custom_steps.test_centos7_scala_cpu(),
-    custom_steps.test_centos7_python3_cpu_mkldnn()
+    custom_steps.test_centos7_scala_cpu()
   ]) 
 }
 ,
diff --git a/ci/jenkins/Jenkinsfile_centos_gpu b/ci/jenkins/Jenkinsfile_centos_gpu
index f06fbb14d7d5..aec3b9054f2f 100644
--- a/ci/jenkins/Jenkinsfile_centos_gpu
+++ b/ci/jenkins/Jenkinsfile_centos_gpu
@@ -34,13 +34,11 @@ utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_
 utils.main_wrapper(
 core_logic: {
   utils.parallel_stage('Build', [
-    custom_steps.compile_centos7_gpu(),
-    custom_steps.compile_centos7_gpu_mkldnn()
+    custom_steps.compile_centos7_gpu()
   ]) 
   
   utils.parallel_stage('Tests', [
-    custom_steps.test_centos7_python3_gpu(),
-    custom_steps.test_centos7_python3_gpu_mkldnn()
+    custom_steps.test_centos7_python3_gpu()
   ]) 
 }
 ,

From f770f8ac9d992b4093335e5842ff134aacd855bf Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Fri, 28 Dec 2018 17:31:16 -0800
Subject: [PATCH 94/96] retrigger


From 28ba12cc9452e1eb54f55529c190740e511038ff Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Sat, 29 Dec 2018 14:11:33 -0800
Subject: [PATCH 95/96] retrigger


From 10e40a84cb1a04f02eecd23527a82dfd1dc71e68 Mon Sep 17 00:00:00 2001
From: Alexander Zai <alexzai@amazon.com>
Date: Sun, 30 Dec 2018 08:33:46 -0800
Subject: [PATCH 96/96] retrigger