From 6042b6dc8ebb037f1943821ee4919968a6e53a52 Mon Sep 17 00:00:00 2001
From: Anna Karbownik <anna.karbownik@intel.com>
Date: Fri, 30 Jul 2021 09:53:47 +0200
Subject: [PATCH 01/16] [FEATURE] Enable dynamic linking with MKL and compiler
 based OpenMP

OneMKL 2021.3 fixed linking OpenMP while using SDL and
MKL_THREADING_LAYER set to GNU.
---
 ci/docker/Dockerfile.build.ubuntu |  11 +-
 cmake/ChooseBlas.cmake            |  17 ++-
 cmake/upstream/FindBLAS.cmake     |   6 +-
 src/initialize.cc                 | 177 ++++++++++++++++++------------
 4 files changed, 133 insertions(+), 78 deletions(-)

diff --git a/ci/docker/Dockerfile.build.ubuntu b/ci/docker/Dockerfile.build.ubuntu
index 843d0f97047e..1bd1b7b57241 100644
--- a/ci/docker/Dockerfile.build.ubuntu
+++ b/ci/docker/Dockerfile.build.ubuntu
@@ -38,11 +38,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
     export OS_RELEASE="$(cat /etc/os-release)" && \
     apt-get update && \
     apt-get install -y wget software-properties-common && \
-    if [[ ${OS_RELEASE} == *"Bionic"* ]]; then \
-        wget -qO - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB | apt-key add -; \
-        apt-add-repository "deb https://apt.repos.intel.com/mkl all main"; \
-        INTEL_MKL="-2020.0-088"; \
-    fi && \
+    wget -qO - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB -O - | apt-key add -; \
+    add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"; \
+    INTEL_MKL="-2021.3.0"; \
     apt-get update && \
     apt-get install -y \
         ## Utilities
@@ -65,7 +63,8 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         g++ \
         g++-7 \
         g++-8 \
-        intel-mkl${INTEL_MKL} \
+        intel-oneapi-mkl${INTEL_MKL} \
+        intel-oneapi-mkl-devel${INTEL_MKL} \
         libomp-dev \
         ## Dependencies
         libgomp1 \
diff --git a/cmake/ChooseBlas.cmake b/cmake/ChooseBlas.cmake
index b4d025e0e5d7..efaa4ec2f79b 100644
--- a/cmake/ChooseBlas.cmake
+++ b/cmake/ChooseBlas.cmake
@@ -122,10 +122,19 @@ set(FORTRAN_DIR \\\"\$\{CMAKE_Fortran_IMPLICIT_LINK_DIRECTORIES\}\\\")
   endif()
 elseif(BLAS STREQUAL "MKL" OR BLAS STREQUAL "mkl")
   # ---[ MKL Options
+  find_path(MKL_INCLUDE_DIR mkl.h
+    HINTS ${INTEL_HOME_ROOT}/mkl ${INTEL_OPT_ROOT}/mkl ${INTEL_OPT_ROOT}/oneapi/mkl/latest
+    PATHS $ENV{MKLROOT} $ENV{MKLROOT}/latest
+    PATH_SUFFIXES include mkl REQUIRED)
+  file(STRINGS ${MKL_INCLUDE_DIR}/mkl_version.h MKL_VERSION_DEF REGEX "INTEL_MKL_VERSION")
+  string(REGEX MATCH "([0-9]+)" MKL_VERSION ${MKL_VERSION_DEF})
   if(UNIX)
     # Single dynamic library interface leads to conflicts between intel omp and llvm omp
     # https://github.com/apache/incubator-mxnet/issues/17641
-    option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" OFF)
+    # Fixed in oneMKL 2021.3: [MKLD-11109] MKL is opening libgomp.so instead of
+    # libgomp.so.1 while SDL=1 & MKL_THREADING_LAYER=GNU
+    cmake_dependent_option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" ON
+      "NOT BLA_STATIC;MKL_VERSION GREATER_EQUAL 20210003" OFF)
   else()
     option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" ON)
   endif()
@@ -161,14 +170,16 @@ elseif(BLAS STREQUAL "MKL" OR BLAS STREQUAL "mkl")
     endif()
   endif()
   # Setting up BLAS_mkl_MKLROOT for non-Ubuntu 20.04 OSes
-  find_path(BLAS_mkl_MKLROOT mkl PATHS $ENV{MKLROOT} ${INTEL_HOME_ROOT} ${INTEL_OPT_ROOT})
+  find_path(BLAS_mkl_MKLROOT include/mkl.h
+    PATHS $ENV{MKLROOT} ${INTEL_HOME_ROOT} ${INTEL_OPT_ROOT} ${INTEL_OPT_ROOT}/oneapi/mkl
+    PATH_SUFFIXES mkl latest)
   find_package(BLAS)
-  find_path(MKL_INCLUDE_DIR mkl.h HINTS ${INTEL_HOME_ROOT}/mkl ${INTEL_OPT_ROOT}/mkl PATHS ENV MKLROOT PATH_SUFFIXES include mkl REQUIRED)
   include_directories(SYSTEM ${MKL_INCLUDE_DIR})
   list(APPEND mshadow_LINKER_LIBS ${BLAS_LIBRARIES})
   add_definitions(-DMSHADOW_USE_CBLAS=0)
   add_definitions(-DMSHADOW_USE_MKL=1)
   add_definitions(-DMXNET_USE_BLAS_MKL=1)
+  message("-- Found MKL (version: ${MKL_VERSION})")
 elseif(BLAS STREQUAL "apple")
   find_package(Accelerate REQUIRED)
   include_directories(SYSTEM ${Accelerate_INCLUDE_DIR})
diff --git a/cmake/upstream/FindBLAS.cmake b/cmake/upstream/FindBLAS.cmake
index 934dd444c565..b22a34dd4184 100644
--- a/cmake/upstream/FindBLAS.cmake
+++ b/cmake/upstream/FindBLAS.cmake
@@ -522,12 +522,16 @@ if(BLA_VENDOR MATCHES "Intel" OR BLA_VENDOR STREQUAL "All")
             get_filename_component(BLAS_mkl_MKLROOT "${BLAS_mkl_MKLROOT}" DIRECTORY)
         endif()
       endif()
+      # MXNET NOTE: This change comes form the newest file version
+      # https://gitlab.kitware.com/cmake/cmake/-/issues/22295
       set(BLAS_mkl_LIB_PATH_SUFFIXES
           "compiler/lib" "compiler/lib/${BLAS_mkl_ARCH_NAME}_${BLAS_mkl_OS_NAME}"
           "compiler/lib/${BLAS_mkl_ARCH_NAME}"
           "mkl/lib" "mkl/lib/${BLAS_mkl_ARCH_NAME}_${BLAS_mkl_OS_NAME}"
           "mkl/lib/${BLAS_mkl_ARCH_NAME}"
-          "lib/${BLAS_mkl_ARCH_NAME}_${BLAS_mkl_OS_NAME}")
+          "lib" "lib/${BLAS_mkl_ARCH_NAME}_${BLAS_mkl_OS_NAME}"
+          "lib/${BLAS_mkl_ARCH_NAME}"
+          )
 
       foreach(IT ${BLAS_SEARCH_LIBS})
         string(REPLACE " " ";" SEARCH_LIBS ${IT})
diff --git a/src/initialize.cc b/src/initialize.cc
index 225eb2dea27c..b7040d01b243 100644
--- a/src/initialize.cc
+++ b/src/initialize.cc
@@ -1,4 +1,4 @@
-/*
+  /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -31,16 +31,17 @@
  * \brief Retrieve the system error message for the last-error code
  * \param err string that gets the error message
  */
-void win_err(char** err) {
+void win_err(char **err) {
   uint32_t dw = GetLastError();
   FormatMessage(
-      FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
-      nullptr,
-      dw,
-      MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
-      reinterpret_cast<char*>(err),
-      0,
-      nullptr);
+        FORMAT_MESSAGE_ALLOCATE_BUFFER |
+        FORMAT_MESSAGE_FROM_SYSTEM |
+        FORMAT_MESSAGE_IGNORE_INSERTS,
+        nullptr,
+        dw,
+        MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+        reinterpret_cast<char*>(err),
+        0, nullptr);
 }
 #else
 #include <cxxabi.h>
@@ -62,6 +63,10 @@ void win_err(char** err) {
 #include "common/utils.h"
 #include "engine/openmp.h"
 
+#if defined(MKL_USE_SINGLE_DYNAMIC_LIBRARY)
+#include <mkl.h>
+#endif
+
 namespace mxnet {
 
 // pthread_atfork handlers, delegated to LibraryInitializer members.
@@ -84,12 +89,19 @@ void pthread_atfork_child() {
 // LibraryInitializer member functions
 
 LibraryInitializer::LibraryInitializer()
-    : original_pid_(common::current_process_id()),
-      mp_worker_nthreads_(dmlc::GetEnv("MXNET_MP_WORKER_NTHREADS", 1)),
-      cpu_worker_nthreads_(dmlc::GetEnv("MXNET_CPU_WORKER_NTHREADS", 1)),
-      mp_cv_num_threads_(dmlc::GetEnv("MXNET_MP_OPENCV_NUM_THREADS", 0)) {
+  : original_pid_(common::current_process_id()),
+    mp_worker_nthreads_(dmlc::GetEnv("MXNET_MP_WORKER_NTHREADS", 1)),
+    cpu_worker_nthreads_(dmlc::GetEnv("MXNET_CPU_WORKER_NTHREADS", 1)),
+    mp_cv_num_threads_(dmlc::GetEnv("MXNET_MP_OPENCV_NUM_THREADS", 0)) {
   dmlc::InitLogging("mxnet");
-  engine::OpenMP::Get();  // force OpenMP initialization
+  #if MKL_USE_SINGLE_DYNAMIC_LIBRARY
+    #if defined( __INTEL_LLVM_COMPILER)
+      mkl_set_threading_layer(MKL_THREADING_INTEL);
+    #else
+      mkl_set_threading_layer(MKL_THREADING_GNU);
+    #endif
+  #endif
+  engine::OpenMP::Get();   // force OpenMP initialization
   install_pthread_atfork_handlers();
 }
 
@@ -105,14 +117,14 @@ bool LibraryInitializer::lib_is_loaded(const std::string& path) const {
  * \return handle a pointer for the loaded library, throws dmlc::error if library can't be loaded
  */
 void* LibraryInitializer::lib_load(const char* path) {
-  void* handle = nullptr;
+  void *handle = nullptr;
   // check if library was already loaded
   if (!lib_is_loaded(path)) {
     // if not, load it
 #if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)
     handle = LoadLibrary(path);
     if (!handle) {
-      char* err_msg = nullptr;
+      char *err_msg = nullptr;
       win_err(&err_msg);
       LOG(FATAL) << "Error loading library: '" << path << "'\n" << err_msg;
       LocalFree(err_msg);
@@ -157,7 +169,7 @@ void LibraryInitializer::lib_close(void* handle) {
 #else
   if (dlclose(handle)) {
     LOG(WARNING) << "LibraryInitializer::lib_close: couldn't close library at address: " << handle
-                 << " loaded from: '" << libpath << "': " << dlerror();
+        << " loaded from: '" << libpath << "': " << dlerror();
   }
 #endif  // _WIN32 or _WIN64 or __WINDOWS__
   loaded_libs.erase(libpath);
@@ -169,11 +181,11 @@ void LibraryInitializer::lib_close(void* handle) {
  * \param func function pointer that gets output address
  * \param name function name to be fetched
  */
-void LibraryInitializer::get_sym(void* handle, void** func, const char* name) {
+void LibraryInitializer::get_sym(void* handle, void** func, char* name) {
 #if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)
   *func = GetProcAddress((HMODULE)handle, name);
   if (!(*func)) {
-    char* err_msg = nullptr;
+    char *err_msg = nullptr;
     win_err(&err_msg);
     LOG(FATAL) << "Error getting function '" << name << "' from library\n" << err_msg;
     LocalFree(err_msg);
@@ -216,6 +228,7 @@ void LibraryInitializer::atfork_child() {
   CustomOperator::Get()->Start();
 }
 
+
 void LibraryInitializer::install_pthread_atfork_handlers() {
 #ifndef _WIN32
   engine::OpenMP::Get()->initialize_process();  // force omp to set its atfork handler first
@@ -223,15 +236,38 @@ void LibraryInitializer::install_pthread_atfork_handlers() {
 #endif
 }
 
+void LibraryInitializer::init_mkl_dynamic_library() {
+#if !(defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__))
+  #if MKL_USE_SINGLE_DYNAMIC_LIBRARY
+    #if USE_INT64_TENSOR_SIZE
+      int interface = MKL_INTERFACE_ILP64;
+    #else
+      int interface = MKL_INTERFACE_LP64;
+    #endif
+    #if defined( __INTEL_LLVM_COMPILER)
+      mkl_set_threading_layer(MKL_THREADING_INTEL);
+    #else
+      mkl_set_threading_layer(MKL_THREADING_GNU);
+      interface += MKL_INTERFACE_GNU;
+    #endif
+    mkl_set_interface_layer(interface);
+  #endif
+#endif
+}
+
+
+
 #if MXNET_USE_SIGNAL_HANDLER && DMLC_LOG_STACK_TRACE
 
-static inline void printStackTrace(FILE* out = stderr, const unsigned int max_frames = 63) {
+static inline void printStackTrace(FILE *out = stderr,
+    const unsigned int max_frames = 63) {
+
 #if !defined(_WIN32) && !defined(_WIN64) && !defined(__WINDOWS__)
   // storage array for stack trace address data
-  void* addrlist[max_frames + 1];
+  void* addrlist[max_frames+1];
 
   // retrieve current stack addresses
-  size_t addrlen = backtrace(addrlist, sizeof(addrlist) / sizeof(void*));
+  size_t addrlen = backtrace(addrlist, sizeof(addrlist)/sizeof(void*));
 
   if (addrlen < 5) {
     return;
@@ -240,6 +276,7 @@ static inline void printStackTrace(FILE* out = stderr, const unsigned int max_fr
   }
   fprintf(out, "Stack trace:\n");
 
+
   // resolve addresses into strings containing "filename(function+address)",
   // Actually it will be ## program address function + offset
   // this array must be free()-ed
@@ -250,7 +287,7 @@ static inline void printStackTrace(FILE* out = stderr, const unsigned int max_fr
 
   // iterate over the returned symbol lines. skip the first, it is the
   // address of this function.
-  for (unsigned int i = 4; i < addrlen; i++) {
+  for (unsigned int i = 4; i < addrlen ; i++) {
     char* begin_name   = nullptr;
     char* begin_offset = nullptr;
     char* end_offset   = nullptr;
@@ -258,37 +295,40 @@ static inline void printStackTrace(FILE* out = stderr, const unsigned int max_fr
     // find parentheses and +address offset surrounding the mangled name
 #ifdef DARWIN
     // OSX style stack trace
-    for (char* p = symbollist[i]; *p; ++p) {
-      if (*p == '_' && *(p - 1) == ' ') {
-        begin_name = p - 1;
+    for (char *p = symbollist[i]; *p; ++p) {
+      if (*p == '_' && *(p-1) == ' ') {
+        begin_name = p-1;
       } else if (*p == '+') {
-        begin_offset = p - 1;
+        begin_offset = p-1;
       }
     }
 
     if (begin_name && begin_offset && begin_name < begin_offset) {
-      *begin_name++   = '\0';
+      *begin_name++ = '\0';
       *begin_offset++ = '\0';
 
       // mangled name is now in [begin_name, begin_offset) and caller
       // offset in [begin_offset, end_offset). now apply
       // __cxa_demangle():
       int status;
-      char* ret = abi::__cxa_demangle(begin_name, &funcname[0], &funcnamesize, &status);
+      char* ret = abi::__cxa_demangle(begin_name, &funcname[0],
+                                      &funcnamesize, &status);
       if (status == 0) {
         funcname = ret;  // use possibly realloc()-ed string
-        fprintf(out, "  %-30s %-40s %s\n", symbollist[i], funcname, begin_offset);
+        fprintf(out, "  %-30s %-40s %s\n",
+                symbollist[i], funcname, begin_offset);
       } else {
         // demangling failed. Output function name as a C function with
         // no arguments.
-        fprintf(out, "  %-30s %-38s() %s\n", symbollist[i], begin_name, begin_offset);
+        fprintf(out, "  %-30s %-38s() %s\n",
+                symbollist[i], begin_name, begin_offset);
       }
     } else {
-      // couldn't parse the line? print the whole line.
-      fprintf(out, "  %-40s\n", symbollist[i]);
+       // couldn't parse the line? print the whole line.
+       fprintf(out, "  %-40s\n", symbollist[i]);
     }
 #else
-    for (char* p = symbollist[i]; *p; ++p) {
+    for (char *p = symbollist[i]; *p; ++p) {
       if (*p == '(') {
         begin_name = p;
       } else if (*p == '+') {
@@ -309,22 +349,24 @@ static inline void printStackTrace(FILE* out = stderr, const unsigned int max_fr
       // offset in [begin_offset, end_offset). now apply
       // __cxa_demangle():
 
-      int status  = 0;
-      char* ret   = abi::__cxa_demangle(begin_name, funcname, &funcnamesize, &status);
+      int status = 0;
+      char* ret = abi::__cxa_demangle(begin_name, funcname,
+                                      &funcnamesize, &status);
       char* fname = begin_name;
       if (status == 0) {
         fname = ret;
       }
 
       if (begin_offset) {
-        fprintf(
-            out, "  %-30s ( %-40s  + %-6s) %s\n", symbollist[i], fname, begin_offset, end_offset);
+        fprintf(out, "  %-30s ( %-40s  + %-6s) %s\n",
+                symbollist[i], fname, begin_offset, end_offset);
       } else {
-        fprintf(out, "  %-30s ( %-40s    %-6s) %s\n", symbollist[i], fname, "", end_offset);
+        fprintf(out, "  %-30s ( %-40s    %-6s) %s\n",
+                symbollist[i], fname, "", end_offset);
       }
     } else {
-      // couldn't parse the line? print the whole line.
-      fprintf(out, "  %-40s\n", symbollist[i]);
+       // couldn't parse the line? print the whole line.
+       fprintf(out, "  %-40s\n", symbollist[i]);
     }
 #endif  // !DARWIN - but is posix
   }
@@ -332,33 +374,32 @@ static inline void printStackTrace(FILE* out = stderr, const unsigned int max_fr
 #endif
 }
 
-#define SIGNAL_HANDLER(SIGNAL, HANDLER_NAME, IS_FATAL)                          \
-  std::shared_ptr<void(int)> HANDLER_NAME(                                      \
-      signal(SIGNAL,                                                            \
-             [](int signum) {                                                   \
-               if (IS_FATAL) {                                                  \
-                 printf("\nFatal Error: %s\n", strsignal(SIGNAL));              \
-                 printStackTrace();                                             \
-                 signal(signum, SIG_DFL);                                       \
-                 raise(signum);                                                 \
-               } else {                                                         \
-                 switch (signum) {                                              \
-                   case SIGSEGV:                                                \
-                     LOG(FATAL) << "InternalError: " << strsignal(SIGNAL);      \
-                     break;                                                     \
-                   case SIGFPE:                                                 \
-                     LOG(FATAL) << "FloatingPointError: " << strsignal(SIGNAL); \
-                     break;                                                     \
-                   case SIGBUS:                                                 \
-                     LOG(FATAL) << "IOError: " << strsignal(SIGNAL);            \
-                     break;                                                     \
-                   default:                                                     \
-                     LOG(FATAL) << "RuntimeError: " << strsignal(SIGNAL);       \
-                     break;                                                     \
-                 }                                                              \
-               }                                                                \
-             }),                                                                \
-      [](auto f) { signal(SIGNAL, f); });
+#define SIGNAL_HANDLER(SIGNAL, HANDLER_NAME, IS_FATAL)               \
+std::shared_ptr<void(int)> HANDLER_NAME(                             \
+  signal(SIGNAL, [](int signum) {                                    \
+    if (IS_FATAL) {                                                  \
+      printf("\nFatal Error: %s\n", strsignal(SIGNAL));              \
+      printStackTrace();                                             \
+      signal(signum, SIG_DFL);                                       \
+      raise(signum);                                                 \
+    } else {                                                         \
+      switch (signum) {                                              \
+        case SIGSEGV:                                                \
+          LOG(FATAL) << "InternalError: " << strsignal(SIGNAL);      \
+          break;                                                     \
+        case SIGFPE:                                                 \
+          LOG(FATAL) << "FloatingPointError: " << strsignal(SIGNAL); \
+          break;                                                     \
+        case SIGBUS:                                                 \
+          LOG(FATAL) << "IOError: " << strsignal(SIGNAL);            \
+          break;                                                     \
+        default:                                                     \
+          LOG(FATAL) << "RuntimeError: " << strsignal(SIGNAL);       \
+          break;                                                     \
+      }                                                              \
+    }                                                                \
+  }),                                                                \
+  [](auto f) { signal(SIGNAL, f); });
 
 SIGNAL_HANDLER(SIGSEGV, SIGSEGVHandler, true);
 SIGNAL_HANDLER(SIGFPE, SIGFPEHandler, false);

From 70071302f62ca8d08d0b3acd13c1721b81cb295b Mon Sep 17 00:00:00 2001
From: Anna Karbownik <anna.karbownik@intel.com>
Date: Fri, 30 Jul 2021 21:31:43 +0200
Subject: [PATCH 02/16] Disabling SDL with MKL threading on Windows

---
 src/initialize.cc | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/initialize.cc b/src/initialize.cc
index b7040d01b243..9345d1052a24 100644
--- a/src/initialize.cc
+++ b/src/initialize.cc
@@ -94,13 +94,17 @@ LibraryInitializer::LibraryInitializer()
     cpu_worker_nthreads_(dmlc::GetEnv("MXNET_CPU_WORKER_NTHREADS", 1)),
     mp_cv_num_threads_(dmlc::GetEnv("MXNET_MP_OPENCV_NUM_THREADS", 0)) {
   dmlc::InitLogging("mxnet");
-  #if MKL_USE_SINGLE_DYNAMIC_LIBRARY
-    #if defined( __INTEL_LLVM_COMPILER)
-      mkl_set_threading_layer(MKL_THREADING_INTEL);
-    #else
-      mkl_set_threading_layer(MKL_THREADING_GNU);
-    #endif
-  #endif
+
+#if !(defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__))
+#if MKL_USE_SINGLE_DYNAMIC_LIBRARY
+#if defined( __INTEL_LLVM_COMPILER)
+  mkl_set_threading_layer(MKL_THREADING_INTEL);
+#else
+  mkl_set_threading_layer(MKL_THREADING_GNU);
+#endif
+#endif
+#endif
+
   engine::OpenMP::Get();   // force OpenMP initialization
   install_pthread_atfork_handlers();
 }

From 72c060ec04c4d5700c0a34bc450366fbde5fcd55 Mon Sep 17 00:00:00 2001
From: Anna Karbownik <anna.karbownik@intel.com>
Date: Wed, 4 Aug 2021 15:26:01 +0200
Subject: [PATCH 03/16] Use multi-threading 'on' as the default option

---
 cmake/ChooseBlas.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/ChooseBlas.cmake b/cmake/ChooseBlas.cmake
index efaa4ec2f79b..ccbe7e30b207 100644
--- a/cmake/ChooseBlas.cmake
+++ b/cmake/ChooseBlas.cmake
@@ -139,7 +139,7 @@ elseif(BLAS STREQUAL "MKL" OR BLAS STREQUAL "mkl")
     option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" ON)
   endif()
   cmake_dependent_option(BLA_STATIC "Use static libraries" ON "NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY" OFF)
-  cmake_dependent_option(MKL_MULTI_THREADED  "Use multi-threading"  ON "NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY" OFF)
+  option(MKL_MULTI_THREADED  "Use multi-threading" ON)
 
   if(BLA_VENDOR)
       message(FATAL_ERROR "Do not set BLA_VENDOR manually. MKL version (BLA_VENDOR) is selected based on MKL_USE_SINGLE_DYNAMIC_LIBRARY, "

From e5dd2a78431b730e92e98efa0bf630bfd0a6b696 Mon Sep 17 00:00:00 2001
From: Anna Karbownik <anna.karbownik@intel.com>
Date: Thu, 5 Aug 2021 18:16:48 +0200
Subject: [PATCH 04/16] Sets the interface layer for Intel oneAPI MKL at
 runtime

---
 cmake/ChooseBlas.cmake |  3 +++
 src/initialize.cc      | 21 ++++++++++++++-------
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/cmake/ChooseBlas.cmake b/cmake/ChooseBlas.cmake
index ccbe7e30b207..39809abd29a8 100644
--- a/cmake/ChooseBlas.cmake
+++ b/cmake/ChooseBlas.cmake
@@ -176,6 +176,9 @@ elseif(BLAS STREQUAL "MKL" OR BLAS STREQUAL "mkl")
   find_package(BLAS)
   include_directories(SYSTEM ${MKL_INCLUDE_DIR})
   list(APPEND mshadow_LINKER_LIBS ${BLAS_LIBRARIES})
+  if(USE_INT64_TENSOR_SIZE)
+    add_definitions(-DUSE_INT64_TENSOR_SIZE=1)
+  endif()
   add_definitions(-DMSHADOW_USE_CBLAS=0)
   add_definitions(-DMSHADOW_USE_MKL=1)
   add_definitions(-DMXNET_USE_BLAS_MKL=1)
diff --git a/src/initialize.cc b/src/initialize.cc
index 9345d1052a24..15587249adfb 100644
--- a/src/initialize.cc
+++ b/src/initialize.cc
@@ -96,13 +96,20 @@ LibraryInitializer::LibraryInitializer()
   dmlc::InitLogging("mxnet");
 
 #if !(defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__))
-#if MKL_USE_SINGLE_DYNAMIC_LIBRARY
-#if defined( __INTEL_LLVM_COMPILER)
-  mkl_set_threading_layer(MKL_THREADING_INTEL);
-#else
-  mkl_set_threading_layer(MKL_THREADING_GNU);
-#endif
-#endif
+  #if MKL_USE_SINGLE_DYNAMIC_LIBRARY
+    #if USE_INT64_TENSOR_SIZE
+      int interface = MKL_INTERFACE_ILP64;
+    #else
+      int interface = MKL_INTERFACE_LP64;
+    #endif
+    #if defined( __INTEL_LLVM_COMPILER)
+      mkl_set_threading_layer(MKL_THREADING_INTEL);
+    #else
+      mkl_set_threading_layer(MKL_THREADING_GNU);
+      interface += MKL_INTERFACE_GNU;
+    #endif
+    mkl_set_interface_layer(interface);
+  #endif
 #endif
 
   engine::OpenMP::Get();   // force OpenMP initialization

From 0c5f9cc544d2fdc5a995d7ef2458103308bcfdba Mon Sep 17 00:00:00 2001
From: Anna Karbownik <anna.karbownik@intel.com>
Date: Mon, 9 Aug 2021 14:42:26 +0200
Subject: [PATCH 05/16] Clean up the apt cache

---
 ci/docker/Dockerfile.build.ubuntu | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/docker/Dockerfile.build.ubuntu b/ci/docker/Dockerfile.build.ubuntu
index 1bd1b7b57241..f8963d3758be 100644
--- a/ci/docker/Dockerfile.build.ubuntu
+++ b/ci/docker/Dockerfile.build.ubuntu
@@ -36,6 +36,7 @@ WORKDIR /work/deps
 SHELL ["/bin/bash", "-c"]
 RUN export DEBIAN_FRONTEND=noninteractive && \
     export OS_RELEASE="$(cat /etc/os-release)" && \
+    apt-get clean && \
     apt-get update && \
     apt-get install -y wget software-properties-common && \
     wget -qO - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB -O - | apt-key add -; \

From e6f764177c3bd62fe44fcf1d0129b4b0f3706789 Mon Sep 17 00:00:00 2001
From: Anna Karbownik <anna.karbownik@intel.com>
Date: Tue, 24 Aug 2021 21:11:00 +0200
Subject: [PATCH 06/16] Moving mkl runtime initialization to the function

---
 src/initialize.cc | 19 +------------------
 src/initialize.h  |  5 +++++
 2 files changed, 6 insertions(+), 18 deletions(-)

diff --git a/src/initialize.cc b/src/initialize.cc
index 15587249adfb..21211f3473b9 100644
--- a/src/initialize.cc
+++ b/src/initialize.cc
@@ -94,24 +94,7 @@ LibraryInitializer::LibraryInitializer()
     cpu_worker_nthreads_(dmlc::GetEnv("MXNET_CPU_WORKER_NTHREADS", 1)),
     mp_cv_num_threads_(dmlc::GetEnv("MXNET_MP_OPENCV_NUM_THREADS", 0)) {
   dmlc::InitLogging("mxnet");
-
-#if !(defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__))
-  #if MKL_USE_SINGLE_DYNAMIC_LIBRARY
-    #if USE_INT64_TENSOR_SIZE
-      int interface = MKL_INTERFACE_ILP64;
-    #else
-      int interface = MKL_INTERFACE_LP64;
-    #endif
-    #if defined( __INTEL_LLVM_COMPILER)
-      mkl_set_threading_layer(MKL_THREADING_INTEL);
-    #else
-      mkl_set_threading_layer(MKL_THREADING_GNU);
-      interface += MKL_INTERFACE_GNU;
-    #endif
-    mkl_set_interface_layer(interface);
-  #endif
-#endif
-
+  init_mkl_dynamic_library();
   engine::OpenMP::Get();   // force OpenMP initialization
   install_pthread_atfork_handlers();
 }
diff --git a/src/initialize.h b/src/initialize.h
index 490a3eed489e..489211c70111 100644
--- a/src/initialize.h
+++ b/src/initialize.h
@@ -91,6 +91,11 @@ class LibraryInitializer {
    */
   void install_pthread_atfork_handlers();
 
+  /**
+  * Sets the interface and threading layer for Intel® oneAPI MKL at run time.
+  * Use with the Single Dynamic Library.
+  */
+  void init_mkl_dynamic_library();
   /**
    * Install signal handlers (UNIX). Has no effect on Windows.
    */

From 6f628a79b5b23331fabcf6e150d9e8f837b4fc7c Mon Sep 17 00:00:00 2001
From: Anna Karbownik <anna.karbownik@intel.com>
Date: Thu, 2 Sep 2021 15:48:59 +0200
Subject: [PATCH 07/16] Cleaning MKL find_path cmake directories

---
 cmake/ChooseBlas.cmake | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/cmake/ChooseBlas.cmake b/cmake/ChooseBlas.cmake
index 39809abd29a8..ed6c7337acd4 100644
--- a/cmake/ChooseBlas.cmake
+++ b/cmake/ChooseBlas.cmake
@@ -23,10 +23,12 @@ set(INTEL_OPT_ROOT "/opt/intel" CACHE PATH "Folder contains root-installed intel
 
 if(DEFINED USE_BLAS)
   set(BLAS "${USE_BLAS}")
-else()
-  # Setting up BLAS_mkl_MKLROOT for non-Ubuntu 20.04 OSes
-  find_path(BLAS_mkl_MKLROOT mkl PATHS $ENV{MKLROOT} ${INTEL_HOME_ROOT} ${INTEL_OPT_ROOT})
-  if(NOT BLAS_mkl_MKLROOT STREQUAL "BLAS_mkl_MKLROOT-NOTFOUND")
+endif()
+if(USE_BLAS MATCHES "MKL" OR USE_BLAS MATCHES "mkl" OR NOT DEFINED USE_BLAS)
+  find_path(MKL_INCLUDE_DIR mkl_version.h
+    PATHS $ENV{MKLROOT} ${INTEL_HOME_ROOT}/mkl ${INTEL_OPT_ROOT}/mkl ${INTEL_OPT_ROOT}/oneapi/mkl/latest
+    PATH_SUFFIXES mkl latest include)
+  if(NOT MKL_INCLUDE_DIR STREQUAL "MKL_INCLUDE_DIR-NOTFOUND")
     set(BLAS "MKL")
   endif()
 endif()
@@ -122,10 +124,6 @@ set(FORTRAN_DIR \\\"\$\{CMAKE_Fortran_IMPLICIT_LINK_DIRECTORIES\}\\\")
   endif()
 elseif(BLAS STREQUAL "MKL" OR BLAS STREQUAL "mkl")
   # ---[ MKL Options
-  find_path(MKL_INCLUDE_DIR mkl.h
-    HINTS ${INTEL_HOME_ROOT}/mkl ${INTEL_OPT_ROOT}/mkl ${INTEL_OPT_ROOT}/oneapi/mkl/latest
-    PATHS $ENV{MKLROOT} $ENV{MKLROOT}/latest
-    PATH_SUFFIXES include mkl REQUIRED)
   file(STRINGS ${MKL_INCLUDE_DIR}/mkl_version.h MKL_VERSION_DEF REGEX "INTEL_MKL_VERSION")
   string(REGEX MATCH "([0-9]+)" MKL_VERSION ${MKL_VERSION_DEF})
   if(UNIX)
@@ -169,10 +167,13 @@ elseif(BLAS STREQUAL "MKL" OR BLAS STREQUAL "mkl")
       endif()
     endif()
   endif()
-  # Setting up BLAS_mkl_MKLROOT for non-Ubuntu 20.04 OSes
-  find_path(BLAS_mkl_MKLROOT include/mkl.h
-    PATHS $ENV{MKLROOT} ${INTEL_HOME_ROOT} ${INTEL_OPT_ROOT} ${INTEL_OPT_ROOT}/oneapi/mkl
-    PATH_SUFFIXES mkl latest)
+  # In case of oneAPI 2021.3 if MKL_INCLUDE_DIR points to the subdirectory 'include',
+  # use the parent directory 'latest' instead
+  file(TO_CMAKE_PATH "${MKL_INCLUDE_DIR}" BLAS_mkl_MKLROOT)
+  get_filename_component(BLAS_mkl_MKLROOT_LAST_DIR "${BLAS_mkl_MKLROOT}" NAME)
+  if(BLAS_mkl_MKLROOT_LAST_DIR STREQUAL "include")
+      get_filename_component(BLAS_mkl_MKLROOT "${BLAS_mkl_MKLROOT}" DIRECTORY)
+  endif()
   find_package(BLAS)
   include_directories(SYSTEM ${MKL_INCLUDE_DIR})
   list(APPEND mshadow_LINKER_LIBS ${BLAS_LIBRARIES})

From ae6686b5d0d1d4cec0243532e577b595114fbb9d Mon Sep 17 00:00:00 2001
From: Anna Karbownik <anna.karbownik@intel.com>
Date: Sun, 19 Sep 2021 21:19:03 +0200
Subject: [PATCH 08/16] [WIP] Adding github runner for MAC OS to check MKL
 specific changes

This is a temporary change to check if adding MKL runtime support
won't crash MacOS.
---
 .github/workflows/os_x_mklbuild.yml           | 57 +++++++++++++++++++
 config/distribution/darwin_cpu_mkl.cmake      | 33 +++++++++++
 config/distribution/linux_cpu_mkl.cmake       | 32 +++++++++++
 src/initialize.cc                             |  2 +-
 .../dependencies/make_shared_dependencies.sh  |  8 ++-
 tools/dependencies/mkl.sh                     | 50 ++++++++++++++++
 tools/dependencies/opencv.sh                  |  2 +-
 tools/staticbuild/build.sh                    |  7 ++-
 tools/staticbuild/build_lib.sh                | 11 +++-
 9 files changed, 196 insertions(+), 6 deletions(-)
 create mode 100644 .github/workflows/os_x_mklbuild.yml
 create mode 100644 config/distribution/darwin_cpu_mkl.cmake
 create mode 100644 config/distribution/linux_cpu_mkl.cmake
 create mode 100644 tools/dependencies/mkl.sh

diff --git a/.github/workflows/os_x_mklbuild.yml b/.github/workflows/os_x_mklbuild.yml
new file mode 100644
index 000000000000..e2aa94c3c744
--- /dev/null
+++ b/.github/workflows/os_x_mklbuild.yml
@@ -0,0 +1,57 @@
+name: continuous build
+
+on: [push, pull_request]
+
+jobs:
+  macosx-x86_64:
+    runs-on: macos-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Compilation cache
+        uses: actions/cache@v2
+        with:
+          path: ~/.ccache
+          # We include the commit sha in the cache key, as new cache entries are
+          # only created if there is no existing entry for the key yet.
+          key: ${{ runner.os }}-ccache-${{ github.sha }}
+          # Restore any ccache cache entry, if none for
+          # ${{ runner.os }}-ccache-${{ github.sha }} exists
+          restore-keys: |
+            ${{ runner.os }}-ccache
+
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.6'
+          architecture: x64
+
+      - name: Install Dependencies
+        run: |
+          brew install nasm automake ninja libtool cmake pkgconfig protobuf hdf5 zlib ccache
+          ccache -M 500M  # Limit the ccache size; Github's overall cache limit is 5GB
+          python -m pip install -r ci/docker/install/requirements
+        shell: bash
+
+      - name: Build project
+        run: |
+          ./tools/staticbuild/build.sh cpu mkl
+
+      - name: Setup Python
+        run: |
+          python -m pip install --user -e python
+
+      - name: Build with Cython
+        run: |
+          cd python
+          python setup.py build_ext --inplace --with-cython
+
+      - name: Test project
+        env:
+          MXNET_ENFORCE_CYTHON: 1
+        run: |
+          python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'not test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
+          MXNET_ENGINE_TYPE=NaiveEngine python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
+          python3 -m pytest --durations=50 --verbose tests/python/unittest/ -k 'not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'serial'
+          python3 -m pytest -n 4 --durations=50 --verbose tests/python/mkl
diff --git a/config/distribution/darwin_cpu_mkl.cmake b/config/distribution/darwin_cpu_mkl.cmake
new file mode 100644
index 000000000000..37aa4c36f78a
--- /dev/null
+++ b/config/distribution/darwin_cpu_mkl.cmake
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set(CMAKE_BUILD_TYPE "Distribution" CACHE STRING "Build type")
+set(CFLAGS "-mno-avx" CACHE STRING "CFLAGS")
+set(CXXFLAGS "-mno-avx" CACHE STRING "CXXFLAGS")
+
+set(USE_BLAS "mkl" CACHE STRING "BLAS Vendor")
+
+set(USE_CUDA OFF CACHE BOOL "Build with CUDA support")
+set(USE_OPENCV ON CACHE BOOL "Build with OpenCV support")
+set(USE_OPENMP OFF CACHE BOOL "Build with Openmp support")
+set(USE_ONEDNN ON CACHE BOOL "Build with ONEDNN support")
+set(USE_LAPACK ON CACHE BOOL "Build with lapack support")
+set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
+set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
+set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
+set(USE_LIBJPEG_TURBO ON CACHE BOOL "Build with libjpeg-turbo")
+set(USE_DIST_KVSTORE ON CACHE BOOL "Build with DIST_KVSTORE support")
diff --git a/config/distribution/linux_cpu_mkl.cmake b/config/distribution/linux_cpu_mkl.cmake
new file mode 100644
index 000000000000..8975e0d8f370
--- /dev/null
+++ b/config/distribution/linux_cpu_mkl.cmake
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set(CMAKE_BUILD_TYPE "Distribution" CACHE STRING "Build type")
+set(CFLAGS "-mno-avx" CACHE STRING "CFLAGS")
+set(CXXFLAGS "-mno-avx" CACHE STRING "CXXFLAGS")
+
+set(USE_BLAS "mkl" CACHE STRING "BLAS Vendor")
+set(USE_CUDA OFF CACHE BOOL "Build with CUDA support")
+set(USE_OPENCV ON CACHE BOOL "Build with OpenCV support")
+set(USE_OPENMP ON CACHE BOOL "Build with Openmp support")
+set(USE_ONEDNN ON CACHE BOOL "Build with ONEDNN support")
+set(USE_LAPACK ON CACHE BOOL "Build with lapack support")
+set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
+set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
+set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
+set(USE_LIBJPEG_TURBO ON CACHE BOOL "Build with libjpeg-turbo")
+set(USE_DIST_KVSTORE ON CACHE BOOL "Build with DIST_KVSTORE support")
diff --git a/src/initialize.cc b/src/initialize.cc
index 21211f3473b9..4de0889d4e53 100644
--- a/src/initialize.cc
+++ b/src/initialize.cc
@@ -175,7 +175,7 @@ void LibraryInitializer::lib_close(void* handle) {
  * \param func function pointer that gets output address
  * \param name function name to be fetched
  */
-void LibraryInitializer::get_sym(void* handle, void** func, char* name) {
+void LibraryInitializer::get_sym(void* handle, void** func, const char* name) {
 #if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)
   *func = GetProcAddress((HMODULE)handle, name);
   if (!(*func)) {
diff --git a/tools/dependencies/make_shared_dependencies.sh b/tools/dependencies/make_shared_dependencies.sh
index 96d3561d446b..4d529930b500 100755
--- a/tools/dependencies/make_shared_dependencies.sh
+++ b/tools/dependencies/make_shared_dependencies.sh
@@ -49,8 +49,8 @@ download () {
     fi
 }
 
-if [[ ! $PLATFORM == 'darwin' ]]; then
-    source ${DIR}/openblas.sh
+if [[ ! $PLATFORM == 'darwin' ]] && [[ ! $BLAS == 'mkl' ]]; then
+        source ${DIR}/openblas.sh
 fi
 source $DIR/libz.sh
 source $DIR/libturbojpeg.sh
@@ -64,6 +64,10 @@ source $DIR/protobuf.sh
 source $DIR/cityhash.sh
 source $DIR/zmq.sh
 source $DIR/lz4.sh
+if [[ $BLAS == 'mkl' ]]; then
+    source ${DIR}/mkl.sh
+fi
+
 
 export LIBRARY_PATH=${LIBRARY_PATH}:$(dirname $(find $DEPS_PATH -type f -name 'libprotoc*' | grep protobuf | head -n 1)):$DEPS_PATH/lib:$DEPS_PATH/lib64
 export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(dirname $(find $DEPS_PATH -type f -name 'libprotoc*' | grep protobuf | head -n 1)):$DEPS_PATH/lib:$DEPS_PATH/lib64
diff --git a/tools/dependencies/mkl.sh b/tools/dependencies/mkl.sh
new file mode 100644
index 000000000000..0564a198ffa3
--- /dev/null
+++ b/tools/dependencies/mkl.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script downloads OneMKL
+# TODO @akarbown: Get more general link to download the OneMKL (instead of the generated one)
+set -ex
+INTEL_MKL="2021.3.0"
+if [[  (! -e /opt/intel/oneapi/mkl/)]]; then
+    >&2 echo "Downloading mkl..."
+
+    if [[ $PLATFORM == 'darwin' ]]; then
+        download \
+            https://registrationcenter-download.intel.com/akdlm/irc_nas/17960/m_onemkl_p_${INTEL_MKL}.517_offline.dmg \
+            ${DEPS_PATH}/m_onemkl_p_${INTEL_MKL}.517_offline.dmg
+        hdiutil attach ${DEPS_PATH}/m_onemkl_p_${INTEL_MKL}.517_offline.dmg
+        pushd /Volumes/m_onemkl_p_${INTEL_MKL}.517_offline/bootstrapper.app/Contents/MacOS/
+        ./install.sh --silent --eula accept
+        popd
+    elif [[ $PLATFORM == 'linux' ]]; then
+        # use wget to fetch the Intel repository public key
+        download \
+            https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
+            ${DEPS_PATH}/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+        # add to your apt sources keyring so that archives signed with this key will be trusted.
+        apt-key add ${DEPS_PATH}/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+        # remove the public key
+        rm ${DEPS_PATH}/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+        add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
+        apt-get update && \
+        apt install -y intel-oneapi-mkl-${INTEL_MKL} intel-oneapi-mkl-common-${INTEL_MKL}  intel-oneapi-mkl-devel
+    else
+        >&2 echo "Not available"
+    fi
+fi
diff --git a/tools/dependencies/opencv.sh b/tools/dependencies/opencv.sh
index fce8c1547757..c0fbe27d7f8f 100755
--- a/tools/dependencies/opencv.sh
+++ b/tools/dependencies/opencv.sh
@@ -22,7 +22,7 @@
 set -ex
 OPENCV_VERSION=3.4.2
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-if [[ $PLATFORM == 'linux' ]]; then
+if [[ $PLATFORM == 'linux' ]] && [[ ! $BLAS == 'mkl' ]]; then
     OPENCV_LAPACK_OPTIONS=" \
           -D OpenBLAS_HOME=$DEPS_PATH \
           -D OpenBLAS_INCLUDE_DIR=$DEPS_PATH/include \
diff --git a/tools/staticbuild/build.sh b/tools/staticbuild/build.sh
index 5ed4ac6d487a..b61084443621 100755
--- a/tools/staticbuild/build.sh
+++ b/tools/staticbuild/build.sh
@@ -18,18 +18,23 @@
 # under the License.
 
 if [ $# -lt 1 ]; then
-    >&2 echo "Usage: build.sh <VARIANT>"
+    >&2 echo "Usage: build.sh <VARIANT> <BLAS>"
 fi
 
 export CURDIR=$PWD
 export DEPS_PATH=$PWD/staticdeps
 export VARIANT=$(echo $1 | tr '[:upper:]' '[:lower:]')
 export PLATFORM=$(uname | tr '[:upper:]' '[:lower:]')
+export BLAS=$(echo $2 | tr '[:upper:]' '[:lower:]')
 
 if [[ $VARIANT == darwin* ]]; then
     export VARIANT="darwin"
 fi
 
+if [[ ! $BLAS ]]; then
+    export BLAS="open"
+fi
+
 NUM_PROC=1
 if [[ ! -z $(command -v nproc) ]]; then
     NUM_PROC=$(nproc)
diff --git a/tools/staticbuild/build_lib.sh b/tools/staticbuild/build_lib.sh
index 429387010f5a..2375fcd0d3d6 100755
--- a/tools/staticbuild/build_lib.sh
+++ b/tools/staticbuild/build_lib.sh
@@ -20,7 +20,11 @@
 set -eo pipefail
 
 # This script builds the libraries of mxnet.
+if [[ ! $BLAS ]] || [[ $BLAS == 'open' ]]; then
 cmake_config=${CURDIR}/config/distribution/${PLATFORM}_${VARIANT}.cmake
+else
+    cmake_config=${CURDIR}/config/distribution/${PLATFORM}_${VARIANT}_${BLAS}.cmake
+fi
 if [[ ! -f $cmake_config ]]; then
     >&2 echo "Couldn't find cmake config $make_config for the current settings."
     exit 1
@@ -36,13 +40,18 @@ cmake -GNinja -C $cmake_config \
       -DCMAKE_OSX_DEPLOYMENT_TARGET=10.13 \
       ..
 ninja
+if [[ $BLAS == 'mkl' ]]; then
+    patchelf --set-rpath "/opt/intel/oneapi/mkl/${INTEL_MKL}/lib/intel64/" --force-rpath libmxnet.so
+fi
 cd -
 
 # Move to lib
 rm -rf lib; mkdir lib;
 if [[ $PLATFORM == 'linux' ]]; then
     cp -L build/libmxnet.so lib/libmxnet.so
-    cp -L $(ldd lib/libmxnet.so | grep libgfortran |  awk '{print $3}') lib/
+    if [[ $BLAS == 'open' ]]; then
+        cp -L $(ldd lib/libmxnet.so | grep libgfortran |  awk '{print $3}') lib/
+    fi
 elif [[ $PLATFORM == 'darwin' ]]; then
     cp -L build/libmxnet.dylib lib/libmxnet.dylib
 fi

From 59db10da3ebf2a7d2ce6984aa2747ceccef3fa22 Mon Sep 17 00:00:00 2001
From: Anna Karbownik <anna.karbownik@intel.com>
Date: Tue, 21 Sep 2021 15:10:08 +0200
Subject: [PATCH 09/16] clang format + mkl workflow rename

---
 .github/workflows/os_x_mklbuild.yml |   2 +-
 src/initialize.cc                   | 177 +++++++++++++---------------
 src/initialize.h                    |   9 +-
 tools/dependencies/mkl.sh           |   2 +-
 4 files changed, 91 insertions(+), 99 deletions(-)

diff --git a/.github/workflows/os_x_mklbuild.yml b/.github/workflows/os_x_mklbuild.yml
index e2aa94c3c744..b76e40b4ec1a 100644
--- a/.github/workflows/os_x_mklbuild.yml
+++ b/.github/workflows/os_x_mklbuild.yml
@@ -1,4 +1,4 @@
-name: continuous build
+name: mkl continuous build
 
 on: [push, pull_request]
 
diff --git a/src/initialize.cc b/src/initialize.cc
index 4de0889d4e53..eaced17aad95 100644
--- a/src/initialize.cc
+++ b/src/initialize.cc
@@ -1,4 +1,4 @@
-  /*
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -22,6 +22,7 @@
  * \brief initialize mxnet library
  */
 #include "initialize.h"
+
 #include <algorithm>
 #include <csignal>
 
@@ -31,17 +32,16 @@
  * \brief Retrieve the system error message for the last-error code
  * \param err string that gets the error message
  */
-void win_err(char **err) {
+void win_err(char** err) {
   uint32_t dw = GetLastError();
   FormatMessage(
-        FORMAT_MESSAGE_ALLOCATE_BUFFER |
-        FORMAT_MESSAGE_FROM_SYSTEM |
-        FORMAT_MESSAGE_IGNORE_INSERTS,
-        nullptr,
-        dw,
-        MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
-        reinterpret_cast<char*>(err),
-        0, nullptr);
+      FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+      nullptr,
+      dw,
+      MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+      reinterpret_cast<char*>(err),
+      0,
+      nullptr);
 }
 #else
 #include <cxxabi.h>
@@ -53,8 +53,9 @@ void win_err(char **err) {
 #endif
 
 #include <dmlc/logging.h>
-#include <mxnet/engine.h>
 #include <mxnet/c_api.h>
+#include <mxnet/engine.h>
+
 #include "./engine/openmp.h"
 #include "./operator/custom/custom-inl.h"
 #if MXNET_USE_OPENCV
@@ -89,13 +90,13 @@ void pthread_atfork_child() {
 // LibraryInitializer member functions
 
 LibraryInitializer::LibraryInitializer()
-  : original_pid_(common::current_process_id()),
-    mp_worker_nthreads_(dmlc::GetEnv("MXNET_MP_WORKER_NTHREADS", 1)),
-    cpu_worker_nthreads_(dmlc::GetEnv("MXNET_CPU_WORKER_NTHREADS", 1)),
-    mp_cv_num_threads_(dmlc::GetEnv("MXNET_MP_OPENCV_NUM_THREADS", 0)) {
+    : original_pid_(common::current_process_id()),
+      mp_worker_nthreads_(dmlc::GetEnv("MXNET_MP_WORKER_NTHREADS", 1)),
+      cpu_worker_nthreads_(dmlc::GetEnv("MXNET_CPU_WORKER_NTHREADS", 1)),
+      mp_cv_num_threads_(dmlc::GetEnv("MXNET_MP_OPENCV_NUM_THREADS", 0)) {
   dmlc::InitLogging("mxnet");
   init_mkl_dynamic_library();
-  engine::OpenMP::Get();   // force OpenMP initialization
+  engine::OpenMP::Get();  // force OpenMP initialization
   install_pthread_atfork_handlers();
 }
 
@@ -111,14 +112,14 @@ bool LibraryInitializer::lib_is_loaded(const std::string& path) const {
  * \return handle a pointer for the loaded library, throws dmlc::error if library can't be loaded
  */
 void* LibraryInitializer::lib_load(const char* path) {
-  void *handle = nullptr;
+  void* handle = nullptr;
   // check if library was already loaded
   if (!lib_is_loaded(path)) {
     // if not, load it
 #if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)
     handle = LoadLibrary(path);
     if (!handle) {
-      char *err_msg = nullptr;
+      char* err_msg = nullptr;
       win_err(&err_msg);
       LOG(FATAL) << "Error loading library: '" << path << "'\n" << err_msg;
       LocalFree(err_msg);
@@ -163,7 +164,7 @@ void LibraryInitializer::lib_close(void* handle) {
 #else
   if (dlclose(handle)) {
     LOG(WARNING) << "LibraryInitializer::lib_close: couldn't close library at address: " << handle
-        << " loaded from: '" << libpath << "': " << dlerror();
+                 << " loaded from: '" << libpath << "': " << dlerror();
   }
 #endif  // _WIN32 or _WIN64 or __WINDOWS__
   loaded_libs.erase(libpath);
@@ -179,7 +180,7 @@ void LibraryInitializer::get_sym(void* handle, void** func, const char* name) {
 #if defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__)
   *func = GetProcAddress((HMODULE)handle, name);
   if (!(*func)) {
-    char *err_msg = nullptr;
+    char* err_msg = nullptr;
     win_err(&err_msg);
     LOG(FATAL) << "Error getting function '" << name << "' from library\n" << err_msg;
     LocalFree(err_msg);
@@ -222,7 +223,6 @@ void LibraryInitializer::atfork_child() {
   CustomOperator::Get()->Start();
 }
 
-
 void LibraryInitializer::install_pthread_atfork_handlers() {
 #ifndef _WIN32
   engine::OpenMP::Get()->initialize_process();  // force omp to set its atfork handler first
@@ -232,36 +232,32 @@ void LibraryInitializer::install_pthread_atfork_handlers() {
 
 void LibraryInitializer::init_mkl_dynamic_library() {
 #if !(defined(_WIN32) || defined(_WIN64) || defined(__WINDOWS__))
-  #if MKL_USE_SINGLE_DYNAMIC_LIBRARY
-    #if USE_INT64_TENSOR_SIZE
-      int interface = MKL_INTERFACE_ILP64;
-    #else
-      int interface = MKL_INTERFACE_LP64;
-    #endif
-    #if defined( __INTEL_LLVM_COMPILER)
-      mkl_set_threading_layer(MKL_THREADING_INTEL);
-    #else
-      mkl_set_threading_layer(MKL_THREADING_GNU);
-      interface += MKL_INTERFACE_GNU;
-    #endif
-    mkl_set_interface_layer(interface);
-  #endif
+#if MKL_USE_SINGLE_DYNAMIC_LIBRARY
+#if USE_INT64_TENSOR_SIZE
+  int interface = MKL_INTERFACE_ILP64;
+#else
+  int interface = MKL_INTERFACE_LP64;
+#endif
+#if defined(__INTEL_LLVM_COMPILER)
+  mkl_set_threading_layer(MKL_THREADING_INTEL);
+#else
+  mkl_set_threading_layer(MKL_THREADING_GNU);
+  interface += MKL_INTERFACE_GNU;
+#endif
+  mkl_set_interface_layer(interface);
+#endif
 #endif
 }
 
-
-
 #if MXNET_USE_SIGNAL_HANDLER && DMLC_LOG_STACK_TRACE
 
-static inline void printStackTrace(FILE *out = stderr,
-    const unsigned int max_frames = 63) {
-
+static inline void printStackTrace(FILE* out = stderr, const unsigned int max_frames = 63) {
 #if !defined(_WIN32) && !defined(_WIN64) && !defined(__WINDOWS__)
   // storage array for stack trace address data
-  void* addrlist[max_frames+1];
+  void* addrlist[max_frames + 1];
 
   // retrieve current stack addresses
-  size_t addrlen = backtrace(addrlist, sizeof(addrlist)/sizeof(void*));
+  size_t addrlen = backtrace(addrlist, sizeof(addrlist) / sizeof(void*));
 
   if (addrlen < 5) {
     return;
@@ -270,7 +266,6 @@ static inline void printStackTrace(FILE *out = stderr,
   }
   fprintf(out, "Stack trace:\n");
 
-
   // resolve addresses into strings containing "filename(function+address)",
   // Actually it will be ## program address function + offset
   // this array must be free()-ed
@@ -281,7 +276,7 @@ static inline void printStackTrace(FILE *out = stderr,
 
   // iterate over the returned symbol lines. skip the first, it is the
   // address of this function.
-  for (unsigned int i = 4; i < addrlen ; i++) {
+  for (unsigned int i = 4; i < addrlen; i++) {
     char* begin_name   = nullptr;
     char* begin_offset = nullptr;
     char* end_offset   = nullptr;
@@ -289,40 +284,37 @@ static inline void printStackTrace(FILE *out = stderr,
     // find parentheses and +address offset surrounding the mangled name
 #ifdef DARWIN
     // OSX style stack trace
-    for (char *p = symbollist[i]; *p; ++p) {
-      if (*p == '_' && *(p-1) == ' ') {
-        begin_name = p-1;
+    for (char* p = symbollist[i]; *p; ++p) {
+      if (*p == '_' && *(p - 1) == ' ') {
+        begin_name = p - 1;
       } else if (*p == '+') {
-        begin_offset = p-1;
+        begin_offset = p - 1;
       }
     }
 
     if (begin_name && begin_offset && begin_name < begin_offset) {
-      *begin_name++ = '\0';
+      *begin_name++   = '\0';
       *begin_offset++ = '\0';
 
       // mangled name is now in [begin_name, begin_offset) and caller
       // offset in [begin_offset, end_offset). now apply
       // __cxa_demangle():
       int status;
-      char* ret = abi::__cxa_demangle(begin_name, &funcname[0],
-                                      &funcnamesize, &status);
+      char* ret = abi::__cxa_demangle(begin_name, &funcname[0], &funcnamesize, &status);
       if (status == 0) {
         funcname = ret;  // use possibly realloc()-ed string
-        fprintf(out, "  %-30s %-40s %s\n",
-                symbollist[i], funcname, begin_offset);
+        fprintf(out, "  %-30s %-40s %s\n", symbollist[i], funcname, begin_offset);
       } else {
         // demangling failed. Output function name as a C function with
         // no arguments.
-        fprintf(out, "  %-30s %-38s() %s\n",
-                symbollist[i], begin_name, begin_offset);
+        fprintf(out, "  %-30s %-38s() %s\n", symbollist[i], begin_name, begin_offset);
       }
     } else {
-       // couldn't parse the line? print the whole line.
-       fprintf(out, "  %-40s\n", symbollist[i]);
+      // couldn't parse the line? print the whole line.
+      fprintf(out, "  %-40s\n", symbollist[i]);
     }
 #else
-    for (char *p = symbollist[i]; *p; ++p) {
+    for (char* p = symbollist[i]; *p; ++p) {
       if (*p == '(') {
         begin_name = p;
       } else if (*p == '+') {
@@ -343,24 +335,22 @@ static inline void printStackTrace(FILE *out = stderr,
       // offset in [begin_offset, end_offset). now apply
       // __cxa_demangle():
 
-      int status = 0;
-      char* ret = abi::__cxa_demangle(begin_name, funcname,
-                                      &funcnamesize, &status);
+      int status  = 0;
+      char* ret   = abi::__cxa_demangle(begin_name, funcname, &funcnamesize, &status);
       char* fname = begin_name;
       if (status == 0) {
         fname = ret;
       }
 
       if (begin_offset) {
-        fprintf(out, "  %-30s ( %-40s  + %-6s) %s\n",
-                symbollist[i], fname, begin_offset, end_offset);
+        fprintf(
+            out, "  %-30s ( %-40s  + %-6s) %s\n", symbollist[i], fname, begin_offset, end_offset);
       } else {
-        fprintf(out, "  %-30s ( %-40s    %-6s) %s\n",
-                symbollist[i], fname, "", end_offset);
+        fprintf(out, "  %-30s ( %-40s    %-6s) %s\n", symbollist[i], fname, "", end_offset);
       }
     } else {
-       // couldn't parse the line? print the whole line.
-       fprintf(out, "  %-40s\n", symbollist[i]);
+      // couldn't parse the line? print the whole line.
+      fprintf(out, "  %-40s\n", symbollist[i]);
     }
 #endif  // !DARWIN - but is posix
   }
@@ -368,32 +358,33 @@ static inline void printStackTrace(FILE *out = stderr,
 #endif
 }
 
-#define SIGNAL_HANDLER(SIGNAL, HANDLER_NAME, IS_FATAL)               \
-std::shared_ptr<void(int)> HANDLER_NAME(                             \
-  signal(SIGNAL, [](int signum) {                                    \
-    if (IS_FATAL) {                                                  \
-      printf("\nFatal Error: %s\n", strsignal(SIGNAL));              \
-      printStackTrace();                                             \
-      signal(signum, SIG_DFL);                                       \
-      raise(signum);                                                 \
-    } else {                                                         \
-      switch (signum) {                                              \
-        case SIGSEGV:                                                \
-          LOG(FATAL) << "InternalError: " << strsignal(SIGNAL);      \
-          break;                                                     \
-        case SIGFPE:                                                 \
-          LOG(FATAL) << "FloatingPointError: " << strsignal(SIGNAL); \
-          break;                                                     \
-        case SIGBUS:                                                 \
-          LOG(FATAL) << "IOError: " << strsignal(SIGNAL);            \
-          break;                                                     \
-        default:                                                     \
-          LOG(FATAL) << "RuntimeError: " << strsignal(SIGNAL);       \
-          break;                                                     \
-      }                                                              \
-    }                                                                \
-  }),                                                                \
-  [](auto f) { signal(SIGNAL, f); });
+#define SIGNAL_HANDLER(SIGNAL, HANDLER_NAME, IS_FATAL)                          \
+  std::shared_ptr<void(int)> HANDLER_NAME(                                      \
+      signal(SIGNAL,                                                            \
+             [](int signum) {                                                   \
+               if (IS_FATAL) {                                                  \
+                 printf("\nFatal Error: %s\n", strsignal(SIGNAL));              \
+                 printStackTrace();                                             \
+                 signal(signum, SIG_DFL);                                       \
+                 raise(signum);                                                 \
+               } else {                                                         \
+                 switch (signum) {                                              \
+                   case SIGSEGV:                                                \
+                     LOG(FATAL) << "InternalError: " << strsignal(SIGNAL);      \
+                     break;                                                     \
+                   case SIGFPE:                                                 \
+                     LOG(FATAL) << "FloatingPointError: " << strsignal(SIGNAL); \
+                     break;                                                     \
+                   case SIGBUS:                                                 \
+                     LOG(FATAL) << "IOError: " << strsignal(SIGNAL);            \
+                     break;                                                     \
+                   default:                                                     \
+                     LOG(FATAL) << "RuntimeError: " << strsignal(SIGNAL);       \
+                     break;                                                     \
+                 }                                                              \
+               }                                                                \
+             }),                                                                \
+      [](auto f) { signal(SIGNAL, f); });
 
 SIGNAL_HANDLER(SIGSEGV, SIGSEGVHandler, true);
 SIGNAL_HANDLER(SIGFPE, SIGFPEHandler, false);
diff --git a/src/initialize.h b/src/initialize.h
index 489211c70111..5a1062afc4a6 100644
--- a/src/initialize.h
+++ b/src/initialize.h
@@ -23,8 +23,9 @@
  */
 
 #include <cstdlib>
-#include <string>
 #include <map>
+#include <string>
+
 #include "dmlc/io.h"
 
 #ifndef MXNET_INITIALIZE_H_
@@ -92,9 +93,9 @@ class LibraryInitializer {
   void install_pthread_atfork_handlers();
 
   /**
-  * Sets the interface and threading layer for Intel® oneAPI MKL at run time.
-  * Use with the Single Dynamic Library.
-  */
+   * Sets the interface and threading layer for Intel® oneAPI MKL at run time.
+   * Use with the Single Dynamic Library.
+   */
   void init_mkl_dynamic_library();
   /**
    * Install signal handlers (UNIX). Has no effect on Windows.
diff --git a/tools/dependencies/mkl.sh b/tools/dependencies/mkl.sh
index 0564a198ffa3..f682ef8c00c3 100644
--- a/tools/dependencies/mkl.sh
+++ b/tools/dependencies/mkl.sh
@@ -30,7 +30,7 @@ if [[  (! -e /opt/intel/oneapi/mkl/)]]; then
             ${DEPS_PATH}/m_onemkl_p_${INTEL_MKL}.517_offline.dmg
         hdiutil attach ${DEPS_PATH}/m_onemkl_p_${INTEL_MKL}.517_offline.dmg
         pushd /Volumes/m_onemkl_p_${INTEL_MKL}.517_offline/bootstrapper.app/Contents/MacOS/
-        ./install.sh --silent --eula accept
+        sudo ./install.sh --silent --eula accept
         popd
     elif [[ $PLATFORM == 'linux' ]]; then
         # use wget to fetch the Intel repository public key

From 32aa4cf96f264f72511e640879497afabbb244d3 Mon Sep 17 00:00:00 2001
From: Anna Karbownik <anna.karbownik@intel.com>
Date: Tue, 21 Sep 2021 17:48:22 +0200
Subject: [PATCH 10/16] Fixing some formatting + installing patchelf

---
 .github/workflows/os_x_mklbuild.yml            | 2 +-
 tools/dependencies/make_shared_dependencies.sh | 2 +-
 tools/staticbuild/build_lib.sh                 | 8 +++++---
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/os_x_mklbuild.yml b/.github/workflows/os_x_mklbuild.yml
index b76e40b4ec1a..6bae8ea07b15 100644
--- a/.github/workflows/os_x_mklbuild.yml
+++ b/.github/workflows/os_x_mklbuild.yml
@@ -29,7 +29,7 @@ jobs:
 
       - name: Install Dependencies
         run: |
-          brew install nasm automake ninja libtool cmake pkgconfig protobuf hdf5 zlib ccache
+          brew install nasm automake ninja libtool cmake pkgconfig protobuf hdf5 zlib ccache patchelf
           ccache -M 500M  # Limit the ccache size; Github's overall cache limit is 5GB
           python -m pip install -r ci/docker/install/requirements
         shell: bash
diff --git a/tools/dependencies/make_shared_dependencies.sh b/tools/dependencies/make_shared_dependencies.sh
index 4d529930b500..684548aca983 100755
--- a/tools/dependencies/make_shared_dependencies.sh
+++ b/tools/dependencies/make_shared_dependencies.sh
@@ -50,7 +50,7 @@ download () {
 }
 
 if [[ ! $PLATFORM == 'darwin' ]] && [[ ! $BLAS == 'mkl' ]]; then
-        source ${DIR}/openblas.sh
+    source ${DIR}/openblas.sh
 fi
 source $DIR/libz.sh
 source $DIR/libturbojpeg.sh
diff --git a/tools/staticbuild/build_lib.sh b/tools/staticbuild/build_lib.sh
index 2375fcd0d3d6..7a0ae4cc51ac 100755
--- a/tools/staticbuild/build_lib.sh
+++ b/tools/staticbuild/build_lib.sh
@@ -21,7 +21,7 @@ set -eo pipefail
 
 # This script builds the libraries of mxnet.
 if [[ ! $BLAS ]] || [[ $BLAS == 'open' ]]; then
-cmake_config=${CURDIR}/config/distribution/${PLATFORM}_${VARIANT}.cmake
+    cmake_config=${CURDIR}/config/distribution/${PLATFORM}_${VARIANT}.cmake
 else
     cmake_config=${CURDIR}/config/distribution/${PLATFORM}_${VARIANT}_${BLAS}.cmake
 fi
@@ -40,8 +40,10 @@ cmake -GNinja -C $cmake_config \
       -DCMAKE_OSX_DEPLOYMENT_TARGET=10.13 \
       ..
 ninja
-if [[ $BLAS == 'mkl' ]]; then
-    patchelf --set-rpath "/opt/intel/oneapi/mkl/${INTEL_MKL}/lib/intel64/" --force-rpath libmxnet.so
+if [[ ! $PLATFORM == 'darwin' ]] && [[ $BLAS == 'mkl' ]]; then
+    patchelf --set-rpath "/opt/intel/oneapi/mkl/${INTEL_MKL}/lib/intel64/:\$ORIGIN" --force-rpath libmxnet.so
+elif [[ $PLATFORM == 'darwin' ]] && [[ $BLAS == 'mkl' ]]; then
+    patchelf --set-rpath "/opt/intel/oneapi/mkl/${INTEL_MKL}/lib/intel64/:\$ORIGIN" --force-rpath libmxnet.dylib
 fi
 cd -
 

From dde35e4d3ae1527f0143f8ce35ebb0105be5e5a8 Mon Sep 17 00:00:00 2001
From: Anna Karbownik <anna.karbownik@intel.com>
Date: Wed, 22 Sep 2021 10:55:36 +0200
Subject: [PATCH 11/16] setting up Mac OS rpath for MKL libraries

---
 tools/staticbuild/build_lib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/staticbuild/build_lib.sh b/tools/staticbuild/build_lib.sh
index 7a0ae4cc51ac..beef339f6d59 100755
--- a/tools/staticbuild/build_lib.sh
+++ b/tools/staticbuild/build_lib.sh
@@ -43,7 +43,7 @@ ninja
 if [[ ! $PLATFORM == 'darwin' ]] && [[ $BLAS == 'mkl' ]]; then
     patchelf --set-rpath "/opt/intel/oneapi/mkl/${INTEL_MKL}/lib/intel64/:\$ORIGIN" --force-rpath libmxnet.so
 elif [[ $PLATFORM == 'darwin' ]] && [[ $BLAS == 'mkl' ]]; then
-    patchelf --set-rpath "/opt/intel/oneapi/mkl/${INTEL_MKL}/lib/intel64/:\$ORIGIN" --force-rpath libmxnet.dylib
+    install_name_tool -add_rpath "/opt/intel/oneapi/mkl/${INTEL_MKL}/lib/:\$ORIGIN" libmxnet.dylib
 fi
 cd -
 

From 4b80278d9e285a3b29b5d6ec112764006d27946e Mon Sep 17 00:00:00 2001
From: Anna Karbownik <anna.karbownik@intel.com>
Date: Thu, 23 Sep 2021 17:04:59 +0200
Subject: [PATCH 12/16] Run only mkl tests

---
 .github/workflows/os_x_mklbuild.yml | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/os_x_mklbuild.yml b/.github/workflows/os_x_mklbuild.yml
index 6bae8ea07b15..7bd460d621a1 100644
--- a/.github/workflows/os_x_mklbuild.yml
+++ b/.github/workflows/os_x_mklbuild.yml
@@ -29,7 +29,7 @@ jobs:
 
       - name: Install Dependencies
         run: |
-          brew install nasm automake ninja libtool cmake pkgconfig protobuf hdf5 zlib ccache patchelf
+          brew install nasm automake ninja libtool cmake pkgconfig protobuf hdf5 zlib ccache
           ccache -M 500M  # Limit the ccache size; Github's overall cache limit is 5GB
           python -m pip install -r ci/docker/install/requirements
         shell: bash
@@ -42,16 +42,9 @@ jobs:
         run: |
           python -m pip install --user -e python
 
-      - name: Build with Cython
-        run: |
-          cd python
-          python setup.py build_ext --inplace --with-cython
-
       - name: Test project
-        env:
-          MXNET_ENFORCE_CYTHON: 1
         run: |
-          python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'not test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
-          MXNET_ENGINE_TYPE=NaiveEngine python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
-          python3 -m pytest --durations=50 --verbose tests/python/unittest/ -k 'not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'serial'
+          # python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'not test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
+          # MXNET_ENGINE_TYPE=NaiveEngine python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
+          # python3 -m pytest --durations=50 --verbose tests/python/unittest/ -k 'not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'serial'
           python3 -m pytest -n 4 --durations=50 --verbose tests/python/mkl

From 30f90b53d4356f249d685a57340c67e4bdd695d3 Mon Sep 17 00:00:00 2001
From: Anna Karbownik <anna.karbownik@intel.com>
Date: Fri, 24 Sep 2021 19:00:47 +0200
Subject: [PATCH 13/16] Fix for finding MKL libraries on MacOs by
 FindBLAS.cmake Turn off SDL for MKL on MacOS as it need fixes.

---
 cmake/ChooseBlas.cmake         | 9 ++++++++-
 cmake/upstream/FindBLAS.cmake  | 5 ++++-
 tools/staticbuild/build_lib.sh | 2 +-
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/cmake/ChooseBlas.cmake b/cmake/ChooseBlas.cmake
index ed6c7337acd4..be41c7d0e01d 100644
--- a/cmake/ChooseBlas.cmake
+++ b/cmake/ChooseBlas.cmake
@@ -126,13 +126,20 @@ elseif(BLAS STREQUAL "MKL" OR BLAS STREQUAL "mkl")
   # ---[ MKL Options
   file(STRINGS ${MKL_INCLUDE_DIR}/mkl_version.h MKL_VERSION_DEF REGEX "INTEL_MKL_VERSION")
   string(REGEX MATCH "([0-9]+)" MKL_VERSION ${MKL_VERSION_DEF})
-  if(UNIX)
+  if(UNIX AND NOT APPLE)
     # Single dynamic library interface leads to conflicts between intel omp and llvm omp
     # https://github.com/apache/incubator-mxnet/issues/17641
     # Fixed in oneMKL 2021.3: [MKLD-11109] MKL is opening libgomp.so instead of
     # libgomp.so.1 while SDL=1 & MKL_THREADING_LAYER=GNU
     cmake_dependent_option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" ON
       "NOT BLA_STATIC;MKL_VERSION GREATER_EQUAL 20210003" OFF)
+  elseif(APPLE)
+    # TODO: Do more research on MKL on MacOs as there seems to be some library issues.
+    # As a consequence link static MKL libraries.
+    # TODO: Single dynamic library (SDL) interface for MacOS seems to have:
+    # 'INTEL MKL ERROR: dlopen(/opt/intel/oneapi/mkl/2021.3.0/lib/libmkl_cdft_core.1.dylib,
+    # 9):Symbol not found: _mkl_dft_dfticomputebackward' issue.
+    option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" OFF)
   else()
     option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" ON)
   endif()
diff --git a/cmake/upstream/FindBLAS.cmake b/cmake/upstream/FindBLAS.cmake
index b22a34dd4184..6f00955cab9f 100644
--- a/cmake/upstream/FindBLAS.cmake
+++ b/cmake/upstream/FindBLAS.cmake
@@ -320,7 +320,10 @@ if(BLA_VENDOR MATCHES "Intel" OR BLA_VENDOR STREQUAL "All")
           set(BLAS_mkl_DLL_SUFFIX "_dll")
         endif()
       else()
-        if(BLA_STATIC)
+        # MXNET NOTE: The second 2 lines differs from CMake source by ${CMAKE_CURRENT_LIST_DIR}
+        # replaced with ${CMAKE_ROOT}/Modules
+        # https://gitlab.kitware.com/cmake/cmake/-/issues/20548
+        if(BLA_STATIC AND NOT APPLE)
           set(BLAS_mkl_START_GROUP "-Wl,--start-group")
           set(BLAS_mkl_END_GROUP "-Wl,--end-group")
         else()
diff --git a/tools/staticbuild/build_lib.sh b/tools/staticbuild/build_lib.sh
index beef339f6d59..8ae16b415df6 100755
--- a/tools/staticbuild/build_lib.sh
+++ b/tools/staticbuild/build_lib.sh
@@ -43,7 +43,7 @@ ninja
 if [[ ! $PLATFORM == 'darwin' ]] && [[ $BLAS == 'mkl' ]]; then
     patchelf --set-rpath "/opt/intel/oneapi/mkl/${INTEL_MKL}/lib/intel64/:\$ORIGIN" --force-rpath libmxnet.so
 elif [[ $PLATFORM == 'darwin' ]] && [[ $BLAS == 'mkl' ]]; then
-    install_name_tool -add_rpath "/opt/intel/oneapi/mkl/${INTEL_MKL}/lib/:\$ORIGIN" libmxnet.dylib
+    install_name_tool -add_rpath "/opt/intel/oneapi/mkl/${INTEL_MKL}/lib/:/opt/intel/oneapi/compiler/${INTEL_MKL}/mac/compiler/lib/:\$ORIGIN" libmxnet.dylib
 fi
 cd -
 

From 51957929de2958e10b6ae0d97ab87f104e113443 Mon Sep 17 00:00:00 2001
From: Anna Karbownik <anna.karbownik@intel.com>
Date: Thu, 30 Sep 2021 14:43:06 +0200
Subject: [PATCH 14/16] Enable linking MxNET with MKL static libraries on MacOS

Add proper mkl_threading flags for Mac Os.
Enable all tests that are for MacOS + MKL tests.
Rebuild numpy with MKL BLAS (instead of OpenBLAS).
---
 .github/workflows/os_x_mklbuild.yml           |  6 +--
 cmake/ChooseBlas.cmake                        |  9 +---
 config/distribution/darwin_cpu_mkl.cmake      |  1 +
 config/distribution/linux_cpu_mkl.cmake       |  2 +
 src/initialize.cc                             |  2 +-
 .../dependencies/make_shared_dependencies.sh  |  5 +++
 tools/dependencies/mkl.sh                     |  6 +--
 tools/dependencies/numpy_mkl.sh               | 45 +++++++++++++++++++
 tools/dependencies/opencv.sh                  |  2 +-
 tools/staticbuild/build_lib.sh                |  2 -
 10 files changed, 62 insertions(+), 18 deletions(-)
 mode change 100644 => 100755 tools/dependencies/mkl.sh
 create mode 100755 tools/dependencies/numpy_mkl.sh

diff --git a/.github/workflows/os_x_mklbuild.yml b/.github/workflows/os_x_mklbuild.yml
index 7bd460d621a1..eb1e6bdd2fd6 100644
--- a/.github/workflows/os_x_mklbuild.yml
+++ b/.github/workflows/os_x_mklbuild.yml
@@ -44,7 +44,7 @@ jobs:
 
       - name: Test project
         run: |
-          # python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'not test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
-          # MXNET_ENGINE_TYPE=NaiveEngine python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
-          # python3 -m pytest --durations=50 --verbose tests/python/unittest/ -k 'not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'serial'
+          python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'not test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
+          MXNET_ENGINE_TYPE=NaiveEngine python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
+          python3 -m pytest --durations=50 --verbose tests/python/unittest/ -k 'not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'serial'
           python3 -m pytest -n 4 --durations=50 --verbose tests/python/mkl
diff --git a/cmake/ChooseBlas.cmake b/cmake/ChooseBlas.cmake
index be41c7d0e01d..ed6c7337acd4 100644
--- a/cmake/ChooseBlas.cmake
+++ b/cmake/ChooseBlas.cmake
@@ -126,20 +126,13 @@ elseif(BLAS STREQUAL "MKL" OR BLAS STREQUAL "mkl")
   # ---[ MKL Options
   file(STRINGS ${MKL_INCLUDE_DIR}/mkl_version.h MKL_VERSION_DEF REGEX "INTEL_MKL_VERSION")
   string(REGEX MATCH "([0-9]+)" MKL_VERSION ${MKL_VERSION_DEF})
-  if(UNIX AND NOT APPLE)
+  if(UNIX)
     # Single dynamic library interface leads to conflicts between intel omp and llvm omp
     # https://github.com/apache/incubator-mxnet/issues/17641
     # Fixed in oneMKL 2021.3: [MKLD-11109] MKL is opening libgomp.so instead of
     # libgomp.so.1 while SDL=1 & MKL_THREADING_LAYER=GNU
     cmake_dependent_option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" ON
       "NOT BLA_STATIC;MKL_VERSION GREATER_EQUAL 20210003" OFF)
-  elseif(APPLE)
-    # TODO: Do more research on MKL on MacOs as there seems to be some library issues.
-    # As a consequence link static MKL libraries.
-    # TODO: Single dynamic library (SDL) interface for MacOS seems to have:
-    # 'INTEL MKL ERROR: dlopen(/opt/intel/oneapi/mkl/2021.3.0/lib/libmkl_cdft_core.1.dylib,
-    # 9):Symbol not found: _mkl_dft_dfticomputebackward' issue.
-    option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" OFF)
   else()
     option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" ON)
   endif()
diff --git a/config/distribution/darwin_cpu_mkl.cmake b/config/distribution/darwin_cpu_mkl.cmake
index 37aa4c36f78a..f4e54a878957 100644
--- a/config/distribution/darwin_cpu_mkl.cmake
+++ b/config/distribution/darwin_cpu_mkl.cmake
@@ -20,6 +20,7 @@ set(CFLAGS "-mno-avx" CACHE STRING "CFLAGS")
 set(CXXFLAGS "-mno-avx" CACHE STRING "CXXFLAGS")
 
 set(USE_BLAS "mkl" CACHE STRING "BLAS Vendor")
+set(BLA_STATIC ON CACHE BOOL "Use static libraries")
 
 set(USE_CUDA OFF CACHE BOOL "Build with CUDA support")
 set(USE_OPENCV ON CACHE BOOL "Build with OpenCV support")
diff --git a/config/distribution/linux_cpu_mkl.cmake b/config/distribution/linux_cpu_mkl.cmake
index 8975e0d8f370..3f8dcfc129f4 100644
--- a/config/distribution/linux_cpu_mkl.cmake
+++ b/config/distribution/linux_cpu_mkl.cmake
@@ -20,6 +20,8 @@ set(CFLAGS "-mno-avx" CACHE STRING "CFLAGS")
 set(CXXFLAGS "-mno-avx" CACHE STRING "CXXFLAGS")
 
 set(USE_BLAS "mkl" CACHE STRING "BLAS Vendor")
+set(BLA_STATIC ON CACHE BOOL "Use static libraries")
+
 set(USE_CUDA OFF CACHE BOOL "Build with CUDA support")
 set(USE_OPENCV ON CACHE BOOL "Build with OpenCV support")
 set(USE_OPENMP ON CACHE BOOL "Build with Openmp support")
diff --git a/src/initialize.cc b/src/initialize.cc
index eaced17aad95..5cb5b63918ab 100644
--- a/src/initialize.cc
+++ b/src/initialize.cc
@@ -238,7 +238,7 @@ void LibraryInitializer::init_mkl_dynamic_library() {
 #else
   int interface = MKL_INTERFACE_LP64;
 #endif
-#if defined(__INTEL_LLVM_COMPILER)
+#if defined(__INTEL_LLVM_COMPILER) || defined(__APPLE__)
   mkl_set_threading_layer(MKL_THREADING_INTEL);
 #else
   mkl_set_threading_layer(MKL_THREADING_GNU);
diff --git a/tools/dependencies/make_shared_dependencies.sh b/tools/dependencies/make_shared_dependencies.sh
index 684548aca983..ac05aa914d38 100755
--- a/tools/dependencies/make_shared_dependencies.sh
+++ b/tools/dependencies/make_shared_dependencies.sh
@@ -66,6 +66,11 @@ source $DIR/zmq.sh
 source $DIR/lz4.sh
 if [[ $BLAS == 'mkl' ]]; then
     source ${DIR}/mkl.sh
+    source ${DIR}/numpy_mkl.sh
+    if [[ $PLATFORM == 'darwin' ]]; then
+        # export this path to find iomp5 needed by MKL according to Intel Link Line Advisor
+        export DYLD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/oneapi/compiler/${INTEL_MKL}/mac/compiler
+    fi
 fi
 
 
diff --git a/tools/dependencies/mkl.sh b/tools/dependencies/mkl.sh
old mode 100644
new mode 100755
index f682ef8c00c3..6522ccf2fdcc
--- a/tools/dependencies/mkl.sh
+++ b/tools/dependencies/mkl.sh
@@ -20,8 +20,8 @@
 # This script downloads OneMKL
 # TODO @akarbown: Get more general link to download the OneMKL (instead of the generated one)
 set -ex
-INTEL_MKL="2021.3.0"
-if [[  (! -e /opt/intel/oneapi/mkl/)]]; then
+export INTEL_MKL="2021.3.0"
+if [[  (! -e /opt/intel/oneapi/mkl/) ]]; then
     >&2 echo "Downloading mkl..."
 
     if [[ $PLATFORM == 'darwin' ]]; then
@@ -43,7 +43,7 @@ if [[  (! -e /opt/intel/oneapi/mkl/)]]; then
         rm ${DEPS_PATH}/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
         add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
         apt-get update && \
-        apt install -y intel-oneapi-mkl-${INTEL_MKL} intel-oneapi-mkl-common-${INTEL_MKL}  intel-oneapi-mkl-devel
+        apt install -y intel-oneapi-mkl-${INTEL_MKL} intel-oneapi-mkl-common-${INTEL_MKL} intel-oneapi-mkl-devel-${INTEL_MKL}
     else
         >&2 echo "Not available"
     fi
diff --git a/tools/dependencies/numpy_mkl.sh b/tools/dependencies/numpy_mkl.sh
new file mode 100755
index 000000000000..5a2112493b5c
--- /dev/null
+++ b/tools/dependencies/numpy_mkl.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script rebuilds numpy so that it will use MKL BLAS instead of OpenBLAS.
+set -x
+
+# check if numpy uses openblas
+set +e
+python3 -c "from numpy import show_config; show_config()" | grep 'openblas_info'
+if [[ $? -eq 0 ]] && [[ -e /opt/intel/oneapi/mkl/ ]] && [[ ! -e ~/.numpy-site.cfg ]]; then
+  # create file and add to it MKL configuration
+  if [[ $PLATFORM == 'darwin' ]]; then
+    echo "[mkl]
+  library_dirs = /opt/intel/oneapi/compiler/$INTEL_MKL/mac/compiler/lib:/opt/intel/oneapi/mkl/$INTEL_MKL/lib
+  include_dirs = /opt/intel/oneapi/mkl/$INTEL_MKL/include
+  libraries = mkl_rt,iomp5
+  extra_link_args = -Wl,-rpath,/opt/intel/oneapi/mkl/$INTEL_MKL/lib,-rpath,/opt/intel/oneapi/compiler/$INTEL_MKL/mac/compiler/lib" >> ~/.numpy-site.cfg
+  else
+    echo "[mkl]
+  library_dirs = /opt/intel/oneapi/compiler/$INTEL_MKL/linux/compiler/lib/intel64_lin:/opt/intel/oneapi/mkl/$INTEL_MKL/lib/intel64
+  include_dirs = /opt/intel/oneapi/mkl/$INTEL_MKL/include
+  libraries = mkl_rt,iomp5
+  extra_link_args = -Wl,-rpath,/opt/intel/oneapi/mkl/$INTEL_MKL/lib/intel64,-rpath,/opt/intel/oneapi/compiler/$INTEL_MKL/linux/compiler/lib/intel64_lin" >> ~/.numpy-site.cfg
+  fi
+
+  # reinstall numpy to use MKL BLAS
+  pip3 install numpy==1.19.5 --no-binary numpy --force-reinstall --no-cache-dir
+fi
+set -e
\ No newline at end of file
diff --git a/tools/dependencies/opencv.sh b/tools/dependencies/opencv.sh
index c0fbe27d7f8f..17d594bad2bd 100755
--- a/tools/dependencies/opencv.sh
+++ b/tools/dependencies/opencv.sh
@@ -22,7 +22,7 @@
 set -ex
 OPENCV_VERSION=3.4.2
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-if [[ $PLATFORM == 'linux' ]] && [[ ! $BLAS == 'mkl' ]]; then
+if [[ $PLATFORM == 'linux' ]] && [[ $BLAS == 'open' ]]; then
     OPENCV_LAPACK_OPTIONS=" \
           -D OpenBLAS_HOME=$DEPS_PATH \
           -D OpenBLAS_INCLUDE_DIR=$DEPS_PATH/include \
diff --git a/tools/staticbuild/build_lib.sh b/tools/staticbuild/build_lib.sh
index 8ae16b415df6..e7d675d11eb0 100755
--- a/tools/staticbuild/build_lib.sh
+++ b/tools/staticbuild/build_lib.sh
@@ -42,8 +42,6 @@ cmake -GNinja -C $cmake_config \
 ninja
 if [[ ! $PLATFORM == 'darwin' ]] && [[ $BLAS == 'mkl' ]]; then
     patchelf --set-rpath "/opt/intel/oneapi/mkl/${INTEL_MKL}/lib/intel64/:\$ORIGIN" --force-rpath libmxnet.so
-elif [[ $PLATFORM == 'darwin' ]] && [[ $BLAS == 'mkl' ]]; then
-    install_name_tool -add_rpath "/opt/intel/oneapi/mkl/${INTEL_MKL}/lib/:/opt/intel/oneapi/compiler/${INTEL_MKL}/mac/compiler/lib/:\$ORIGIN" libmxnet.dylib
 fi
 cd -
 

From ca1bcbe78d3fbeff9c909f82d1b2490985078a2c Mon Sep 17 00:00:00 2001
From: Anna Karbownik <anna.karbownik@intel.com>
Date: Thu, 7 Oct 2021 11:32:04 +0200
Subject: [PATCH 15/16] Excluding MKL bf16 tests as CI MacOs machines seems not
 to have avx512 support.

---
 .github/workflows/os_x_mklbuild.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/os_x_mklbuild.yml b/.github/workflows/os_x_mklbuild.yml
index eb1e6bdd2fd6..b44ca00ea82c 100644
--- a/.github/workflows/os_x_mklbuild.yml
+++ b/.github/workflows/os_x_mklbuild.yml
@@ -47,4 +47,4 @@ jobs:
           python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'not test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
           MXNET_ENGINE_TYPE=NaiveEngine python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
           python3 -m pytest --durations=50 --verbose tests/python/unittest/ -k 'not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'serial'
-          python3 -m pytest -n 4 --durations=50 --verbose tests/python/mkl
+          python3 -m pytest -n 4 --durations=50 --verbose tests/python/mkl -k 'not test_bf16_operator'

From 3cd9340b49167eae1778058bd143d7318add2217 Mon Sep 17 00:00:00 2001
From: Anna Karbownik <anna.karbownik@intel.com>
Date: Sun, 10 Oct 2021 22:23:16 +0200
Subject: [PATCH 16/16] Remove forcing rpath and some unnecessary comments

---
 .github/workflows/os_x_mklbuild.yml | 8 ++++----
 cmake/upstream/FindBLAS.cmake       | 2 --
 tools/dependencies/mkl.sh           | 1 -
 tools/dependencies/numpy_mkl.sh     | 2 +-
 tools/staticbuild/build_lib.sh      | 3 ---
 5 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/os_x_mklbuild.yml b/.github/workflows/os_x_mklbuild.yml
index b44ca00ea82c..9f6bee4affc8 100644
--- a/.github/workflows/os_x_mklbuild.yml
+++ b/.github/workflows/os_x_mklbuild.yml
@@ -44,7 +44,7 @@ jobs:
 
       - name: Test project
         run: |
-          python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'not test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
-          MXNET_ENGINE_TYPE=NaiveEngine python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
-          python3 -m pytest --durations=50 --verbose tests/python/unittest/ -k 'not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'serial'
-          python3 -m pytest -n 4 --durations=50 --verbose tests/python/mkl -k 'not test_bf16_operator'
+          python -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'not test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
+          MXNET_ENGINE_TYPE=NaiveEngine python -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
+          python -m pytest --durations=50 --verbose tests/python/unittest/ -k 'not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'serial'
+          python -m pytest -n 4 --durations=50 --verbose tests/python/mkl -k 'not test_bf16_operator'
diff --git a/cmake/upstream/FindBLAS.cmake b/cmake/upstream/FindBLAS.cmake
index 6f00955cab9f..b8fbca390062 100644
--- a/cmake/upstream/FindBLAS.cmake
+++ b/cmake/upstream/FindBLAS.cmake
@@ -525,8 +525,6 @@ if(BLA_VENDOR MATCHES "Intel" OR BLA_VENDOR STREQUAL "All")
             get_filename_component(BLAS_mkl_MKLROOT "${BLAS_mkl_MKLROOT}" DIRECTORY)
         endif()
       endif()
-      # MXNET NOTE: This change comes form the newest file version
-      # https://gitlab.kitware.com/cmake/cmake/-/issues/22295
       set(BLAS_mkl_LIB_PATH_SUFFIXES
           "compiler/lib" "compiler/lib/${BLAS_mkl_ARCH_NAME}_${BLAS_mkl_OS_NAME}"
           "compiler/lib/${BLAS_mkl_ARCH_NAME}"
diff --git a/tools/dependencies/mkl.sh b/tools/dependencies/mkl.sh
index 6522ccf2fdcc..863ffd29ff98 100755
--- a/tools/dependencies/mkl.sh
+++ b/tools/dependencies/mkl.sh
@@ -18,7 +18,6 @@
 # under the License.
 
 # This script downloads OneMKL
-# TODO @akarbown: Get more general link to download the OneMKL (instead of the generated one)
 set -ex
 export INTEL_MKL="2021.3.0"
 if [[  (! -e /opt/intel/oneapi/mkl/) ]]; then
diff --git a/tools/dependencies/numpy_mkl.sh b/tools/dependencies/numpy_mkl.sh
index 5a2112493b5c..ba003902d104 100755
--- a/tools/dependencies/numpy_mkl.sh
+++ b/tools/dependencies/numpy_mkl.sh
@@ -42,4 +42,4 @@ if [[ $? -eq 0 ]] && [[ -e /opt/intel/oneapi/mkl/ ]] && [[ ! -e ~/.numpy-site.cf
   # reinstall numpy to use MKL BLAS
   pip3 install numpy==1.19.5 --no-binary numpy --force-reinstall --no-cache-dir
 fi
-set -e
\ No newline at end of file
+set -e
diff --git a/tools/staticbuild/build_lib.sh b/tools/staticbuild/build_lib.sh
index e7d675d11eb0..6fbc25509cab 100755
--- a/tools/staticbuild/build_lib.sh
+++ b/tools/staticbuild/build_lib.sh
@@ -40,9 +40,6 @@ cmake -GNinja -C $cmake_config \
       -DCMAKE_OSX_DEPLOYMENT_TARGET=10.13 \
       ..
 ninja
-if [[ ! $PLATFORM == 'darwin' ]] && [[ $BLAS == 'mkl' ]]; then
-    patchelf --set-rpath "/opt/intel/oneapi/mkl/${INTEL_MKL}/lib/intel64/:\$ORIGIN" --force-rpath libmxnet.so
-fi
 cd -
 
 # Move to lib