From 636897718f8468be0dd5c4077258eb14c7416717 Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Wed, 4 Mar 2020 08:55:01 -0800 Subject: [PATCH] Fix MKL static link & default to static link on unix (#17751) * Fix MKL static link & default to static link on unix Fixes https://github.com/apache/incubator-mxnet/issues/17641 * Test cmake MKL build on CI --- CMakeLists.txt | 11 +++++---- ci/docker/runtime_functions.sh | 42 ++++++++++++++++----------------- ci/jenkins/Jenkins_steps.groovy | 16 ++++++------- cmake/Modules/FindMKL.cmake | 22 ++++++++++------- 4 files changed, 48 insertions(+), 43 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 80469acf6fd9..a0b4e37ad3cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -425,15 +425,13 @@ if(USE_OPENMP) $<$:/w>) endfunction() - # This should build on Windows, but there's some problem and I don't have a Windows box, so - # could a Windows user please fix? if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp/CMakeLists.txt AND SYSTEM_ARCHITECTURE STREQUAL "x86_64" AND NOT CMAKE_BUILD_TYPE STREQUAL "Distribution" + AND NOT BLAS STREQUAL "MKL" AND NOT MSVC AND NOT CMAKE_CROSSCOMPILING) load_omp() - list(REMOVE_ITEM mxnet_LINKER_LIBS iomp5) list(APPEND mxnet_LINKER_LIBS omp) if(UNIX) list(APPEND mxnet_LINKER_LIBS pthread) @@ -446,8 +444,11 @@ if(USE_OPENMP) if(OPENMP_FOUND) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") + if(NOT BLAS STREQUAL "MKL") + # Linker flags for Intel OMP are already set in case MKL is used. Only set if not MKL + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") + endif() add_definitions(-DMXNET_USE_OPENMP=1) endif() endif() diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 5454c0091c6c..eb5748466d42 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -453,18 +453,16 @@ build_ubuntu_cpu_openblas_make() { build_ubuntu_cpu_mkl() { set -ex - export CC="ccache gcc" - export CXX="ccache g++" - make \ - DEV=1 \ - USE_CPP_PACKAGE=1 \ - USE_BLAS=mkl \ - USE_TVM_OP=1 \ - USE_MKLDNN=0 \ - USE_INTEL_PATH=/opt/intel \ - USE_DIST_KVSTORE=1 \ - USE_SIGNAL_HANDLER=1 \ - -j$(nproc) + cd /work/build + cmake \ + -DCMAKE_BUILD_TYPE="RelWithDebInfo" \ + -DUSE_MKLDNN=OFF \ + -DUSE_CUDA=OFF \ + -DUSE_TVM_OP=ON \ + -DUSE_MKL_IF_AVAILABLE=ON \ + -DUSE_BLAS=MKL \ + -GNinja /work/mxnet + ninja } build_ubuntu_cpu_cmake_debug() { @@ -644,16 +642,16 @@ build_ubuntu_cpu_mkldnn() { build_ubuntu_cpu_mkldnn_mkl() { set -ex - build_ccache_wrappers - - make \ - DEV=1 \ - USE_CPP_PACKAGE=1 \ - USE_TVM_OP=1 \ - USE_BLAS=mkl \ - USE_SIGNAL_HANDLER=1 \ - USE_INTEL_PATH=/opt/intel/ \ - -j$(nproc) + cd /work/build + cmake \ + -DCMAKE_BUILD_TYPE="RelWithDebInfo" \ + -DUSE_MKLDNN=ON \ + -DUSE_CUDA=OFF \ + -DUSE_TVM_OP=ON \ + -DUSE_MKL_IF_AVAILABLE=ON \ + -DUSE_BLAS=MKL \ + -GNinja /work/mxnet + ninja } build_ubuntu_gpu() { diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 9cc7794a77bb..c54ce6312733 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -175,13 +175,13 @@ def compile_unix_int64_gpu() { } def compile_unix_mkl_cpu() { - return ['CPU: MKL Makefile': { + return ['CPU: MKL': { node(NODE_LINUX_CPU) { ws('workspace/build-cpu-mkl') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkl', false) - utils.pack_lib('cpu_mkl', mx_mkldnn_lib_make) + utils.pack_lib('cpu_mkl', mx_lib) } } } @@ -217,13 +217,13 @@ def compile_unix_mkldnn_cpu_make() { } def compile_unix_mkldnn_mkl_cpu() { - return ['CPU: MKLDNN_MKL Makefile': { + return ['CPU: MKLDNN_MKL': { node(NODE_LINUX_CPU) { ws('workspace/build-mkldnn-cpu') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkldnn_mkl', false) - utils.pack_lib('mkldnn_mkl_cpu', mx_mkldnn_lib_make) + utils.pack_lib('mkldnn_mkl_cpu', mx_mkldnn_lib) } } } @@ -791,11 +791,11 @@ def test_unix_python3_cpu() { } def test_unix_python3_mkl_cpu() { - return ['Python3: MKL-CPU Makefile': { + return ['Python3: MKL-CPU': { node(NODE_LINUX_CPU) { ws('workspace/ut-python3-cpu') { try { - utils.unpack_and_init('cpu_mkl', mx_lib_make) + utils.unpack_and_init('cpu_mkl', mx_lib) python3_ut('ubuntu_cpu') utils.publish_test_coverage() } finally { @@ -907,11 +907,11 @@ def test_unix_python3_mkldnn_cpu() { } def test_unix_python3_mkldnn_mkl_cpu() { - return ['Python3: MKLDNN-MKL-CPU Makefile': { + return ['Python3: MKLDNN-MKL-CPU': { node(NODE_LINUX_CPU) { ws('workspace/ut-python3-mkldnn-mkl-cpu') { try { - utils.unpack_and_init('mkldnn_mkl_cpu', mx_mkldnn_lib_make) + utils.unpack_and_init('mkldnn_mkl_cpu', mx_lib) python3_ut_mkldnn('ubuntu_cpu') utils.publish_test_coverage() } finally { diff --git a/cmake/Modules/FindMKL.cmake b/cmake/Modules/FindMKL.cmake index 51eff8fe09aa..9e7adbf37318 100644 --- a/cmake/Modules/FindMKL.cmake +++ b/cmake/Modules/FindMKL.cmake @@ -19,8 +19,6 @@ # # Options: # -# USE_MKLDNN : Search for MKL:ML library variant -# # MKL_USE_SINGLE_DYNAMIC_LIBRARY : use single dynamic library interface # MKL_USE_STATIC_LIBS : use static libraries # MKL_MULTI_THREADED : use multi-threading @@ -45,8 +43,14 @@ set(INTEL_ROOT "/opt/intel" CACHE PATH "Folder contains intel libs") # ---[ Options - option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" ON) - cmake_dependent_option(MKL_USE_STATIC_LIBS "Use static libraries" OFF "NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY" OFF) + if(UNIX) + # Single dynamic library interface leads to conflicts between intel omp and llvm omp + # https://github.com/apache/incubator-mxnet/issues/17641 + option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" OFF) + else() + option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" ON) + endif() + cmake_dependent_option(MKL_USE_STATIC_LIBS "Use static libraries" ON "NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY" OFF) cmake_dependent_option(MKL_MULTI_THREADED "Use multi-threading" ON "NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY" OFF) option(MKL_USE_ILP64 "Use ilp64 data model" OFF) cmake_dependent_option(MKL_USE_CLUSTER "Use cluster functions" OFF "CMAKE_SIZEOF_VOID_P EQUAL 4" OFF) @@ -122,7 +126,6 @@ set(INTEL_ROOT "/opt/intel" CACHE PATH "Folder contains intel libs") list(APPEND MKL_LIBRARIES ${${__mkl_lib_upper}_LIBRARY}) endforeach() - if(NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY) if (MKL_USE_STATIC_LIBS) set(__iomp5_libs iomp5 libiomp5mt.lib) @@ -135,15 +138,18 @@ set(INTEL_ROOT "/opt/intel" CACHE PATH "Folder contains intel libs") list(APPEND __looked_for INTEL_INCLUDE_DIR) endif() - find_library(MKL_RTL_LIBRARY ${__iomp5_libs} + find_library(IOMP_LIBRARY ${__iomp5_libs} PATHS ${INTEL_RTL_ROOT} ${INTEL_ROOT}/compiler ${MKL_ROOT}/.. ${MKL_ROOT}/../compiler PATH_SUFFIXES ${__path_suffixes} DOC "Path to Path to OpenMP runtime library") - list(APPEND __looked_for MKL_RTL_LIBRARY) - list(APPEND MKL_LIBRARIES ${MKL_RTL_LIBRARY}) + list(APPEND __looked_for IOMP_LIBRARY) + list(APPEND MKL_LIBRARIES ${IOMP_LIBRARY}) endif() + if(MKL_USE_STATIC_LIBS AND UNIX) + set(MKL_LIBRARIES -Wl,--start-group "${MKL_LIBRARIES}" -Wl,--end-group) + endif() include(FindPackageHandleStandardArgs)