Enable DIST_KVSTORE by default in staticbuild (#18796)

* Enable DIST_KVSTORE by default in staticbuild

set(USE_DIST_KVSTORE ON CACHE BOOL "Build with DIST_KVSTORE support")

* Ensure static linkage of dependencies

* Fix for OS X

* Fix shell syntax

* Alternate approach to force static linkage of libprotobuf

CMakeLists.txt

@@ -147,6 +147,7 @@ if(CMAKE_BUILD_TYPE STREQUAL "Distribution" AND UNIX AND NOT APPLE)
   # Enforce DT_PATH instead of DT_RUNPATH
   set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--disable-new-dtags")
   set(CMAKE_EXE_LINKER_FLAGS "-Wl,--disable-new-dtags")
+  set(Protobuf_USE_STATIC_LIBS ON)
 endif()
 
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/upstream;${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules;${CMAKE_MODULE_PATH}")

config/distribution/darwin_cpu.cmake

@@ -31,3 +31,4 @@ set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
 set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
 set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
 set(USE_LIBJPEG_TURBO ON CACHE BOOL "Build with libjpeg-turbo")
+set(USE_DIST_KVSTORE ON CACHE BOOL "Build with DIST_KVSTORE support")

config/distribution/linux_cpu.cmake

@@ -29,3 +29,4 @@ set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
 set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
 set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
 set(USE_LIBJPEG_TURBO ON CACHE BOOL "Build with libjpeg-turbo")
+set(USE_DIST_KVSTORE ON CACHE BOOL "Build with DIST_KVSTORE support")

config/distribution/linux_cu100.cmake

@@ -31,6 +31,7 @@ set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
 set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
 set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
 set(USE_LIBJPEG_TURBO ON CACHE BOOL "Build with libjpeg-turbo")
+set(USE_DIST_KVSTORE ON CACHE BOOL "Build with DIST_KVSTORE support")
 
 set(CUDACXX "/usr/local/cuda-10.0/bin/nvcc" CACHE STRING "Cuda compiler")
 set(MXNET_CUDA_ARCH "3.0;5.0;6.0;7.0" CACHE STRING "Cuda architectures")

config/distribution/linux_cu101.cmake

@@ -33,6 +33,7 @@ set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
 set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
 set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
 set(USE_LIBJPEG_TURBO ON CACHE BOOL "Build with libjpeg-turbo")
+set(USE_DIST_KVSTORE ON CACHE BOOL "Build with DIST_KVSTORE support")
 
 set(CUDACXX "/usr/local/cuda-10.1/bin/nvcc" CACHE STRING "Cuda compiler")
 set(MXNET_CUDA_ARCH "3.0;5.0;6.0;7.0" CACHE STRING "Cuda architectures")

config/distribution/linux_cu102.cmake

@@ -31,6 +31,7 @@ set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
 set(USE_SSE ON CACHE BOOL "Build with x86 SSE instruction support")
 set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
 set(USE_LIBJPEG_TURBO ON CACHE BOOL "Build with libjpeg-turbo")
+set(USE_DIST_KVSTORE ON CACHE BOOL "Build with DIST_KVSTORE support")
 
 set(CUDACXX "/usr/local/cuda-10.2/bin/nvcc" CACHE STRING "Cuda compiler")
 set(MXNET_CUDA_ARCH "3.0;5.0;6.0;7.0" CACHE STRING "Cuda architectures")

tools/dependencies/cityhash.sh

@@ -20,6 +20,12 @@
 # This script builds the static library of cityhash that can be used as dependency of mxnet.
 set -ex
 CITYHASH_VERSION=1.1.1
+if [[ $PLATFORM == 'darwin' ]]; then
+    DY_EXT="dylib"
+else
+    DY_EXT="so"
+fi
+
 if [[ ! -f $DEPS_PATH/lib/libcityhash.a ]]; then
     # Download and build cityhash
     >&2 echo "Building cityhash..."
@@ -30,5 +36,6 @@ if [[ ! -f $DEPS_PATH/lib/libcityhash.a ]]; then
     ./configure -prefix=$DEPS_PATH --enable-sse4.2
     $MAKE CXXFLAGS="-g -O3 -msse4.2"
     $MAKE install
+    rm $DEPS_PATH/lib/*cityhash*$DY_EXT*
     popd
 fi

tools/dependencies/libpng.sh

@@ -20,8 +20,7 @@
 # This script builds the static library of libpng that can be used as dependency of mxnet/opencv.
 set -ex
 PNG_VERSION=1.6.35
-if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then
-    # download and build libpng
+if [[ ! -f $DEPS_PATH/lib/libpng.a ]] && [[ ! -f $DEPS_PATH/lib64/libpng.a ]]; then    # download and build libpng
     >&2 echo "Building libpng..."
     download \
         https://github.com/glennrp/libpng/archive/v${PNG_VERSION}.zip \

tools/dependencies/libturbojpeg.sh

@@ -25,8 +25,8 @@ if [[ $PLATFORM == 'darwin' ]]; then
     JPEG_NASM_OPTION="-D CMAKE_ASM_NASM_COMPILER=/usr/local/bin/nasm"
 fi
 
-if [[ ! -f $DEPS_PATH/lib/libjpeg.a ]] || [[ ! -f $DEPS_PATH/lib/libturbojpeg.a ]]; then
-    # download and build libjpeg
+if [[ ( ! -f $DEPS_PATH/lib/libjpeg.a ) || ( ! -f $DEPS_PATH/lib/libturbojpeg.a ) ]] && \
+       [[ ( ! -f $DEPS_PATH/lib64/libjpeg.a ) || ( ! -f $DEPS_PATH/lib64/libturbojpeg.a ) ]]; then    # download and build libjpeg
     >&2 echo "Building libjpeg-turbo..."
     download \
         https://github.com/libjpeg-turbo/libjpeg-turbo/archive/${TURBO_JPEG_VERSION}.zip \

tools/dependencies/lz4.sh

@@ -20,6 +20,12 @@
 # This script builds the static library of lz4 that can be used as dependency of mxnet.
 set -ex
 LZ4_VERSION=r130
+if [[ $PLATFORM == 'darwin' ]]; then
+    DY_EXT="dylib"
+else
+    DY_EXT="so"
+fi
+
 if [[ ! -f $DEPS_PATH/lib/liblz4.a ]]; then
     # Download and build lz4
     >&2 echo "Building lz4..."
@@ -31,5 +37,6 @@ if [[ ! -f $DEPS_PATH/lib/liblz4.a ]]; then
     cd $DEPS_PATH/lz4-$LZ4_VERSION
     $MAKE
     $MAKE PREFIX=$DEPS_PATH install
+    rm $DEPS_PATH/lib/*lz4*$DY_EXT*
     popd
 fi

tools/dependencies/openssl.sh

@@ -20,8 +20,8 @@
 # This script builds the static library of openssl that can be used as dependency of mxnet.
 set -ex
 OPENSSL_VERSION=1.1.1b
-if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f $DEPS_PATH/lib/libcrypto.a ]]; then
-    # download and build openssl
+if [[ ( ! -f $DEPS_PATH/lib/libssl.a ) || ( ! -f $DEPS_PATH/lib/libcrypto.a ) ]] && \
+       [[ ( ! -f $DEPS_PATH/lib64/libssl.a ) || ( ! -f $DEPS_PATH/lib64/libcrypto.a ) ]]; then    # download and build openssl
     >&2 echo "Building openssl..."
     OPENSSL_VERSION=$(echo $OPENSSL_VERSION | sed 's/\./_/g')
     download \

tools/dependencies/protobuf.sh

@@ -20,11 +20,6 @@
 # This script builds the static library of protobuf along with protoc, that can be used as dependency of mxnet.
 set -ex
 PROTOBUF_VERSION=3.5.1
-if [[ $PLATFORM == 'darwin' ]]; then
-    DY_EXT="dylib"
-else
-    DY_EXT="so"
-fi
 
 LIBPROTOBUF="$DEPS_PATH/lib/libprotobuf.$DY_EXT"
 LIBPROTOC="$DEPS_PATH/lib/libprotoc.$DY_EXT"

tools/dependencies/zmq.sh

@@ -20,6 +20,12 @@
 # This script builds the static library of zeroMQ that can be used as dependency of mxnet.
 set -ex
 ZEROMQ_VERSION=4.2.2
+if [[ $PLATFORM == 'darwin' ]]; then
+    DY_EXT="dylib"
+else
+    DY_EXT="so"
+fi
+
 if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then
     # Download and build zmq
     >&2 echo "Building zmq..."
@@ -39,8 +45,11 @@ if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then
     $MAKE install
 
     if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then
+        rm $DEPS_PATH/lib64/*zmq*$DY_EXT*
         mkdir -p $DEPS_PATH/lib
         cp $DEPS_PATH/lib64/*zmq* $DEPS_PATH/lib
+    else
+        rm $DEPS_PATH/lib/*zmq*$DY_EXT*
     fi
 
     popd