From b055ae372c201724283916182c75625cb12f8567 Mon Sep 17 00:00:00 2001 From: sL1pKn07 Date: Wed, 11 Mar 2020 16:32:13 +0100 Subject: [PATCH 1/4] Add SOVERSION when build shared libmxnet.so library https://en.wikipedia.org/wiki/Soname https://cmake.org/cmake/help/latest/prop_tgt/SOVERSION.html --- CMakeLists.txt | 11 ++++- R-package/Makefile | 4 +- cd/Jenkinsfile_utils.groovy | 2 +- cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy | 2 +- cd/mxnet_lib/static/Jenkins_pipeline.groovy | 2 +- cd/utils/test_artifact_repository.py | 10 ++-- ci/docker/runtime_functions.sh | 2 +- ci/jenkins/Jenkins_steps.groovy | 46 +++++++++---------- python/setup.py | 2 +- tests/nightly/Jenkinsfile | 2 +- tests/nightly/JenkinsfileForBinaries | 2 +- .../JenkinsfileForMBCC | 2 +- tools/staticbuild/build_lib_cmake.sh | 2 +- 13 files changed, 49 insertions(+), 40 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e2134268b39d..0df6fbca66ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,13 @@ cmake_minimum_required(VERSION 3.13) +file(READ "include/mxnet/base.h" ver) +string(REGEX MATCH "MXNET_MAJOR ([0-9]*)" _ ${ver}) +set(ver_major ${CMAKE_MATCH_1}) +string(REGEX MATCH "MXNET_MINOR ([0-9]*)" _ ${ver}) +set(ver_minor ${CMAKE_MATCH_1}) +string(REGEX MATCH "MXNET_PATCH ([0-9]*)" _ ${ver}) +set(ver_patch ${CMAKE_MATCH_1}) + # workaround to store CMAKE_CROSSCOMPILING because is getting reset by the project command if(CMAKE_CROSSCOMPILING) set(__CMAKE_CROSSCOMPILING ${CMAKE_CROSSCOMPILING}) @@ -86,7 +94,7 @@ if(USE_TVM_OP) endif() message(STATUS "CMake version '${CMAKE_VERSION}' using generator '${CMAKE_GENERATOR}'") -project(mxnet C CXX) +project(${PROJECT_NAME} VERSION ${ver_major}.${ver_minor}.${ver_patch} LANGUAGES C CXX) if(USE_CUDA) cmake_minimum_required(VERSION 3.13.2) # CUDA 10 (Turing) detection available starting 3.13.2 include(CheckLanguage) @@ -701,6 +709,7 @@ if(UNIX) target_link_libraries(mxnet PRIVATE mxnet_static) target_link_libraries(mxnet_static PUBLIC ${CMAKE_DL_LIBS}) set_target_properties(mxnet_static PROPERTIES OUTPUT_NAME mxnet) + set_target_properties(mxnet PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR}) if(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") target_compile_options(mxnet_static PRIVATE "$<$:-Werror>") # Ignore erroneous compiler warnings: diff --git a/R-package/Makefile b/R-package/Makefile index 5a8bc42aca93..efb76a9ff908 100644 --- a/R-package/Makefile +++ b/R-package/Makefile @@ -5,12 +5,12 @@ rpkg: mkdir -p R-package/inst/libs cp src/io/image_recordio.h R-package/src if [ -d "lib" ]; then \ - cp -rf lib/libmxnet.so R-package/inst/libs; \ + cp -rf lib/libmxnet.so* R-package/inst/libs; \ if [ -e "lib/libtvm_runtime.so" ]; then \ cp -rf lib/libtvm_runtime.so R-package/inst/libs; \ fi; \ else \ - cp -rf build/libmxnet.so R-package/inst/libs; \ + cp -rf build/libmxnet.so* R-package/inst/libs; \ if [ -e "build/libtvm_runtime.so" ]; then \ cp -rf build/libtvm_runtime.so R-package/inst/libs; \ fi; \ diff --git a/cd/Jenkinsfile_utils.groovy b/cd/Jenkinsfile_utils.groovy index 7ba470ae5c9f..22d25779cf5a 100644 --- a/cd/Jenkinsfile_utils.groovy +++ b/cd/Jenkinsfile_utils.groovy @@ -131,7 +131,7 @@ def restore_artifact(variant, libtype) { // move libraries to lib directory dir('lib') { - sh "mv ../mxnet_artifact/libmxnet.so ." + sh "mv ../mxnet_artifact/libmxnet.so* ." if (fileExists('../mxnet_artifact/dependencies')) { sh """find "../mxnet_artifact/dependencies" -type f -name "*.so*" -exec mv {} . \\;""" sh "ls ." diff --git a/cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy b/cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy index 52ad150fe6f9..7d9560074b48 100644 --- a/cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy +++ b/cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy @@ -24,7 +24,7 @@ // NOTE: the following variables are referenced in the mxnet_lib_pipeline jenkins file imported bellow // libmxnet location -libmxnet = 'build/libmxnet.so' +libmxnet = 'build/libmxnet.so*' // licenses licenses = 'licenses/*' diff --git a/cd/mxnet_lib/static/Jenkins_pipeline.groovy b/cd/mxnet_lib/static/Jenkins_pipeline.groovy index abbafdbef075..9e746561e07b 100644 --- a/cd/mxnet_lib/static/Jenkins_pipeline.groovy +++ b/cd/mxnet_lib/static/Jenkins_pipeline.groovy @@ -26,7 +26,7 @@ // NOTE: the following variables are referenced in the mxnet_lib_pipeline jenkins file imported bellow // libmxnet location -libmxnet = 'lib/libmxnet.so' +libmxnet = 'lib/libmxnet.so*' // licenses licenses = 'licenses/*' diff --git a/cd/utils/test_artifact_repository.py b/cd/utils/test_artifact_repository.py index 2ab5d910c87b..879031b4aa08 100644 --- a/cd/utils/test_artifact_repository.py +++ b/cd/utils/test_artifact_repository.py @@ -164,7 +164,7 @@ def test_probe_variant_native(self, mock_features): Tests 'native' is returned if MKLDNN and CUDA features are OFF """ mock_features.return_value = {'MKLDNN': False, 'CUDA': False} - self.assertEqual(probe_mxnet_variant('libmxnet.so'), 'native') + self.assertEqual(probe_mxnet_variant('libmxnet.so*'), 'native') @patch('artifact_repository.get_libmxnet_features') def test_probe_variant_cpu(self, mock_features): @@ -172,7 +172,7 @@ def test_probe_variant_cpu(self, mock_features): Tests 'cpu' is returned if MKLDNN is ON and CUDA is OFF """ mock_features.return_value = {'MKLDNN': True, 'CUDA': False} - self.assertEqual(probe_mxnet_variant('libmxnet.so'), 'cpu') + self.assertEqual(probe_mxnet_variant('libmxnet.so*'), 'cpu') @patch('artifact_repository.get_libmxnet_features') @patch('artifact_repository.get_cuda_version') @@ -182,7 +182,7 @@ def test_probe_variant_cuda(self, mock_cuda_version, mock_features): """ mock_features.return_value = {'MKLDNN': True, 'CUDA': True} mock_cuda_version.return_value = '100' - self.assertEqual(probe_mxnet_variant('libmxnet.so'), 'cu100') + self.assertEqual(probe_mxnet_variant('libmxnet.so*'), 'cu100') @patch('artifact_repository.get_libmxnet_features') def test_probe_variant_cuda_returns_none_on_no_features(self, mock_features): @@ -190,7 +190,7 @@ def test_probe_variant_cuda_returns_none_on_no_features(self, mock_features): Tests None is returned if the mxnet features could not be extracted from the libmxnet.so file """ mock_features.return_value = None - self.assertIsNone(probe_mxnet_variant('libmxnet.so')) + self.assertIsNone(probe_mxnet_variant('libmxnet.so*')) @patch('artifact_repository.get_libmxnet_features') @patch('artifact_repository.get_cuda_version') @@ -201,7 +201,7 @@ def test_probe_variant_cuda_mkl(self, mock_cuda_version, mock_features): mock_features.return_value = {'MKLDNN': True, 'CUDA': True} mock_cuda_version.return_value = None with self.assertRaises(RuntimeError): - probe_mxnet_variant('libmxnet.so') + probe_mxnet_variant('libmxnet.so*') def test_probe_artifact_repository_bucket(self): """ diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index e1f36996d43d..2b9dbfeabed1 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -136,7 +136,7 @@ build_wheel() { unzip -d ${TMPDIR} ${WHEEL} rm ${WHEEL} cd ${TMPDIR} - mv *.data/data/mxnet/libmxnet.so mxnet + mv *.data/data/mxnet/libmxnet.so* mxnet zip -r ${WHEEL} . cp ${WHEEL} ${BUILD_DIR} rm -rf ${TMPDIR} diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index e6e7c5d278d3..a50b0db37bcf 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -23,27 +23,27 @@ utils = load('ci/Jenkinsfile_utils.groovy') // mxnet libraries -mx_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so' -mx_lib_cython = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, python/mxnet/_cy3/*.so, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_ffi/_cy3/*.so' -mx_lib_make = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' +mx_lib = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so' +mx_lib_cython = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, python/mxnet/_cy3/*.so*, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_ffi/_cy3/*.so*' +mx_lib_make = 'lib/libmxnet.so*, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' // Python wheels mx_pip = 'build/*.whl' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. -mx_cmake_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' -mx_cmake_lib_no_tvm_op = 'build/libmxnet.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' -mx_cmake_lib_cython = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so' +mx_cmake_lib = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' +mx_cmake_lib_no_tvm_op = 'build/libmxnet.so*, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' +mx_cmake_lib_cython = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, python/mxnet/_cy3/*.so*, python/mxnet/_ffi/_cy3/*.so*' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. -mx_cmake_lib_debug = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/tests/mxnet_unit_tests' -mx_mkldnn_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so' -mx_mkldnn_lib_make = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' -mx_tensorrt_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so' -mx_lib_cpp_examples = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/cpp-package/example/*, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so' -mx_lib_cpp_examples_make = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so' -mx_lib_cpp_capi_make = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, libsample_lib.so, lib/libmkldnn.so.1, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so, build/tests/cpp/mxnet_unit_tests' -mx_lib_cpp_examples_no_tvm_op = 'build/libmxnet.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so, build/cpp-package/example/*, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so' -mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, build/cpp-package/example/*' +mx_cmake_lib_debug = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/tests/mxnet_unit_tests' +mx_mkldnn_lib = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so' +mx_mkldnn_lib_make = 'lib/libmxnet.so*, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' +mx_tensorrt_lib = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so' +mx_lib_cpp_examples = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/cpp-package/example/*, python/mxnet/_cy3/*.so*, python/mxnet/_ffi/_cy3/*.so*' +mx_lib_cpp_examples_make = 'lib/libmxnet.so*, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, python/mxnet/_cy3/*.so*, python/mxnet/_ffi/_cy3/*.so*' +mx_lib_cpp_capi_make = 'lib/libmxnet.so*, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, libsample_lib.so, lib/libmkldnn.so.1, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, python/mxnet/_cy3/*.so*, python/mxnet/_ffi/_cy3/*.so*, build/tests/cpp/mxnet_unit_tests' +mx_lib_cpp_examples_no_tvm_op = 'build/libmxnet.so*, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so, build/cpp-package/example/*, python/mxnet/_cy3/*.so*, python/mxnet/_ffi/_cy3/*.so*' +mx_lib_cpp_examples_cpu = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, build/cpp-package/example/*' // Python unittest for CPU // Python 3 @@ -1474,7 +1474,7 @@ def compile_unix_lite() { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() utils.docker_run('ubuntu_cpu_lite', 'build_ubuntu_cpu_docs', false) - utils.pack_lib('libmxnet', 'lib/libmxnet.so', false) + utils.pack_lib('libmxnet', 'lib/libmxnet.so*', false) } } } @@ -1502,7 +1502,7 @@ def docs_python() { node(NODE_LINUX_CPU) { ws('workspace/docs') { timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('libmxnet', 'lib/libmxnet.so', false) + utils.unpack_and_init('libmxnet', 'lib/libmxnet.so*', false) utils.docker_run('ubuntu_cpu_python', 'build_python_docs', false) if (should_pack_website()) { utils.pack_lib('python-artifacts', 'docs/_build/python-artifacts.tgz', false) @@ -1520,7 +1520,7 @@ def docs_c() { node(NODE_LINUX_CPU) { ws('workspace/docs') { timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('libmxnet', 'lib/libmxnet.so', false) + utils.unpack_and_init('libmxnet', 'lib/libmxnet.so*', false) utils.docker_run('ubuntu_cpu_c', 'build_c_docs', false) if (should_pack_website()) { utils.pack_lib('c-artifacts', 'docs/_build/c-artifacts.tgz', false) @@ -1538,7 +1538,7 @@ def docs_julia() { node(NODE_LINUX_CPU) { ws('workspace/docs') { timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('libmxnet', 'lib/libmxnet.so', false) + utils.unpack_and_init('libmxnet', 'lib/libmxnet.so*', false) utils.docker_run('ubuntu_cpu_julia', 'build_julia_docs', false) if (should_pack_website()) { utils.pack_lib('julia-artifacts', 'docs/_build/julia-artifacts.tgz', false) @@ -1556,7 +1556,7 @@ def docs_r() { node(NODE_LINUX_CPU) { ws('workspace/docs') { timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('libmxnet', 'lib/libmxnet.so', false) + utils.unpack_and_init('libmxnet', 'lib/libmxnet.so*', false) utils.docker_run('ubuntu_cpu_r', 'build_r_docs', false) if (should_pack_website()) { utils.pack_lib('r-artifacts', 'docs/_build/r-artifacts.tgz', false) @@ -1575,7 +1575,7 @@ def docs_scala() { node(NODE_LINUX_CPU) { ws('workspace/docs') { timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('libmxnet', 'lib/libmxnet.so', false) + utils.unpack_and_init('libmxnet', 'lib/libmxnet.so*', false) utils.docker_run('ubuntu_cpu_scala', 'build_scala_docs', false) if (should_pack_website()) { utils.pack_lib('scala-artifacts', 'docs/_build/scala-artifacts.tgz', false) @@ -1594,7 +1594,7 @@ def docs_java() { node(NODE_LINUX_CPU) { ws('workspace/docs') { timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('libmxnet', 'lib/libmxnet.so', false) + utils.unpack_and_init('libmxnet', 'lib/libmxnet.so*', false) utils.docker_run('ubuntu_cpu_scala', 'build_java_docs', false) if (should_pack_website()) { utils.pack_lib('java-artifacts', 'docs/_build/java-artifacts.tgz', false) @@ -1613,7 +1613,7 @@ def docs_clojure() { node(NODE_LINUX_CPU) { ws('workspace/docs') { timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('libmxnet', 'lib/libmxnet.so', false) + utils.unpack_and_init('libmxnet', 'lib/libmxnet.so*', false) utils.docker_run('ubuntu_cpu_scala', 'build_clojure_docs', false) if (should_pack_website()) { utils.pack_lib('clojure-artifacts', 'docs/_build/clojure-artifacts.tgz', false) diff --git a/python/setup.py b/python/setup.py index dcd84cef1ea1..c2f44b4ca35e 100644 --- a/python/setup.py +++ b/python/setup.py @@ -94,7 +94,7 @@ def config_cython(): libraries=libraries, extra_link_args=extra_link_args, language="c++")) - + path = "mxnet/_ffi/_cython" for fn in os.listdir(path): if not fn.endswith(".pyx"): diff --git a/tests/nightly/Jenkinsfile b/tests/nightly/Jenkinsfile index 9ea133e718d2..da70db0854a0 100755 --- a/tests/nightly/Jenkinsfile +++ b/tests/nightly/Jenkinsfile @@ -18,7 +18,7 @@ // //This is a Jenkinsfile for nightly tests. The format and some functions have been picked up from the top-level Jenkinsfile -mx_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so' +mx_lib = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so' node('utility') { // Loading the utilities requires a node context unfortunately diff --git a/tests/nightly/JenkinsfileForBinaries b/tests/nightly/JenkinsfileForBinaries index 81c022c28990..9aec09135ad4 100755 --- a/tests/nightly/JenkinsfileForBinaries +++ b/tests/nightly/JenkinsfileForBinaries @@ -18,7 +18,7 @@ // //This is a Jenkinsfile for nightly tests. The format and some functions have been picked up from the top-level Jenkinsfile -mx_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so, build/cpp-package/example/*' +mx_lib = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so, build/cpp-package/example/*' node('utility') { // Loading the utilities requires a node context unfortunately diff --git a/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC b/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC index e42e495dc230..92af3ed14676 100644 --- a/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC +++ b/tests/nightly/model_backwards_compatibility_check/JenkinsfileForMBCC @@ -18,7 +18,7 @@ // //This is a Jenkinsfile for the model backwards compatibility checker. The format and some functions have been picked up from the top-level Jenkinsfile. -mx_lib = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so' +mx_lib = 'build/libmxnet.so*, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so' node('restricted-utility') { // Loading the utilities requires a node context unfortunately diff --git a/tools/staticbuild/build_lib_cmake.sh b/tools/staticbuild/build_lib_cmake.sh index 6a4bbec7afcf..80cccbf22f7a 100755 --- a/tools/staticbuild/build_lib_cmake.sh +++ b/tools/staticbuild/build_lib_cmake.sh @@ -37,7 +37,7 @@ cd - # Move to lib rm -rf lib; mkdir lib; if [[ $PLATFORM == 'linux' ]]; then - cp -L build/libmxnet.so lib/libmxnet.so + cp -L build/libmxnet.so* lib/ cp -L staticdeps/lib/libopenblas.so lib/libopenblas.so.0 if [[ -f /usr/lib/gcc/x86_64-linux-gnu/4.8/libgfortran.so ]]; then cp -L /usr/lib/gcc/x86_64-linux-gnu/4.8/libgfortran.so lib/libgfortran.so.3 From d2659e29affad5d1d848c7cf608d9768b26559e5 Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Tue, 28 Jul 2020 20:45:53 +0000 Subject: [PATCH 2/4] Resolve remaining merge conflicts --- ci/jenkins/Jenkins_steps.groovy | 123 -------------------------------- 1 file changed, 123 deletions(-) diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 71cb44f67d0d..55da8e673843 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -1051,13 +1051,8 @@ def compile_unix_lite(lib_name) { ws('workspace/docs') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() -<<<<<<< HEAD - utils.docker_run('ubuntu_cpu_lite', 'build_ubuntu_cpu_docs', false) - utils.pack_lib('libmxnet', 'lib/libmxnet.so*', false) -======= utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_docs', false) utils.pack_lib(lib_name, mx_lib, false) ->>>>>>> origin/master } } } @@ -1085,11 +1080,7 @@ def docs_python(lib_name) { node(NODE_LINUX_CPU) { ws('workspace/docs') { timeout(time: max_time, unit: 'MINUTES') { -<<<<<<< HEAD - utils.unpack_and_init('libmxnet', 'lib/libmxnet.so*', false) -======= utils.unpack_and_init(lib_name, mx_lib, false) ->>>>>>> origin/master utils.docker_run('ubuntu_cpu_python', 'build_python_docs', false) if (should_pack_website()) { utils.pack_lib('python-artifacts', 'docs/_build/python-artifacts.tgz', false) @@ -1101,120 +1092,6 @@ def docs_python(lib_name) { } -<<<<<<< HEAD -// Call this function from Jenkins to generate just the C and C++ API microsite artifacts. -def docs_c() { - return ['C Docs': { - node(NODE_LINUX_CPU) { - ws('workspace/docs') { - timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('libmxnet', 'lib/libmxnet.so*', false) - utils.docker_run('ubuntu_cpu_c', 'build_c_docs', false) - if (should_pack_website()) { - utils.pack_lib('c-artifacts', 'docs/_build/c-artifacts.tgz', false) - } - } - } - } - }] -} - - -// Call this function from Jenkins to generate just the Julia API microsite artifacts. -def docs_julia() { - return ['Julia Docs': { - node(NODE_LINUX_CPU) { - ws('workspace/docs') { - timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('libmxnet', 'lib/libmxnet.so*', false) - utils.docker_run('ubuntu_cpu_julia', 'build_julia_docs', false) - if (should_pack_website()) { - utils.pack_lib('julia-artifacts', 'docs/_build/julia-artifacts.tgz', false) - } - } - } - } - }] -} - - -// Call this function from Jenkins to generate just the R API PDF artifact. -def docs_r() { - return ['R Docs': { - node(NODE_LINUX_CPU) { - ws('workspace/docs') { - timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('libmxnet', 'lib/libmxnet.so*', false) - utils.docker_run('ubuntu_cpu_r', 'build_r_docs', false) - if (should_pack_website()) { - utils.pack_lib('r-artifacts', 'docs/_build/r-artifacts.tgz', false) - } - } - } - } - }] -} - - -// Call this function from Jenkins to generate just the Scala API microsite artifacts. -// It will also generate the Scala package. -def docs_scala() { - return ['Scala Docs': { - node(NODE_LINUX_CPU) { - ws('workspace/docs') { - timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('libmxnet', 'lib/libmxnet.so*', false) - utils.docker_run('ubuntu_cpu_scala', 'build_scala_docs', false) - if (should_pack_website()) { - utils.pack_lib('scala-artifacts', 'docs/_build/scala-artifacts.tgz', false) - } - } - } - } - }] -} - - -// Call this function from Jenkins to generate just the Java API microsite artifacts. -// It will also generate the Scala package. -def docs_java() { - return ['Java Docs': { - node(NODE_LINUX_CPU) { - ws('workspace/docs') { - timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('libmxnet', 'lib/libmxnet.so*', false) - utils.docker_run('ubuntu_cpu_scala', 'build_java_docs', false) - if (should_pack_website()) { - utils.pack_lib('java-artifacts', 'docs/_build/java-artifacts.tgz', false) - } - } - } - } - }] -} - - -// Call this function from Jenkins to generate just the Clojure API microsite artifacts. -// It will also generate the Scala package. -def docs_clojure() { - return ['Clojure Docs': { - node(NODE_LINUX_CPU) { - ws('workspace/docs') { - timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('libmxnet', 'lib/libmxnet.so*', false) - utils.docker_run('ubuntu_cpu_scala', 'build_clojure_docs', false) - if (should_pack_website()) { - utils.pack_lib('clojure-artifacts', 'docs/_build/clojure-artifacts.tgz', false) - } - } - } - } - }] -} - - -======= ->>>>>>> origin/master // Call this function from Jenkins to generate just the main website artifacts. def docs_jekyll() { return ['Main Jekyll Website': { From 598be31daf8543b24815a8fda592f2e313a3de2e Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Thu, 6 Aug 2020 22:46:08 +0000 Subject: [PATCH 3/4] Delete unnecessary files --- R-package/Makefile | 33 -------------- tools/staticbuild/build_lib_cmake.sh | 68 ---------------------------- 2 files changed, 101 deletions(-) delete mode 100644 R-package/Makefile delete mode 100755 tools/staticbuild/build_lib_cmake.sh diff --git a/R-package/Makefile b/R-package/Makefile deleted file mode 100644 index efb76a9ff908..000000000000 --- a/R-package/Makefile +++ /dev/null @@ -1,33 +0,0 @@ -rcpplint: - ./3rdparty/dmlc-core/scripts/lint.py mxnet-rcpp all R-package/src - -rpkg: - mkdir -p R-package/inst/libs - cp src/io/image_recordio.h R-package/src - if [ -d "lib" ]; then \ - cp -rf lib/libmxnet.so* R-package/inst/libs; \ - if [ -e "lib/libtvm_runtime.so" ]; then \ - cp -rf lib/libtvm_runtime.so R-package/inst/libs; \ - fi; \ - else \ - cp -rf build/libmxnet.so* R-package/inst/libs; \ - if [ -e "build/libtvm_runtime.so" ]; then \ - cp -rf build/libtvm_runtime.so R-package/inst/libs; \ - fi; \ - fi - - mkdir -p R-package/inst/include - cp -rl include/* R-package/inst/include - Rscript -e "if(!require(devtools)){install.packages('devtools', repo = 'https://cloud.r-project.org/')}" - Rscript -e "if(!require(roxygen2)||packageVersion('roxygen2') < '6.1.1'){install.packages('roxygen2', repo = 'https://cloud.r-project.org/')}" - Rscript -e "library(devtools); library(methods); options(repos=c(CRAN='https://cloud.r-project.org/')); install_deps(pkg='R-package', dependencies = TRUE)" - cp R-package/dummy.NAMESPACE R-package/NAMESPACE # NAMESPACE will be replaced by devtools::document later - echo "import(Rcpp)" >> R-package/NAMESPACE - R CMD INSTALL R-package - Rscript -e "require(mxnet); mxnet:::mxnet.export('R-package'); warnings()" - Rscript -e "devtools::document('R-package');warnings()" - R CMD INSTALL R-package - -rpkgtest: - Rscript -e 'require(testthat);res<-test_dir("R-package/tests/testthat");if(!testthat:::all_passed(res)){stop("Test failures", call. = FALSE)}' - Rscript -e 'res<-covr:::package_coverage("R-package");fileConn<-file(paste("r-package_coverage_",toString(runif(1)),".json"));writeLines(covr:::to_codecov(res), fileConn);close(fileConn)' diff --git a/tools/staticbuild/build_lib_cmake.sh b/tools/staticbuild/build_lib_cmake.sh deleted file mode 100755 index 80cccbf22f7a..000000000000 --- a/tools/staticbuild/build_lib_cmake.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eo pipefail - -# This script builds the libraries of mxnet. -cmake_config=${CURDIR}/config/distribution/${PLATFORM}_${VARIANT}.cmake -if [[ ! -f $cmake_config ]]; then - >&2 echo "Couldn't find cmake config $make_config for the current settings." - exit 1 -fi - -git submodule update --init --recursive || true - -# Build libmxnet.so -rm -rf build; mkdir build; cd build -cmake -GNinja -C $cmake_config -DCMAKE_PREFIX_PATH=${DEPS_PATH} -DCMAKE_FIND_ROOT_PATH=${DEPS_PATH} .. -ninja -cd - - -# Move to lib -rm -rf lib; mkdir lib; -if [[ $PLATFORM == 'linux' ]]; then - cp -L build/libmxnet.so* lib/ - cp -L staticdeps/lib/libopenblas.so lib/libopenblas.so.0 - if [[ -f /usr/lib/gcc/x86_64-linux-gnu/4.8/libgfortran.so ]]; then - cp -L /usr/lib/gcc/x86_64-linux-gnu/4.8/libgfortran.so lib/libgfortran.so.3 - elif [[ -f /usr/lib/x86_64-linux-gnu/libgfortran.so.3 ]]; then - cp -L /usr/lib/x86_64-linux-gnu/libgfortran.so.3 lib/libgfortran.so.3 - else - cp -L /usr/lib/x86_64-linux-gnu/libgfortran.so.4 lib/libgfortran.so.4 - fi - cp -L /usr/lib/x86_64-linux-gnu/libquadmath.so.0 lib/libquadmath.so.0 -elif [[ $PLATFORM == 'darwin' ]]; then - cp -L build/libmxnet.dylib lib/libmxnet.dylib -fi - -# Print the linked objects on libmxnet.so ->&2 echo "Checking linked objects on libmxnet.so..." -if [[ ! -z $(command -v readelf) ]]; then - readelf -d lib/libmxnet.so - strip --strip-unneeded lib/libmxnet.so -elif [[ ! -z $(command -v otool) ]]; then - otool -L lib/libmxnet.dylib - strip -u -r -x lib/libmxnet.dylib -else - >&2 echo "Not available" -fi - -if [[ ! -L deps ]]; then - ln -s staticdeps deps -fi From 3e2f02b14d712590593705c54cd2cb640c90df79 Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Thu, 6 Aug 2020 23:08:33 +0000 Subject: [PATCH 4/4] Remove duplicate project() --- CMakeLists.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c3ec446d19f..3082e26c5dd9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,7 +14,7 @@ if(CMAKE_CROSSCOMPILING) set(__CMAKE_CROSSCOMPILING_OVERRIDE ON) endif() -project(mxnet C CXX) +project(mxnet VERSION ${ver_major}.${ver_minor}.${ver_patch} LANGUAGES C CXX) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS ON) # GNU extensions used by src/operator/random/shuffle_op.cc @@ -111,8 +111,6 @@ if(USE_TVM_OP) add_definitions(-DMXNET_USE_TVM_OP=1) endif() -message(STATUS "CMake version '${CMAKE_VERSION}' using generator '${CMAKE_GENERATOR}'") -project(${PROJECT_NAME} VERSION ${ver_major}.${ver_minor}.${ver_patch} LANGUAGES C CXX) if(USE_CUDA) cmake_minimum_required(VERSION 3.13.2) # CUDA 10 (Turing) detection available starting 3.13.2 include(CheckLanguage)