From ca1a91286d87321bc3f298066c5748ce3414cf5c Mon Sep 17 00:00:00 2001 From: Kellen Sunderland Date: Tue, 11 Sep 2018 22:07:23 +0200 Subject: [PATCH 1/4] Add ASAN sanitizer to CI --- CMakeLists.txt | 9 ++ Jenkinsfile | 158 +++++++++++--------------- ci/Jenkinsfile_utils.groovy | 3 +- ci/build.py | 3 + ci/docker/Dockerfile.build.ubuntu_cpu | 3 + ci/docker/install/ubuntu_gcc8.sh | 23 ++++ ci/docker/runtime_functions.sh | 37 ++++++ 7 files changed, 146 insertions(+), 90 deletions(-) create mode 100755 ci/docker/install/ubuntu_gcc8.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index adff53336654..18bf0461468f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,6 +38,7 @@ mxnet_option(BUILD_CPP_EXAMPLES "Build cpp examples" ON) mxnet_option(INSTALL_EXAMPLES "Install the example source files." OFF) mxnet_option(USE_SIGNAL_HANDLER "Print stack traces on segfaults." OFF) mxnet_option(USE_TENSORRT "Enable infeference optimization with TensorRT." OFF) +mxnet_option(USE_ASAN "Enable Clang/GCC ASAN sanitizers." OFF) message(STATUS "CMAKE_SYSTEM_NAME ${CMAKE_SYSTEM_NAME}") if(USE_CUDA AND NOT USE_OLDCMAKECUDA) @@ -285,6 +286,14 @@ else() endif() endif() +if(USE_ASAN) + set(CMAKE_CFLAGS "${CMAKE_CFLAGS} -fno-omit-frame-pointer -fsanitize=address") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -fsanitize=address") + set(CMAKE_LINKER_FLAGS "${CMAKE_LINKER_FLAGS} -fsanitize=address") + set(GTEST_LIBRARIES "${GTEST_LIBRARIES} -fsanitize=address") + list(APPEND mxnet_LINKER_LIBS asan) +endif() + list(APPEND mxnet_LINKER_LIBS ${mshadow_LINKER_LIBS}) foreach(var ${C_CXX_INCLUDE_DIRECTORIES}) diff --git a/Jenkinsfile b/Jenkinsfile index 346cb19ce46f..eca32f9b8b91 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -31,6 +31,9 @@ mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-c mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0' mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so' +mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, build/cpp-package/example/lenet, build/cpp-package/example/alexnet, build/cpp-package/example/googlenet, build/cpp-package/example/lenet_with_mxdataiter, build/cpp-package/example/resnet, build/cpp-package/example/mlp, build/cpp-package/example/mlp_cpu, build/cpp-package/example/mlp_gpu, build/cpp-package/example/test_score, build/cpp-package/example/test_optimizer' +mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/mlp_cpu' + // timeout in minutes max_time = 120 @@ -50,6 +53,13 @@ def python3_ut(docker_container_name) { } } +// Python 3 +def python3_ut_asan(docker_container_name) { + timeout(time: max_time, unit: 'MINUTES') { + utils.docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu_asan', false) + } +} + def python3_ut_mkldnn(docker_container_name) { timeout(time: max_time, unit: 'MINUTES') { utils.docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu_mkldnn', false) @@ -152,6 +162,17 @@ core_logic: { } } }, + 'CPU: ASAN': { + node(NODE_LINUX_CPU) { + ws('workspace/build-cpu-asan') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git() + utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_cmake_asan', false) + utils.pack_lib('cpu_asan', mx_lib_cpp_examples_cpu) + } + } + } + }, 'CPU: Openblas, debug': { node(NODE_LINUX_CPU) { ws('workspace/build-cpu-openblas') { @@ -254,17 +275,7 @@ core_logic: { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() utils.docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda91_cudnn7', false) - utils.pack_lib('gpu', mx_dist_lib) - stash includes: 'build/cpp-package/example/lenet', name: 'cpp_lenet' - stash includes: 'build/cpp-package/example/alexnet', name: 'cpp_alexnet' - stash includes: 'build/cpp-package/example/googlenet', name: 'cpp_googlenet' - stash includes: 'build/cpp-package/example/lenet_with_mxdataiter', name: 'cpp_lenet_with_mxdataiter' - stash includes: 'build/cpp-package/example/resnet', name: 'cpp_resnet' - stash includes: 'build/cpp-package/example/mlp', name: 'cpp_mlp' - stash includes: 'build/cpp-package/example/mlp_cpu', name: 'cpp_mlp_cpu' - stash includes: 'build/cpp-package/example/mlp_gpu', name: 'cpp_mlp_gpu' - stash includes: 'build/cpp-package/example/test_score', name: 'cpp_test_score' - stash includes: 'build/cpp-package/example/test_optimizer', name: 'cpp_test_optimizer' + utils.pack_lib('gpu', mx_lib_cpp_examples) } } } @@ -431,8 +442,7 @@ core_logic: { node(NODE_LINUX_CPU) { ws('workspace/ut-python2-cpu') { try { - utils.init_git() - utils.unpack_lib('cpu', mx_lib) + utils.unpack_and_init('cpu', mx_lib) python2_ut('ubuntu_cpu') utils.publish_test_coverage() } finally { @@ -447,8 +457,7 @@ core_logic: { node(NODE_LINUX_CPU) { ws('workspace/ut-python3-cpu') { try { - utils.init_git() - utils.unpack_lib('cpu', mx_lib) + utils.unpack_and_init('cpu', mx_lib) python3_ut('ubuntu_cpu') utils.publish_test_coverage() } finally { @@ -458,12 +467,19 @@ core_logic: { } } }, + 'CPU ASAN': { + node(NODE_LINUX_CPU) { + ws('workspace/ut-python3-cpu-asan') { + utils.unpack_and_init('cpu_asan', mx_lib_cpp_examples_cpu) + utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_asan', false) + } + } + }, 'Python3: CPU debug': { node(NODE_LINUX_CPU) { ws('workspace/ut-python3-cpu-debug') { try { - utils.init_git() - utils.unpack_lib('cpu_debug', mx_cmake_lib_debug) + utils.unpack_and_init('cpu_debug', mx_cmake_lib_debug) python3_ut('ubuntu_cpu') } finally { utils.collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_cpu_debug_unittest.xml') @@ -476,8 +492,7 @@ core_logic: { node(NODE_LINUX_GPU) { ws('workspace/ut-python2-gpu') { try { - utils.init_git() - utils.unpack_lib('gpu', mx_lib) + utils.unpack_and_init('gpu', mx_lib) python2_gpu_ut('ubuntu_gpu') utils.publish_test_coverage() } finally { @@ -490,8 +505,7 @@ core_logic: { node(NODE_LINUX_GPU) { ws('workspace/ut-python3-gpu') { try { - utils.init_git() - utils.unpack_lib('gpu', mx_lib) + utils.unpack_and_init('gpu', mx_lib) python3_gpu_ut('ubuntu_gpu') utils.publish_test_coverage() } finally { @@ -505,8 +519,7 @@ core_logic: { ws('workspace/ut-python2-quantize-gpu') { timeout(time: max_time, unit: 'MINUTES') { try { - utils.init_git() - utils.unpack_lib('gpu', mx_lib) + utils.unpack_and_init('gpu', mx_lib) utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_python2_quantization_gpu', true) utils.publish_test_coverage() } finally { @@ -521,8 +534,7 @@ core_logic: { ws('workspace/ut-python3-quantize-gpu') { timeout(time: max_time, unit: 'MINUTES') { try { - utils.init_git() - utils.unpack_lib('gpu', mx_lib) + utils.unpack_and_init('gpu', mx_lib) utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_python3_quantization_gpu', true) utils.publish_test_coverage() } finally { @@ -536,8 +548,7 @@ core_logic: { node(NODE_LINUX_CPU) { ws('workspace/ut-python2-mkldnn-cpu') { try { - utils.init_git() - utils.unpack_lib('mkldnn_cpu', mx_mkldnn_lib) + utils.unpack_and_init('mkldnn_cpu', mx_mkldnn_lib) python2_ut('ubuntu_cpu') utils.publish_test_coverage() } finally { @@ -552,8 +563,7 @@ core_logic: { node(NODE_LINUX_GPU) { ws('workspace/ut-python2-mkldnn-gpu') { try { - utils.init_git() - utils.unpack_lib('mkldnn_gpu', mx_mkldnn_lib) + utils.unpack_and_init('mkldnn_gpu', mx_mkldnn_lib) python2_gpu_ut('ubuntu_gpu') utils.publish_test_coverage() } finally { @@ -566,8 +576,7 @@ core_logic: { node(NODE_LINUX_CPU) { ws('workspace/ut-python3-mkldnn-cpu') { try { - utils.init_git() - utils.unpack_lib('mkldnn_cpu', mx_mkldnn_lib) + utils.unpack_and_init('mkldnn_cpu', mx_mkldnn_lib) python3_ut_mkldnn('ubuntu_cpu') utils.publish_test_coverage() } finally { @@ -581,8 +590,7 @@ core_logic: { node(NODE_LINUX_GPU) { ws('workspace/ut-python3-mkldnn-gpu') { try { - utils.init_git() - utils.unpack_lib('mkldnn_gpu', mx_mkldnn_lib) + utils.unpack_and_init('mkldnn_gpu', mx_mkldnn_lib) python3_gpu_ut('ubuntu_gpu') utils.publish_test_coverage() } finally { @@ -595,8 +603,7 @@ core_logic: { node(NODE_LINUX_GPU) { ws('workspace/ut-python3-mkldnn-gpu-nocudnn') { try { - utils.init_git() - utils.unpack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib) + utils.unpack_and_init('mkldnn_gpu_nocudnn', mx_mkldnn_lib) python3_gpu_ut_nocudnn('ubuntu_gpu') utils.publish_test_coverage() } finally { @@ -610,8 +617,7 @@ core_logic: { ws('workspace/build-centos7-cpu') { timeout(time: max_time, unit: 'MINUTES') { try { - utils.init_git() - utils.unpack_lib('centos7_cpu', mx_lib) + utils.unpack_and_init('centos7_cpu', mx_lib) utils.docker_run('centos7_cpu', 'unittest_centos7_cpu', false) utils.publish_test_coverage() } finally { @@ -627,8 +633,7 @@ core_logic: { ws('workspace/build-centos7-gpu') { timeout(time: max_time, unit: 'MINUTES') { try { - utils.init_git() - utils.unpack_lib('centos7_gpu', mx_lib) + utils.unpack_and_init('centos7_gpu', mx_lib) utils.docker_run('centos7_gpu', 'unittest_centos7_gpu', true) utils.publish_test_coverage() } finally { @@ -643,8 +648,7 @@ core_logic: { ws('workspace/build-tensorrt') { timeout(time: max_time, unit: 'MINUTES') { try { - utils.init_git() - utils.unpack_lib('tensorrt', mx_tensorrt_lib) + utils.unpack_and_init('tensorrt', mx_tensorrt_lib) utils.docker_run('ubuntu_gpu_tensorrt', 'unittest_ubuntu_tensorrt_gpu', true) utils.publish_test_coverage() } finally { @@ -658,8 +662,7 @@ core_logic: { node(NODE_LINUX_CPU) { ws('workspace/ut-scala-cpu') { timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('cpu', mx_dist_lib) + utils.unpack_and_init('cpu', mx_dist_lib) utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_scala', false) utils.publish_test_coverage() } @@ -670,8 +673,7 @@ core_logic: { node(NODE_LINUX_CPU) { ws('workspace/ut-clojure-cpu') { timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('cpu', mx_dist_lib) + utils.unpack_and_init('cpu', mx_dist_lib) utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_clojure', false) utils.publish_test_coverage() } @@ -682,8 +684,7 @@ core_logic: { node(NODE_LINUX_CPU) { ws('workspace/ut-perl-cpu') { timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('cpu', mx_lib) + utils.unpack_and_init('cpu', mx_lib) utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpugpu_perl', false) utils.publish_test_coverage() } @@ -694,8 +695,7 @@ core_logic: { node(NODE_LINUX_GPU) { ws('workspace/ut-perl-gpu') { timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('gpu', mx_lib) + utils.unpack_and_init('gpu', mx_lib) utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_cpugpu_perl', true) utils.publish_test_coverage() } @@ -706,8 +706,7 @@ core_logic: { node(NODE_LINUX_GPU) { ws('workspace/ut-cpp-gpu') { timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('cmake_gpu', mx_cmake_lib) + utils.unpack_and_init('cmake_gpu', mx_cmake_lib) utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true) utils.publish_test_coverage() } @@ -718,8 +717,7 @@ core_logic: { node(NODE_LINUX_GPU) { ws('workspace/ut-cpp-mkldnn-gpu') { timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib) + utils.unpack_and_init('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib) utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true) utils.publish_test_coverage() } @@ -730,8 +728,7 @@ core_logic: { node(NODE_LINUX_CPU) { ws('workspace/ut-r-cpu') { timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('cpu', mx_lib) + utils.unpack_and_init('cpu', mx_lib) utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_R', false) utils.publish_test_coverage() } @@ -742,8 +739,7 @@ core_logic: { node(NODE_LINUX_GPU) { ws('workspace/ut-r-gpu') { timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('gpu', mx_lib) + utils.unpack_and_init('gpu', mx_lib) utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_R', true) utils.publish_test_coverage() } @@ -833,8 +829,7 @@ core_logic: { node(NODE_LINUX_CPU) { ws('workspace/it-onnx-cpu') { timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('cpu', mx_lib) + utils.unpack_and_init('cpu', mx_lib) utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx', false) utils.publish_test_coverage() } @@ -845,14 +840,24 @@ core_logic: { node(NODE_LINUX_GPU) { ws('workspace/it-python-gpu') { timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('gpu', mx_lib) + utils.unpack_and_init('gpu', mx_lib) utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_python', true) utils.publish_test_coverage() } } } }, + 'cpp-package GPU': { + node(NODE_LINUX_GPU) { + ws('workspace/it-cpp-package') { + timeout(time: max_time, unit: 'MINUTES') { + utils.unpack_and_init('gpu', mx_lib_cpp_examples) + utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_cpp_package', true) + utils.publish_test_coverage() + } + } + } + }, // Disabled due to: https://github.com/apache/incubator-mxnet/issues/11407 // 'Caffe GPU': { // node(NODE_LINUX_GPU) { @@ -866,34 +871,11 @@ core_logic: { // } // } // }, - 'cpp-package GPU': { - node(NODE_LINUX_GPU) { - ws('workspace/it-cpp-package') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('gpu', mx_lib) - unstash 'cpp_lenet' - unstash 'cpp_alexnet' - unstash 'cpp_googlenet' - unstash 'cpp_lenet_with_mxdataiter' - unstash 'cpp_resnet' - unstash 'cpp_mlp' - unstash 'cpp_mlp_cpu' - unstash 'cpp_mlp_gpu' - unstash 'cpp_test_score' - unstash 'cpp_test_optimizer' - utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_cpp_package', true) - utils.publish_test_coverage() - } - } - } - }, 'dist-kvstore tests GPU': { node(NODE_LINUX_GPU) { ws('workspace/it-dist-kvstore') { timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('gpu', mx_lib) + utils.unpack_and_init('gpu', mx_lib) utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_dist_kvstore', true) utils.publish_test_coverage() } @@ -908,8 +890,7 @@ core_logic: { node(NODE_LINUX_CPU) { ws('workspace/it-dist-kvstore') { timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('cpu', mx_lib) + utils.unpack_and_init('cpu', mx_lib) utils.docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_dist_kvstore', false) utils.publish_test_coverage() } @@ -920,8 +901,7 @@ core_logic: { node(NODE_LINUX_GPU) { ws('workspace/ut-scala-gpu') { timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.unpack_lib('gpu', mx_dist_lib) + utils.unpack_and_init('gpu', mx_dist_lib) utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_scala', true) utils.publish_test_coverage() } diff --git a/ci/Jenkinsfile_utils.groovy b/ci/Jenkinsfile_utils.groovy index 571c0b952e3d..ca5f234a2c57 100644 --- a/ci/Jenkinsfile_utils.groovy +++ b/ci/Jenkinsfile_utils.groovy @@ -66,7 +66,8 @@ echo ${libs} | sed -e 's/,/ /g' | xargs md5sum } // unpack libraries saved before -def unpack_lib(name, libs) { +def unpack_and_init(name, libs) { + init_git() unstash name sh """ echo "Unpacked ${libs} from ${name}" diff --git a/ci/build.py b/ci/build.py index df9e97bdb5fd..b7c86adda715 100755 --- a/ci/build.py +++ b/ci/build.py @@ -246,6 +246,8 @@ def container_run(platform: str, docker_cmd_list = [ get_docker_binary(nvidia_runtime), 'run', + "--cap-add", + "SYS_PTRACE", # Required by ASAN '--rm', '--shm-size={}'.format(shared_memory_size), # mount mxnet root @@ -287,6 +289,7 @@ def container_run(platform: str, command=command, shm_size=shared_memory_size, user='{}:{}'.format(os.getuid(), os.getgid()), + cap_add='SYS_PTRACE', volumes={ mx_root: {'bind': '/work/mxnet', 'mode': 'rw'}, diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu index 6751465758c8..f45c8da4af87 100755 --- a/ci/docker/Dockerfile.build.ubuntu_cpu +++ b/ci/docker/Dockerfile.build.ubuntu_cpu @@ -48,6 +48,9 @@ RUN /work/ubuntu_perl.sh COPY install/ubuntu_clang.sh /work/ RUN /work/ubuntu_clang.sh +COPY install/ubuntu_gcc8.sh /work/ +RUN /work/ubuntu_gcc8.sh + COPY install/ubuntu_mklml.sh /work/ RUN /work/ubuntu_mklml.sh diff --git a/ci/docker/install/ubuntu_gcc8.sh b/ci/docker/install/ubuntu_gcc8.sh new file mode 100755 index 000000000000..0a9dc605745a --- /dev/null +++ b/ci/docker/install/ubuntu_gcc8.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +sudo add-apt-repository ppa:jonathonf/gcc-8.0 +sudo add-apt-repository ppa:jonathonf/gcc-7.3 +sudo apt-get update +sudo apt-get install -y gcc-8 g++-8 diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index be12e0561916..f0a7e21c5095 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -336,6 +336,35 @@ build_ubuntu_cpu_cmake_debug() { popd } +build_ubuntu_cpu_cmake_asan() { + set -ex + + pushd . + cd /work/build + export CC="gcc-8" + export CXX="g++-8" + cmake \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DUSE_CUDA=OFF \ + -DUSE_MKL_IF_AVAILABLE=OFF \ + -DUSE_OPENMP=OFF \ + -DUSE_OPENCV=OFF \ + -DCMAKE_BUILD_TYPE=Debug \ + -DUSE_GPERFTOOLS=OFF \ + -DUSE_JEMALLOC=OFF \ + -DUSE_ASAN=ON \ + -DUSE_CPP_PACKAGE=ON \ + -DMXNET_USE_CPU=ON \ + /work/mxnet + make -j $(nproc) mxnet + # Disable leak detection but enable ASAN to link with ASAN but not fail with build tooling. + ASAN_OPTIONS=detect_leaks=0 \ + LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.5 \ + make -j $(nproc) mlp_cpu + popd +} + build_ubuntu_cpu_clang39() { set -ex export CXX=clang++-3.9 @@ -797,6 +826,14 @@ integrationtest_ubuntu_gpu_caffe() { python tools/caffe_converter/test_converter.py } +integrationtest_ubuntu_cpu_asan() { + set -ex + export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.5 + cd /work/mxnet/build/cpp-package/example/ + /work/mxnet/cpp-package/example/get_data.sh + ./mlp_cpu +} + integrationtest_ubuntu_gpu_cpp_package() { set -ex cpp-package/tests/ci_test.sh From 3a1565dde92365a506ca12c4802372eeaa089e53 Mon Sep 17 00:00:00 2001 From: Kellen Sunderland Date: Tue, 18 Sep 2018 12:00:53 -0700 Subject: [PATCH 2/4] [MXNET-953] Use gcc8 via cmake options Use CMake to specify the compiler used for ASAN. This is more portable and doesn't rely on env vars. --- ci/docker/runtime_functions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index f0a7e21c5095..f16030e14c93 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -341,11 +341,11 @@ build_ubuntu_cpu_cmake_asan() { pushd . cd /work/build - export CC="gcc-8" - export CXX="g++-8" cmake \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER=/usr/bin/g++-8 \ + -DCMAKE_C_COMPILER=/usr/bin/gcc-8 \ -DUSE_CUDA=OFF \ -DUSE_MKL_IF_AVAILABLE=OFF \ -DUSE_OPENMP=OFF \ From b09519295ff3c87b17049dbda942dd75fd5650df Mon Sep 17 00:00:00 2001 From: Kellen Sunderland Date: Tue, 18 Sep 2018 15:49:17 -0700 Subject: [PATCH 3/4] [MXNET-953] Don't fail build on leak detection --- ci/docker/runtime_functions.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index f16030e14c93..f5241d539019 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -829,6 +829,9 @@ integrationtest_ubuntu_gpu_caffe() { integrationtest_ubuntu_cpu_asan() { set -ex export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.5 + + # We do not want to fail the build on ASAN errors until memory leaks have been addressed. + export ASAN_OPTIONS=exitcode=0 cd /work/mxnet/build/cpp-package/example/ /work/mxnet/cpp-package/example/get_data.sh ./mlp_cpu From 0933b08af0bf992b6e966617a5d648e38ac9c571 Mon Sep 17 00:00:00 2001 From: Kellen Sunderland Date: Tue, 18 Sep 2018 17:36:12 -0700 Subject: [PATCH 4/4] [MXNET-953] Include deps required by scala test --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index eca32f9b8b91..bc033aaa117d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -31,7 +31,7 @@ mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-c mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0' mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so' -mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, build/cpp-package/example/lenet, build/cpp-package/example/alexnet, build/cpp-package/example/googlenet, build/cpp-package/example/lenet_with_mxdataiter, build/cpp-package/example/resnet, build/cpp-package/example/mlp, build/cpp-package/example/mlp_cpu, build/cpp-package/example/mlp_gpu, build/cpp-package/example/test_score, build/cpp-package/example/test_optimizer' +mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/lenet, build/cpp-package/example/alexnet, build/cpp-package/example/googlenet, build/cpp-package/example/lenet_with_mxdataiter, build/cpp-package/example/resnet, build/cpp-package/example/mlp, build/cpp-package/example/mlp_cpu, build/cpp-package/example/mlp_gpu, build/cpp-package/example/test_score, build/cpp-package/example/test_optimizer' mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/mlp_cpu' // timeout in minutes