diff --git a/3rdparty/openmp b/3rdparty/openmp index 37c72127e903..b76842ed1698 160000 --- a/3rdparty/openmp +++ b/3rdparty/openmp @@ -1 +1 @@ -Subproject commit 37c72127e90360a020f351f18d9cccfc30e5145a +Subproject commit b76842ed16984ae5edcbbc4b00a94fda20419431 diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d329f5f1079..8f0974df1dc6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2) +cmake_minimum_required(VERSION 3.13) # workaround to store CMAKE_CROSSCOMPILING because is getting reset by the project command if(CMAKE_CROSSCOMPILING) @@ -240,7 +240,7 @@ if(USE_TENSORRT) endif() # please note that when you enable this, you might run into an linker not being able to work properly due to large code injection. -# you can find more information here https://github.com/apache/incubator-mxnet/issues/15971 +# you can find more information here https://github.com/apache/incubator-mxnet/issues/15971 if(ENABLE_TESTCOVERAGE) message(STATUS "Compiling with test coverage support enabled. This will result in additional files being written to your source directory!") find_program( GCOV_PATH gcov ) @@ -436,6 +436,16 @@ endif() # ---[ OpenMP if(USE_OPENMP) + + function(load_omp) + # Intel/llvm OpenMP: https://github.com/llvm-mirror/openmp + set(OPENMP_STANDALONE_BUILD TRUE) + set(LIBOMP_ENABLE_SHARED TRUE) + set(CMAKE_BUILD_TYPE Release) + set(OPENMP_ENABLE_LIBOMPTARGET OFF CACHE BOOL "LLVM OpenMP offloading support") # Requires CMP0077 CMake 3.13 + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp) + endfunction() + find_package(OpenMP REQUIRED) # This should build on Windows, but there's some problem and I don't have a Windows box, so # could a Windows user please fix? @@ -443,11 +453,7 @@ if(USE_OPENMP) AND SYSTEM_ARCHITECTURE STREQUAL "x86_64" AND NOT MSVC AND NOT CMAKE_CROSSCOMPILING) - - # Intel/llvm OpenMP: https://github.com/llvm-mirror/openmp - set(OPENMP_STANDALONE_BUILD TRUE) - set(LIBOMP_ENABLE_SHARED TRUE) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp) + load_omp() list(REMOVE_ITEM mxnet_LINKER_LIBS iomp5) list(APPEND mxnet_LINKER_LIBS omp) if(UNIX) diff --git a/ci/build_windows.py b/ci/build_windows.py index ce77c316ab20..5839e8d793d1 100755 --- a/ci/build_windows.py +++ b/ci/build_windows.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python # -*- coding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one @@ -28,7 +28,9 @@ import platform import shutil import sys +import tempfile import time +import zipfile from distutils.dir_util import copy_tree from enum import Enum from subprocess import check_call @@ -147,22 +149,33 @@ def windows_build(args): mxnet_root = get_mxnet_root() logging.info("Found MXNet root: {}".format(mxnet_root)) - with remember_cwd(): - os.chdir(path) - cmd = "\"{}\" && cmake -G \"NMake Makefiles JOM\" {} {}".format(args.vcvars, - CMAKE_FLAGS[args.flavour], - mxnet_root) - logging.info("Generating project with CMake:\n{}".format(cmd)) - check_call(cmd, shell=True) - - cmd = "\"{}\" && jom".format(args.vcvars) - logging.info("Building with jom:\n{}".format(cmd)) - - t0 = int(time.time()) - check_call(cmd, shell=True) - - logging.info("Build flavour: {} complete in directory: \"{}\"".format(args.flavour, os.path.abspath(path))) - logging.info("Build took {}".format(datetime.timedelta(seconds=int(time.time() - t0)))) + url = 'https://github.com/Kitware/CMake/releases/download/v3.16.1/cmake-3.16.1-win64-x64.zip' + with tempfile.TemporaryDirectory() as tmpdir: + cmake_file_path = download_file(url, tmpdir) + with zipfile.ZipFile(cmake_file_path, 'r') as zip_ref: + # Create $tmpdir\cmake-3.16.1-win64-x64\bin\cmake.exe + zip_ref.extractall(tmpdir) + + with remember_cwd(): + os.chdir(path) + cmd = "\"{}\" && {} -G \"NMake Makefiles JOM\" {} {}".format( + args.vcvars, + os.path.join(tmpdir, 'cmake-3.16.1-win64-x64', 'bin', 'cmake.exe'), + CMAKE_FLAGS[args.flavour], mxnet_root) + logging.info("Generating project with CMake:\n{}".format(cmd)) + check_call(cmd, shell=True) + + cmd = "\"{}\" && jom".format(args.vcvars) + logging.info("Building with jom:\n{}".format(cmd)) + + t0 = int(time.time()) + check_call(cmd, shell=True) + + logging.info( + "Build flavour: {} complete in directory: \"{}\"".format( + args.flavour, os.path.abspath(path))) + logging.info("Build took {}".format( + datetime.timedelta(seconds=int(time.time() - t0)))) windows_package(args) @@ -262,4 +275,3 @@ def main(): if __name__ == '__main__': sys.exit(main()) - diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android_armv7 index a2e98cd2efe1..2c923a015b63 100644 --- a/ci/docker/Dockerfile.build.android_armv7 +++ b/ci/docker/Dockerfile.build.android_armv7 @@ -18,7 +18,7 @@ # # Dockerfile to build MXNet for Android ARMv7 -FROM mxnetcipinned/dockcross-base:11262018 +FROM dockcross/base MAINTAINER Pedro Larroy "pllarroy@amazon.com" # The cross-compiling emulator diff --git a/ci/docker/Dockerfile.build.android_armv8 b/ci/docker/Dockerfile.build.android_armv8 index f7de86763457..ca62288129bb 100644 --- a/ci/docker/Dockerfile.build.android_armv8 +++ b/ci/docker/Dockerfile.build.android_armv8 @@ -18,7 +18,7 @@ # # Dockerfile to build MXNet for Android ARM64/ARMv8 -FROM mxnetcipinned/dockcross-base:11262018 +FROM dockcross/base MAINTAINER Pedro Larroy "pllarroy@amazon.com" RUN apt-get update && apt-get install -y \ @@ -82,4 +82,4 @@ RUN /work/ubuntu_adduser.sh COPY runtime_functions.sh /work/ -WORKDIR /work/build \ No newline at end of file +WORKDIR /work/build diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6 index 60e223b7a60f..e6a7ffe758b9 100644 --- a/ci/docker/Dockerfile.build.armv6 +++ b/ci/docker/Dockerfile.build.armv6 @@ -18,7 +18,7 @@ # # Dockerfile to build MXNet for ARMv6 -FROM mxnetcipinned/dockcross-linux-armv6:11262018 +FROM dockcross/linux-armv6 ENV ARCH armv6l ENV HOSTCC gcc diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7 index 0b557d5839e9..bad9ab214050 100644 --- a/ci/docker/Dockerfile.build.armv7 +++ b/ci/docker/Dockerfile.build.armv7 @@ -18,7 +18,7 @@ # # Dockerfile to build MXNet for Android ARMv7 -FROM mxnetcipinned/dockcross-linux-armv7:11262018 +FROM dockcross/linux-armv7 ENV ARCH armv7l ENV HOSTCC gcc diff --git a/ci/docker/Dockerfile.build.armv8 b/ci/docker/Dockerfile.build.armv8 index ef9c95865590..bd2373180f0b 100644 --- a/ci/docker/Dockerfile.build.armv8 +++ b/ci/docker/Dockerfile.build.armv8 @@ -18,7 +18,7 @@ # # Dockerfile to build MXNet for ARM64/ARMv8 -FROM mxnetcipinned/dockcross-linux-arm64:11262018 +FROM dockcross/linux-arm64 ENV ARCH aarch64 ENV HOSTCC gcc diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson index 07097887f87d..e31ee43a93d8 100644 --- a/ci/docker/Dockerfile.build.jetson +++ b/ci/docker/Dockerfile.build.jetson @@ -22,7 +22,7 @@ FROM nvidia/cuda:9.0-cudnn7-devel as cudabuilder -FROM mxnetcipinned/dockcross-linux-arm64:11262018 +FROM dockcross/linux-arm64 ENV ARCH aarch64 ENV HOSTCC gcc diff --git a/ci/docker/install/requirements b/ci/docker/install/requirements index cbfc521e2c08..fd716f5fa815 100644 --- a/ci/docker/install/requirements +++ b/ci/docker/install/requirements @@ -26,8 +26,8 @@ h5py==2.8.0rc1 mock==2.0.0 nose==1.3.7 nose-timer==0.7.3 -numpy>1.16.0,<2.0.0 +numpy>1.16.0,<1.18.0 pylint==2.3.1; python_version >= '3.0' requests<2.19.0,>=2.18.4 -scipy==1.0.1 +scipy==1.2.1 six==1.11.0 diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh index 3cb806e0aadd..87e9dec65557 100755 --- a/ci/docker/install/ubuntu_core.sh +++ b/ci/docker/install/ubuntu_core.sh @@ -53,11 +53,10 @@ apt-get install -y \ ln -s /usr/lib/x86_64-linux-gnu/libturbojpeg.so.0.1.0 /usr/lib/x86_64-linux-gnu/libturbojpeg.so -# Note: we specify an exact cmake version to work around a cmake 3.10 CUDA 10 issue. -# Reference: https://github.com/clab/dynet/issues/1457 +# CMake 3.13.2+ is required mkdir /opt/cmake && cd /opt/cmake -wget -nv https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh -sh cmake-3.12.4-Linux-x86_64.sh --prefix=/opt/cmake --skip-license +wget -nv https://cmake.org/files/v3.13/cmake-3.13.5-Linux-x86_64.sh +sh cmake-3.13.5-Linux-x86_64.sh --prefix=/opt/cmake --skip-license ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake -rm cmake-3.12.4-Linux-x86_64.sh +rm cmake-3.13.5-Linux-x86_64.sh cmake --version diff --git a/ci/util.py b/ci/util.py index 4b3a399184f9..cd5665d04df8 100644 --- a/ci/util.py +++ b/ci/util.py @@ -15,12 +15,15 @@ # specific language governing permissions and limitations # under the License. -import os import contextlib import logging import logging.config +import os +import subprocess import sys +import requests + def get_mxnet_root() -> str: curpath = os.path.abspath(os.path.dirname(__file__)) @@ -130,3 +133,31 @@ def config_logging(): # or sensitive information logging.getLogger("botocore").setLevel(logging.WARNING) logging.getLogger("requests").setLevel(logging.WARNING) + + +# Takes url and downloads it to the dest_path directory on Windows. +def download_file(url, dest_path): + file_name = url.split('/')[-1] + full_path = "{}\\{}".format(dest_path, file_name) + logging.info("Downloading: {}".format(full_path)) + r = requests.get(url, stream=True) + if r.status_code == 404: + return r.status_code + elif r.status_code != 200: + logging.error("{} returned status code {}".format(url, r.status_code)) + with open(full_path, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + return full_path + + +# Takes arguments and runs command on host. Shell is disabled by default. +def run_command(args, shell=False): + try: + logging.info("Issuing command: {}".format(args)) + res = subprocess.check_output(args, shell=shell, timeout=1800).decode("utf-8").replace("\r\n", "") + logging.info("Output: {}".format(res)) + except subprocess.CalledProcessError as e: + raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) + return res diff --git a/src/engine/openmp.cc b/src/engine/openmp.cc index 8946d91ef718..0d31f71aa9a3 100644 --- a/src/engine/openmp.cc +++ b/src/engine/openmp.cc @@ -41,6 +41,7 @@ OpenMP *OpenMP::Get() { OpenMP::OpenMP() : omp_num_threads_set_in_environment_(is_env_set("OMP_NUM_THREADS")) { #ifdef _OPENMP + initialize_process(); const int max = dmlc::GetEnv("MXNET_OMP_MAX_THREADS", INT_MIN); if (max != INT_MIN) { omp_thread_max_ = max; @@ -61,6 +62,12 @@ OpenMP::OpenMP() #endif } +void OpenMP:: initialize_process() { +#ifdef _OPENMP + omp_get_num_procs(); // will force OpenMP to be initialized +#endif +} + void OpenMP::on_start_worker_thread(bool use_omp) { #ifdef _OPENMP if (!omp_num_threads_set_in_environment_) { diff --git a/src/engine/openmp.h b/src/engine/openmp.h index 800ea2f91b62..94b83e3aa25b 100644 --- a/src/engine/openmp.h +++ b/src/engine/openmp.h @@ -74,6 +74,13 @@ class OpenMP { */ void on_start_worker_thread(bool use_omp); + /*! + * \brief Initialize a new process to use omp (after a fork, + * in case you're starting threads in the atfork() that may interfere + * with the initialization. Can serialize the init with this first. + */ + void initialize_process(); + /*! * \brief Get the OpenMP object's singleton pointer * \return Singleton OpenMP object pointer diff --git a/src/initialize.cc b/src/initialize.cc index 071e8d32e548..a3dbce22a384 100644 --- a/src/initialize.cc +++ b/src/initialize.cc @@ -209,6 +209,7 @@ void LibraryInitializer::atfork_child() { #if MXNET_USE_OPENCV && !__APPLE__ cv::setNumThreads(mp_cv_num_threads_); #endif // MXNET_USE_OPENCV + engine::OpenMP::Get()->initialize_process(); engine::OpenMP::Get()->set_thread_max(1); engine::OpenMP::Get()->set_enabled(false); Engine::Get()->Start(); @@ -218,6 +219,7 @@ void LibraryInitializer::atfork_child() { void LibraryInitializer::install_pthread_atfork_handlers() { #ifndef _WIN32 + engine::OpenMP::Get()->initialize_process(); // force omp to set its atfork handler first pthread_atfork(pthread_atfork_prepare, pthread_atfork_parent, pthread_atfork_child); #endif } diff --git a/tests/python/unittest/test_metric.py b/tests/python/unittest/test_metric.py index a1e5128d8ac6..e7273fba35d5 100644 --- a/tests/python/unittest/test_metric.py +++ b/tests/python/unittest/test_metric.py @@ -18,6 +18,7 @@ import mxnet as mx import numpy as np import scipy +from scipy.stats import pearsonr import json import math from common import with_seed @@ -267,7 +268,7 @@ def test_pearsonr(): pred1 = mx.nd.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]]) label1 = mx.nd.array([[1, 0], [0, 1], [0, 1]]) pearsonr_expected_np = np.corrcoef(pred1.asnumpy().ravel(), label1.asnumpy().ravel())[0, 1] - pearsonr_expected_scipy, _ = scipy.stats.pearsonr(pred1.asnumpy().ravel(), label1.asnumpy().ravel()) + pearsonr_expected_scipy, _ = pearsonr(pred1.asnumpy().ravel(), label1.asnumpy().ravel()) macro_pr = mx.metric.create('pearsonr', average='macro') micro_pr = mx.metric.create('pearsonr', average='micro') @@ -289,7 +290,7 @@ def test_pearsonr(): label12 = mx.nd.array([[1, 0], [0, 1], [0, 1], [1, 0], [0, 1], [0, 1]]) pearsonr_expected_np = np.corrcoef(pred12.asnumpy().ravel(), label12.asnumpy().ravel())[0, 1] - pearsonr_expected_scipy, _ = scipy.stats.pearsonr(pred12.asnumpy().ravel(), label12.asnumpy().ravel()) + pearsonr_expected_scipy, _ = pearsonr(pred12.asnumpy().ravel(), label12.asnumpy().ravel()) macro_pr.reset() micro_pr.update([label2], [pred2]) diff --git a/tests/python/unittest/test_numpy_interoperability.py b/tests/python/unittest/test_numpy_interoperability.py index da3c17cbb7a7..ed97f6b1fcf0 100644 --- a/tests/python/unittest/test_numpy_interoperability.py +++ b/tests/python/unittest/test_numpy_interoperability.py @@ -576,7 +576,7 @@ def _add_workload_reshape(): # OpArgMngr.add_workload('reshape', b, (2, 2), order='F') # Items are not equal with order='F' a = np.array(_np.ones((0, 2))) - OpArgMngr.add_workload('reshape', a, -1, 2) + OpArgMngr.add_workload('reshape', a, (-1, 2)) def _add_workload_rint(array_pool):