diff --git a/CMakeLists.txt b/CMakeLists.txt index fcbe707aa9ef..f0779abd06d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2) +cmake_minimum_required(VERSION 3.13) # workaround to store CMAKE_CROSSCOMPILING because is getting reset by the project command if(CMAKE_CROSSCOMPILING) @@ -452,6 +452,7 @@ if(USE_OPENMP) set(OPENMP_STANDALONE_BUILD TRUE) set(LIBOMP_ENABLE_SHARED TRUE) set(CMAKE_BUILD_TYPE Release) + set(OPENMP_ENABLE_LIBOMPTARGET OFF CACHE BOOL "LLVM OpenMP offloading support") # Requires CMP0077 CMake 3.13 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp) endfunction() diff --git a/ci/build_windows.py b/ci/build_windows.py index ce77c316ab20..5839e8d793d1 100755 --- a/ci/build_windows.py +++ b/ci/build_windows.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python # -*- coding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one @@ -28,7 +28,9 @@ import platform import shutil import sys +import tempfile import time +import zipfile from distutils.dir_util import copy_tree from enum import Enum from subprocess import check_call @@ -147,22 +149,33 @@ def windows_build(args): mxnet_root = get_mxnet_root() logging.info("Found MXNet root: {}".format(mxnet_root)) - with remember_cwd(): - os.chdir(path) - cmd = "\"{}\" && cmake -G \"NMake Makefiles JOM\" {} {}".format(args.vcvars, - CMAKE_FLAGS[args.flavour], - mxnet_root) - logging.info("Generating project with CMake:\n{}".format(cmd)) - check_call(cmd, shell=True) - - cmd = "\"{}\" && jom".format(args.vcvars) - logging.info("Building with jom:\n{}".format(cmd)) - - t0 = int(time.time()) - check_call(cmd, shell=True) - - logging.info("Build flavour: {} complete in directory: \"{}\"".format(args.flavour, os.path.abspath(path))) - logging.info("Build took {}".format(datetime.timedelta(seconds=int(time.time() - t0)))) + url = 'https://github.com/Kitware/CMake/releases/download/v3.16.1/cmake-3.16.1-win64-x64.zip' + with tempfile.TemporaryDirectory() as tmpdir: + cmake_file_path = download_file(url, tmpdir) + with zipfile.ZipFile(cmake_file_path, 'r') as zip_ref: + # Create $tmpdir\cmake-3.16.1-win64-x64\bin\cmake.exe + zip_ref.extractall(tmpdir) + + with remember_cwd(): + os.chdir(path) + cmd = "\"{}\" && {} -G \"NMake Makefiles JOM\" {} {}".format( + args.vcvars, + os.path.join(tmpdir, 'cmake-3.16.1-win64-x64', 'bin', 'cmake.exe'), + CMAKE_FLAGS[args.flavour], mxnet_root) + logging.info("Generating project with CMake:\n{}".format(cmd)) + check_call(cmd, shell=True) + + cmd = "\"{}\" && jom".format(args.vcvars) + logging.info("Building with jom:\n{}".format(cmd)) + + t0 = int(time.time()) + check_call(cmd, shell=True) + + logging.info( + "Build flavour: {} complete in directory: \"{}\"".format( + args.flavour, os.path.abspath(path))) + logging.info("Build took {}".format( + datetime.timedelta(seconds=int(time.time() - t0)))) windows_package(args) @@ -262,4 +275,3 @@ def main(): if __name__ == '__main__': sys.exit(main()) - diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android_armv7 index a2e98cd2efe1..2c923a015b63 100644 --- a/ci/docker/Dockerfile.build.android_armv7 +++ b/ci/docker/Dockerfile.build.android_armv7 @@ -18,7 +18,7 @@ # # Dockerfile to build MXNet for Android ARMv7 -FROM mxnetcipinned/dockcross-base:11262018 +FROM dockcross/base MAINTAINER Pedro Larroy "pllarroy@amazon.com" # The cross-compiling emulator diff --git a/ci/docker/Dockerfile.build.android_armv8 b/ci/docker/Dockerfile.build.android_armv8 index f7de86763457..ca62288129bb 100644 --- a/ci/docker/Dockerfile.build.android_armv8 +++ b/ci/docker/Dockerfile.build.android_armv8 @@ -18,7 +18,7 @@ # # Dockerfile to build MXNet for Android ARM64/ARMv8 -FROM mxnetcipinned/dockcross-base:11262018 +FROM dockcross/base MAINTAINER Pedro Larroy "pllarroy@amazon.com" RUN apt-get update && apt-get install -y \ @@ -82,4 +82,4 @@ RUN /work/ubuntu_adduser.sh COPY runtime_functions.sh /work/ -WORKDIR /work/build \ No newline at end of file +WORKDIR /work/build diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6 index 60e223b7a60f..e6a7ffe758b9 100644 --- a/ci/docker/Dockerfile.build.armv6 +++ b/ci/docker/Dockerfile.build.armv6 @@ -18,7 +18,7 @@ # # Dockerfile to build MXNet for ARMv6 -FROM mxnetcipinned/dockcross-linux-armv6:11262018 +FROM dockcross/linux-armv6 ENV ARCH armv6l ENV HOSTCC gcc diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7 index 0b557d5839e9..bad9ab214050 100644 --- a/ci/docker/Dockerfile.build.armv7 +++ b/ci/docker/Dockerfile.build.armv7 @@ -18,7 +18,7 @@ # # Dockerfile to build MXNet for Android ARMv7 -FROM mxnetcipinned/dockcross-linux-armv7:11262018 +FROM dockcross/linux-armv7 ENV ARCH armv7l ENV HOSTCC gcc diff --git a/ci/docker/Dockerfile.build.armv8 b/ci/docker/Dockerfile.build.armv8 index ef9c95865590..bd2373180f0b 100644 --- a/ci/docker/Dockerfile.build.armv8 +++ b/ci/docker/Dockerfile.build.armv8 @@ -18,7 +18,7 @@ # # Dockerfile to build MXNet for ARM64/ARMv8 -FROM mxnetcipinned/dockcross-linux-arm64:11262018 +FROM dockcross/linux-arm64 ENV ARCH aarch64 ENV HOSTCC gcc diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson index 07097887f87d..e31ee43a93d8 100644 --- a/ci/docker/Dockerfile.build.jetson +++ b/ci/docker/Dockerfile.build.jetson @@ -22,7 +22,7 @@ FROM nvidia/cuda:9.0-cudnn7-devel as cudabuilder -FROM mxnetcipinned/dockcross-linux-arm64:11262018 +FROM dockcross/linux-arm64 ENV ARCH aarch64 ENV HOSTCC gcc diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh index 8426b539f4c4..bd5d1f6fdf6f 100755 --- a/ci/docker/install/ubuntu_core.sh +++ b/ci/docker/install/ubuntu_core.sh @@ -57,11 +57,10 @@ apt-get install -y \ ln -s /usr/lib/x86_64-linux-gnu/libturbojpeg.so.0.1.0 /usr/lib/x86_64-linux-gnu/libturbojpeg.so -# Note: we specify an exact cmake version to work around a cmake 3.10 CUDA 10 issue. -# Reference: https://github.com/clab/dynet/issues/1457 +# CMake 3.13.2+ is required mkdir /opt/cmake && cd /opt/cmake -wget -nv https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh -sh cmake-3.12.4-Linux-x86_64.sh --prefix=/opt/cmake --skip-license +wget -nv https://cmake.org/files/v3.13/cmake-3.13.5-Linux-x86_64.sh +sh cmake-3.13.5-Linux-x86_64.sh --prefix=/opt/cmake --skip-license ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake -rm cmake-3.12.4-Linux-x86_64.sh +rm cmake-3.13.5-Linux-x86_64.sh cmake --version diff --git a/ci/util.py b/ci/util.py index 4b3a399184f9..cd5665d04df8 100644 --- a/ci/util.py +++ b/ci/util.py @@ -15,12 +15,15 @@ # specific language governing permissions and limitations # under the License. -import os import contextlib import logging import logging.config +import os +import subprocess import sys +import requests + def get_mxnet_root() -> str: curpath = os.path.abspath(os.path.dirname(__file__)) @@ -130,3 +133,31 @@ def config_logging(): # or sensitive information logging.getLogger("botocore").setLevel(logging.WARNING) logging.getLogger("requests").setLevel(logging.WARNING) + + +# Takes url and downloads it to the dest_path directory on Windows. +def download_file(url, dest_path): + file_name = url.split('/')[-1] + full_path = "{}\\{}".format(dest_path, file_name) + logging.info("Downloading: {}".format(full_path)) + r = requests.get(url, stream=True) + if r.status_code == 404: + return r.status_code + elif r.status_code != 200: + logging.error("{} returned status code {}".format(url, r.status_code)) + with open(full_path, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + return full_path + + +# Takes arguments and runs command on host. Shell is disabled by default. +def run_command(args, shell=False): + try: + logging.info("Issuing command: {}".format(args)) + res = subprocess.check_output(args, shell=shell, timeout=1800).decode("utf-8").replace("\r\n", "") + logging.info("Output: {}".format(res)) + except subprocess.CalledProcessError as e: + raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) + return res