From 44fb83ddce020db180ee8b3d73b4203d87d9dfbb Mon Sep 17 00:00:00 2001 From: Zach Kimberg Date: Wed, 12 Dec 2018 14:54:44 -0800 Subject: [PATCH] Jenkins nightly maven with static build script and gpu --- .gitignore | 1 + ci/docker/Dockerfile.publish.ubuntu1604_cpu | 42 ++++ ci/docker/Dockerfile.publish.ubuntu1604_gpu | 42 ++++ ci/docker/install/ubuntu_base.sh | 4 + ci/docker/install/ubuntu_scala.sh | 3 - ci/publish/Jenkinsfile | 27 +-- ci/publish/scala/build.sh | 4 +- ci/publish/scala/deploy.sh | 16 +- ci/publish/scala/fullDeploy.sh | 0 ci/publish/scala/test.sh | 9 +- make/maven/maven_linux_cu92.mk | 185 ++++++++++++++++++ scala-package/deploy/pom.xml | 2 +- .../mxnetexamples/rnn/ExampleRNNSuite.scala | 11 +- scala-package/init/pom.xml | 7 + scala-package/packageTest/README.md | 10 +- scala-package/packageTest/examples/pom.xml | 51 ++++- scala-package/packageTest/pom.xml | 2 +- scala-package/pom.xml | 32 ++- tools/dependencies/cityhash.sh | 8 +- tools/dependencies/curl.sh | 8 +- tools/dependencies/eigen.sh | 6 +- tools/dependencies/libpng.sh | 8 +- tools/dependencies/libtiff.sh | 8 +- tools/dependencies/libturbojpeg.sh | 8 +- tools/dependencies/libz.sh | 8 +- tools/dependencies/lz4.sh | 8 +- .../dependencies/make_shared_dependencies.sh | 2 + tools/dependencies/openblas.sh | 9 +- tools/dependencies/opencv.sh | 8 +- tools/dependencies/openssl.sh | 8 +- tools/dependencies/protobuf.sh | 9 +- tools/dependencies/zmq.sh | 9 +- tools/setup_gpu_build_tools.sh | 2 + tools/staticbuild/build.sh | 68 +++++++ tools/{build => staticbuild}/build_lib.sh | 24 +-- tools/{build => staticbuild}/build_wheel.sh | 0 36 files changed, 560 insertions(+), 89 deletions(-) create mode 100644 ci/docker/Dockerfile.publish.ubuntu1604_cpu create mode 100644 ci/docker/Dockerfile.publish.ubuntu1604_gpu mode change 100644 => 100755 ci/publish/scala/fullDeploy.sh create mode 100644 make/maven/maven_linux_cu92.mk create mode 100755 tools/staticbuild/build.sh rename tools/{build => staticbuild}/build_lib.sh (77%) rename tools/{build => staticbuild}/build_wheel.sh (100%) diff --git a/.gitignore b/.gitignore index 7eb8e7d6e777..fc7992675e28 100644 --- a/.gitignore +++ b/.gitignore @@ -146,6 +146,7 @@ tools/pip_package/mxnet # temporary path for building dependencies when building wheel ./deps/ +./staticdeps/ bld ./tmp/* *.jar diff --git a/ci/docker/Dockerfile.publish.ubuntu1604_cpu b/ci/docker/Dockerfile.publish.ubuntu1604_cpu new file mode 100644 index 000000000000..df284be2b37e --- /dev/null +++ b/ci/docker/Dockerfile.publish.ubuntu1604_cpu @@ -0,0 +1,42 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU + +FROM ubuntu:16.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/Dockerfile.publish.ubuntu1604_gpu b/ci/docker/Dockerfile.publish.ubuntu1604_gpu new file mode 100644 index 000000000000..2a1f8594e752 --- /dev/null +++ b/ci/docker/Dockerfile.publish.ubuntu1604_gpu @@ -0,0 +1,42 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to run MXNet on Ubuntu 16.04 for GPU + +FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu16.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ + +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/install/ubuntu_base.sh b/ci/docker/install/ubuntu_base.sh index b34c0b3e18f1..f36e53279f57 100755 --- a/ci/docker/install/ubuntu_base.sh +++ b/ci/docker/install/ubuntu_base.sh @@ -33,4 +33,8 @@ apt-get install -y \ software-properties-common \ sudo \ unzip \ + expect \ + gnupg \ + gnupg2 \ + gnupg-agent \ wget diff --git a/ci/docker/install/ubuntu_scala.sh b/ci/docker/install/ubuntu_scala.sh index 5bade47463e2..8b23d614b490 100755 --- a/ci/docker/install/ubuntu_scala.sh +++ b/ci/docker/install/ubuntu_scala.sh @@ -36,9 +36,6 @@ apt-get install -y \ openjdk-8-jdk \ openjdk-8-jre \ software-properties-common \ - gnupg \ - gnupg2 \ - gnupg-agent \ scala # Ubuntu 14.04 diff --git a/ci/publish/Jenkinsfile b/ci/publish/Jenkinsfile index 9a360c6b5bed..c14ab2502cda 100644 --- a/ci/publish/Jenkinsfile +++ b/ci/publish/Jenkinsfile @@ -21,7 +21,7 @@ // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ //mxnet libraries -mx_scala_pub = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/**, scala-package/local-snapshot/**' +mx_scala_pub = 'lib/**, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/**/target/**, scala-package/*/target/repo/**' // timeout in minutes max_time = 120 @@ -34,8 +34,9 @@ node('restricted-utility') { utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') // CPU and GPU. OSX nodes are not currently supported by Jenkins -def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU] +def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU_P3] def scalaOSMap = ['cpu': 'linux-x86_64-cpu', 'gpu': 'linux-x86_64-gpu'] +def scalaVariantMap = ['cpu': 'cpu', 'gpu': 'cu92'] def wrapStep(nodeToRun, workspaceName, step) { return { @@ -50,13 +51,13 @@ def wrapStep(nodeToRun, workspaceName, step) { } def toBuild = [:] -def labels = ['cpu'] // , 'gpu'] +def labels = ['cpu', 'gpu'] for (x in labels) { def label = x // Required due to language - toBuild["Scala Build ${label}"] = wrapStep(nodeMap[label], "build-scala-${label}") { - withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) { + toBuild["Scala Build ${label}"] = wrapStep(nodeMap['cpu'], "build-scala-${label}") { + withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}", "mxnet_variant=${scalaVariantMap[label]}"]) { utils.init_git() - utils.docker_run("ubuntu_${label}", 'publish_scala_build', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE') + utils.docker_run("publish.ubuntu1404_cpu", 'publish_scala_build', false, '500m', 'MAVEN_PUBLISH_OS_TYPE mxnet_variant') utils.pack_lib("scala_${label}", mx_scala_pub, false) } } @@ -69,8 +70,10 @@ for (x in labels) { for (y in systems) { def system = y // Required due to language toTest["Scala Test ${system} ${label}"] = wrapStep(nodeMap[label], "test-scala-${system}-${label}") { - utils.unpack_and_init("scala_${label}", mx_scala_pub, false) - utils.docker_run("publish.test.${system}_${label}", 'publish_scala_test', label == 'gpu') + withEnv(["mxnet_variant=${scalaVariantMap[label]}"]) { + utils.unpack_and_init("scala_${label}", mx_scala_pub, false) + utils.docker_run("publish.test.${system}_${label}", 'publish_scala_test', label == 'gpu', '500m', 'mxnet_variant') + } } } } @@ -78,10 +81,10 @@ for (x in labels) { def toDeploy = [:] for (x in labels) { def label = x // Required due to language - toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap[label], "deploy-scala-${label}") { - withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) { + toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap['cpu'], "deploy-scala-${label}") { + withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}", "mxnet_variant=${scalaVariantMap[label]}"]) { utils.unpack_and_init("scala_${label}", mx_scala_pub, false) - utils.docker_run("ubuntu_${label}", 'publish_scala_deploy', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION') + utils.docker_run("publish.ubuntu1604_cpu", 'publish_scala_deploy', false, '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION mxnet_variant') } } } @@ -101,7 +104,7 @@ core_logic: { , failure_handler: { if (currentBuild.result == "FAILURE") { - // emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}' + emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}' } } ) diff --git a/ci/publish/scala/build.sh b/ci/publish/scala/build.sh index 17f969afe142..11386ce2114f 100755 --- a/ci/publish/scala/build.sh +++ b/ci/publish/scala/build.sh @@ -22,7 +22,9 @@ set -ex # MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu # export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu -bash scala-package/dev/compile-mxnet-backend.sh $MAVEN_PUBLISH_OS_TYPE ./ +source tools/staticbuild/build.sh $mxnet_variant maven + +set -ex # Compile tests for discovery later cd scala-package diff --git a/ci/publish/scala/deploy.sh b/ci/publish/scala/deploy.sh index 4eb33907eeb5..00c67214742b 100755 --- a/ci/publish/scala/deploy.sh +++ b/ci/publish/scala/deploy.sh @@ -18,12 +18,10 @@ set -ex -# Setup Environment Variables -# MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu -# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu - -# Run python to configure keys -python3 ci/publish/scala/buildkey.py +# On Jenkins, run python script to configure keys +if [[ $BUILD_ID ]]; then + python3 ci/publish/scala/buildkey.py +fi # Updating cache mkdir -p ~/.gnupg @@ -37,5 +35,7 @@ cd scala-package mvn -B deploy -Pnightly -# Clear all password .xml files, exp files, and gpg key files -rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp +# On Jenkins, clear all password .xml files, exp files, and gpg key files +if [[ $BUILD_ID ]]; then + rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp +fi diff --git a/ci/publish/scala/fullDeploy.sh b/ci/publish/scala/fullDeploy.sh old mode 100644 new mode 100755 diff --git a/ci/publish/scala/test.sh b/ci/publish/scala/test.sh index 5cef35ca3c2b..32e36299a6fc 100755 --- a/ci/publish/scala/test.sh +++ b/ci/publish/scala/test.sh @@ -24,5 +24,10 @@ fi # Test cd scala-package/packageTest -# make testlocal CI=1 -make testsnapshot UNIT=1 CI=1 + +if [[ $mxnet_variant == cu* ]]; then + export SCALA_TEST_ON_GPU=1 + make testlocal USE_CUDA=1 CI=1 +else + make testlocal CI=1 +fi diff --git a/make/maven/maven_linux_cu92.mk b/make/maven/maven_linux_cu92.mk new file mode 100644 index 000000000000..0ed18a71b976 --- /dev/null +++ b/make/maven/maven_linux_cu92.mk @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet for making maven package +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with options for MXNet developer +DEV = 0 + +# whether compile with debug +DEBUG = 0 + +# whether compiler with profiler +USE_PROFILER = 1 + +# whether to turn on signal handler (e.g. segfault logger) +USE_SIGNAL_HANDLER = 0 + +# the additional link flags you want to add +ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff -ljpeg -lz -ldl -lgfortran -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections + +# the additional compile flags you want to add +ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +# in default use atlas for linux while apple for osx +USE_BLAS=openblas + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# whether use CUDA during compile +USE_CUDA = 1 + +# add the path to CUDA library to link and compile flag +# if you have already add them to environment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.2 + +# whether to use CuDNN library +USE_CUDNN = 1 + +# whether to use NCCL library +USE_NCCL = 1 + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 lines for compatibility. +# CUDA_ARCH := + +# whether use cuda runtime compiling for writing kernels in native language (i.e. Python) +USE_NVRTC = 1 + +# use openmp for parallelization +USE_OPENMP = 1 +USE_OPERATOR_TUNING = 1 +USE_LIBJPEG_TURBO = 1 + +# whether use MKL-DNN library +USE_MKLDNN = 0 + + +# MKL ML Library for Intel CPU/Xeon Phi +# Please refer to MKL_README.md for details + +# MKL ML Library folder, need to be root for /usr/local +# Change to User Home directory for standard user +# For USE_BLAS!=mkl only +MKLML_ROOT=/usr/local + +# whether use MKL2017 library +USE_MKL2017 = 0 + +# whether use MKL2017 experimental feature for high performance +# Prerequisite USE_MKL2017=1 +USE_MKL2017_EXPERIMENTAL = 0 + +# whether use NNPACK library +USE_NNPACK = 0 + +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = $(DEPS_PATH)/lib + +# add path to intel library, you may need it for MKL, if you did not add the path +# to environment variable +USE_INTEL_PATH = NONE + +# If use MKL, choose static link automatically to allow python wrapper +ifeq ($(USE_BLAS), mkl) +USE_STATIC_MKL = 1 +else +USE_STATIC_MKL = NONE +endif + +#---------------------------- +# Settings for power and arm arch +#---------------------------- +ARCH := $(shell uname -a) +ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64)) + USE_SSE=0 +else + USE_SSE=1 +endif + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable multi-machine supporting +USE_DIST_KVSTORE = 1 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 1 + +#---------------------------- +# additional operators +#---------------------------- + +# path to folders containing projects specific operators that you don't want to put in src/operators +EXTRA_OPERATORS = + + +#---------------------------- +# plugins +#---------------------------- + +# whether to use caffe integration. This requires installing caffe. +# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH +# CAFFE_PATH = $(HOME)/caffe +# MXNET_PLUGINS += plugin/caffe/caffe.mk + +# whether to use torch integration. This requires installing torch. +# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH +# TORCH_PATH = $(HOME)/torch +# MXNET_PLUGINS += plugin/torch/torch.mk + +# WARPCTC_PATH = $(HOME)/warp-ctc +# MXNET_PLUGINS += plugin/warpctc/warpctc.mk + +# whether to use sframe integration. This requires build sframe +# git@github.com:dato-code/SFrame.git +# SFRAME_PATH = $(HOME)/SFrame +# MXNET_PLUGINS += plugin/sframe/plugin.mk diff --git a/scala-package/deploy/pom.xml b/scala-package/deploy/pom.xml index 542bf6c16880..4e9da8908be1 100644 --- a/scala-package/deploy/pom.xml +++ b/scala-package/deploy/pom.xml @@ -23,7 +23,6 @@ mxnet-full_2.11-${platform}-${flavor} 1.5.0-SNAPSHOT apache.snapshots.https - file://${project.build.directory}/repo @@ -118,6 +117,7 @@ ${project.description} + false ${repositoryId} ${repo_url} ${project.groupId} diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala index 14fb7b85e9b3..2ccd38fc4f9c 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala @@ -49,14 +49,19 @@ class ExampleRNNSuite extends FunSuite with BeforeAndAfterAll { System.getenv("SCALA_TEST_ON_GPU").toInt == 1) { ctx = Context.gpu() } - LstmBucketing.runTraining(tempDirPath + "/RNN/sherlockholmes.train.txt", - tempDirPath + "/RNN/sherlockholmes.valid.txt", Array(ctx), 1) + if (!System.getenv().containsKey("CI")) { + LstmBucketing.runTraining(tempDirPath + "/RNN/sherlockholmes.train.txt", + tempDirPath + "/RNN/sherlockholmes.valid.txt", Array(ctx), 1) + } else { + logger.info("Skipping test on CI...") + } } test("Example CI: Test TrainCharRNN") { val tempDirPath = System.getProperty("java.io.tmpdir") if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && - System.getenv("SCALA_TEST_ON_GPU").toInt == 1) { + System.getenv("SCALA_TEST_ON_GPU").toInt == 1 && + !System.getenv().containsKey("CI")) { val ctx = Context.gpu() TrainCharRnn.runTrainCharRnn(tempDirPath + "/RNN/obama.txt", tempDirPath, ctx, 1) diff --git a/scala-package/init/pom.xml b/scala-package/init/pom.xml index a0bb6be384b5..c514177db17c 100644 --- a/scala-package/init/pom.xml +++ b/scala-package/init/pom.xml @@ -60,6 +60,13 @@ + + org.apache.maven.plugins + maven-deploy-plugin + + true + + diff --git a/scala-package/packageTest/README.md b/scala-package/packageTest/README.md index 3f1eeb842f07..e9980f353759 100644 --- a/scala-package/packageTest/README.md +++ b/scala-package/packageTest/README.md @@ -7,7 +7,7 @@ This is an project created to run the test suite on a fully packaged mxnet jar. To setup the packageTest, you must first build your tests. To build the tests, follow these steps from the mxnet main directory: 1. Build MXNet and the scala package from source following the directions [here](https://mxnet.incubator.apache.org/install/scala_setup.html#source) -2. Build the tests by running `make scalatestcompile`. +2. Build the tests by running `mvn test-compile`. 3. Follow setup instructions below for your testing goal ## Running @@ -18,13 +18,13 @@ There are three different modes of operation for testing based on the location o If you have a jar file, you can install it to your maven cache repository(`~/.m2/repository`). This might be useful if you acquire the .jar file from elsewhere. To install, it is easiest to use `mvn install:install-file -Dfile= -DpomFile=`. If the pom file is not available, you can also run `mvn install:install-file -Dfile= -DgroupId= -DartifactId= -Dversion= -Dpackaging=`. With the full mxnet jar, this might look like `mvn install:install-file -Dfile= -DgroupId=org.apache.mxnet -DartifactId=mxnet-full_2.11-linux-x86_64-cpu -Dversion=1.3.0 -Dpackaging=jar`. -You can also run `make scalainstall` to install from a local build. +You can also run `mvn install` to install from a local build. After installing, run `make testinstall` in the package test directory to run the tests. Note that unless you also install an additional mxnetexamples jar, you can only run the unit tests. ### Test Local Deployment -To test the jars that would be produced by a deployment, you can run `make scaladeploylocal` from the main mxnet directory. This produces a local snapshot located at `scala-package/local-snapshot`. To test this local snapshot, run `make testlocal`. +To test the jars that would be produced by a deployment, you can run `mvn deploy` from the main mxnet directory. This produces a local snapshot located at `scala-package/deploy/target/repo`. To test this local snapshot, run `make testlocal`. It also installs the component packages needed for testing the examples in `scala-package/*/target/repo`. ### Remote Repository Snapshot @@ -36,11 +36,13 @@ Test the snapshot repo using `make testsnapshot` or a different repo using `make You are able to run unit tests, integration tests, or both using this utility. To run the unit tests, add the flag `UNIT=1` to make (e.g. `make testsnapshot UNIT=1`). Use `INTEGRATION=1` for integration tests. The default behavior is to run both the unit and integration tests. However, the integration tests require that the mxnet examples be installed in addition to the full mxnet package (see test mode instructions above). +For running on GPU, add the flag `USE_CUDA=1`. + An additional option, you can specify the mxnet version with `MXNET_VERSION=1.3.1-SNAPSHOT`. ## Cleaning Up -You can clean temporary files and target artifacts by running `make scalaclean`. +You can clean temporary files and target artifacts by running `make clean`. ## Troubleshooting diff --git a/scala-package/packageTest/examples/pom.xml b/scala-package/packageTest/examples/pom.xml index e11be657e225..070b78dda99b 100644 --- a/scala-package/packageTest/examples/pom.xml +++ b/scala-package/packageTest/examples/pom.xml @@ -21,6 +21,53 @@ false + + fromLocal + + + parent + file://${basedir}/../../target/repo + + true + + + + init + file://${basedir}/../../init/target/repo + + true + + + + macros + file://${basedir}/../../macros/target/repo + + true + + + + core + file://${basedir}/../../core/target/repo + + true + + + + infer + file://${basedir}/../../infer/target/repo + + true + + + + examples + file://${basedir}/../../examples/target/repo + + true + + + + @@ -39,8 +86,8 @@ org.apache.mxnet - mxnet-examples_${mxnet.scalaprofile} - ${mxnet.version} + mxnet-examples + INTERNAL test diff --git a/scala-package/packageTest/pom.xml b/scala-package/packageTest/pom.xml index f7d9e3b180bc..f6a16dd77c9d 100644 --- a/scala-package/packageTest/pom.xml +++ b/scala-package/packageTest/pom.xml @@ -41,7 +41,7 @@ fromLocal - local-snapshot + full file://${basedir}/../deploy/target/repo true diff --git a/scala-package/pom.xml b/scala-package/pom.xml index d39075f2c99b..07baeab9b63f 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -43,6 +43,7 @@ $ ${project.basedir}/.. true + file://${project.build.directory}/repo pom @@ -158,6 +159,34 @@ + + staging + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + + + nightly + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + @@ -337,7 +366,8 @@ org.apache.maven.plugins maven-deploy-plugin - true + false + deployrepo::default::${repo_url} diff --git a/tools/dependencies/cityhash.sh b/tools/dependencies/cityhash.sh index 81cc9cbaad3a..6bc663e906fa 100755 --- a/tools/dependencies/cityhash.sh +++ b/tools/dependencies/cityhash.sh @@ -18,15 +18,17 @@ # under the License. # This script builds the static library of cityhash that can be used as dependency of mxnet. +set -ex CITYHASH_VERSION=1.1.1 if [[ ! -f $DEPS_PATH/lib/libcityhash.a ]]; then # Download and build cityhash >&2 echo "Building cityhash..." git clone https://github.com/google/cityhash $DEPS_PATH/cityhash-$CITYHASH_VERSION + pushd . cd $DEPS_PATH/cityhash-$CITYHASH_VERSION git reset --hard 8af9b8c2b889d80c22d6bc26ba0df1afb79a30db ./configure -prefix=$DEPS_PATH --enable-sse4.2 - make CXXFLAGS="-g -O3 -msse4.2" - make install - cd - + $MAKE CXXFLAGS="-g -O3 -msse4.2" + $MAKE install + popd fi diff --git a/tools/dependencies/curl.sh b/tools/dependencies/curl.sh index 9633edb78538..bb947715f2ac 100755 --- a/tools/dependencies/curl.sh +++ b/tools/dependencies/curl.sh @@ -18,12 +18,14 @@ # under the License. # This script builds the static library of libcurl that can be used as dependency of mxnet. +set -ex LIBCURL_VERSION=7.61.0 if [[ ! -f $DEPS_PATH/lib/libcurl.a ]]; then # download and build libcurl >&2 echo "Building libcurl..." curl -s -L https://curl.haxx.se/download/curl-$LIBCURL_VERSION.zip -o $DEPS_PATH/libcurl.zip unzip -q $DEPS_PATH/libcurl.zip -d $DEPS_PATH + pushd . cd $DEPS_PATH/curl-$LIBCURL_VERSION if [[ $PLATFORM == 'linux' ]]; then CONFIG_FLAG="" @@ -58,7 +60,7 @@ if [[ ! -f $DEPS_PATH/lib/libcurl.a ]]; then --disable-gopher \ --disable-manual \ --prefix=$DEPS_PATH - make - make install - cd - + $MAKE + $MAKE install + popd fi diff --git a/tools/dependencies/eigen.sh b/tools/dependencies/eigen.sh index ac2f75a03a52..a0cd8fcc95ef 100755 --- a/tools/dependencies/eigen.sh +++ b/tools/dependencies/eigen.sh @@ -18,6 +18,7 @@ # under the License. # This script imports the headers from eigen3 that can be used to in opencv. +set -ex EIGEN_VERSION=3.3.4 if [[ ! -d $DEPS_PATH/include/eigen3 ]]; then # download eigen @@ -25,10 +26,11 @@ if [[ ! -d $DEPS_PATH/include/eigen3 ]]; then curl -s -L https://github.com/eigenteam/eigen-git-mirror/archive/$EIGEN_VERSION.zip -o $DEPS_PATH/eigen.zip unzip -q $DEPS_PATH/eigen.zip -d $DEPS_PATH mkdir -p $DEPS_PATH/eigen-git-mirror-$EIGEN_VERSION/build + pushd . cd $DEPS_PATH/eigen-git-mirror-$EIGEN_VERSION/build cmake \ -D CMAKE_BUILD_TYPE=RELEASE \ -D CMAKE_INSTALL_PREFIX=$DEPS_PATH .. - make install - cd - + $MAKE install + popd fi diff --git a/tools/dependencies/libpng.sh b/tools/dependencies/libpng.sh index d1523c654478..3faa9f027dcb 100755 --- a/tools/dependencies/libpng.sh +++ b/tools/dependencies/libpng.sh @@ -18,6 +18,7 @@ # under the License. # This script builds the static library of libpng that can be used as dependency of mxnet/opencv. +set -ex PNG_VERSION=1.6.34 if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then # download and build libpng @@ -25,6 +26,7 @@ if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then curl -s -L https://github.com/glennrp/libpng/archive/v$PNG_VERSION.zip -o $DEPS_PATH/libpng.zip unzip -q $DEPS_PATH/libpng.zip -d $DEPS_PATH mkdir -p $DEPS_PATH/libpng-$PNG_VERSION/build + pushd . cd $DEPS_PATH/libpng-$PNG_VERSION/build cmake \ -D PNG_SHARED=OFF \ @@ -32,9 +34,9 @@ if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then -D CMAKE_BUILD_TYPE=RELEASE \ -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ -D CMAKE_C_FLAGS=-fPIC .. - make - make install + $MAKE + $MAKE install mkdir -p $DEPS_PATH/include/libpng ln -s $DEPS_PATH/include/png.h $DEPS_PATH/include/libpng/png.h - cd - + popd fi diff --git a/tools/dependencies/libtiff.sh b/tools/dependencies/libtiff.sh index 14dcb2d7bde0..2a402bbcac27 100755 --- a/tools/dependencies/libtiff.sh +++ b/tools/dependencies/libtiff.sh @@ -18,15 +18,17 @@ # under the License. # This script builds the static library of libtiff that can be used as dependency of mxnet/opencv. +set -ex TIFF_VERSION="4-0-9" if [[ ! -f $DEPS_PATH/lib/libtiff.a ]]; then # download and build libtiff >&2 echo "Building libtiff..." curl -s -L https://gitlab.com/libtiff/libtiff/-/archive/Release-v$TIFF_VERSION/libtiff-Release-v$TIFF_VERSION.zip -o $DEPS_PATH/libtiff.zip unzip -q $DEPS_PATH/libtiff.zip -d $DEPS_PATH + pushd . cd $DEPS_PATH/libtiff-Release-v$TIFF_VERSION ./configure --quiet --disable-shared --disable-jpeg --disable-zlib --disable-jbig --disable-lzma --prefix=$DEPS_PATH - make - make install - cd - + $MAKE + $MAKE install + popd fi diff --git a/tools/dependencies/libturbojpeg.sh b/tools/dependencies/libturbojpeg.sh index 4991906f8878..ac813ebec1c7 100755 --- a/tools/dependencies/libturbojpeg.sh +++ b/tools/dependencies/libturbojpeg.sh @@ -19,6 +19,7 @@ # This script builds the static library of libturbojpeg that can be used as dependency of # mxnet/opencv. +set -ex TURBO_JPEG_VERSION=1.5.90 if [[ $PLATFORM == 'darwin' ]]; then JPEG_NASM_OPTION="-D CMAKE_ASM_NASM_COMPILER=/usr/local/bin/nasm" @@ -30,6 +31,7 @@ if [[ ! -f $DEPS_PATH/lib/libjpeg.a ]] || [[ ! -f $DEPS_PATH/lib/libturbojpeg.a curl -s -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/$TURBO_JPEG_VERSION.zip -o $DEPS_PATH/libjpeg.zip unzip -q $DEPS_PATH/libjpeg.zip -d $DEPS_PATH mkdir -p $DEPS_PATH/libjpeg-turbo-$TURBO_JPEG_VERSION/build + pushd . cd $DEPS_PATH/libjpeg-turbo-$TURBO_JPEG_VERSION/build cmake \ -G"Unix Makefiles" \ @@ -41,7 +43,7 @@ if [[ ! -f $DEPS_PATH/lib/libjpeg.a ]] || [[ ! -f $DEPS_PATH/lib/libturbojpeg.a -D WITH_JPEG8=TRUE \ $JPEG_NASM_OPTION \ -D ENABLE_SHARED=FALSE .. - make - make install - cd - + $MAKE + $MAKE install + popd fi diff --git a/tools/dependencies/libz.sh b/tools/dependencies/libz.sh index 927f1de82e72..c5f9953c8d1c 100755 --- a/tools/dependencies/libz.sh +++ b/tools/dependencies/libz.sh @@ -18,6 +18,7 @@ # under the License. # This script builds the static library of libz that can be used as dependency of mxnet. +set -ex ZLIB_VERSION=1.2.6 if [[ ! -f $DEPS_PATH/lib/libz.a ]]; then # Download and build zlib @@ -25,12 +26,13 @@ if [[ ! -f $DEPS_PATH/lib/libz.a ]]; then curl -s -L https://github.com/LuaDist/zlib/archive/$ZLIB_VERSION.zip -o $DEPS_PATH/zlib.zip unzip -q $DEPS_PATH/zlib.zip -d $DEPS_PATH mkdir -p $DEPS_PATH/zlib-$ZLIB_VERSION/build + pushd . cd $DEPS_PATH/zlib-$ZLIB_VERSION/build cmake \ -D CMAKE_BUILD_TYPE=RELEASE \ -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ -D BUILD_SHARED_LIBS=OFF .. - make - make install - cd - + $MAKE + $MAKE install + popd fi diff --git a/tools/dependencies/lz4.sh b/tools/dependencies/lz4.sh index a4269bf29bb9..478c9925e967 100755 --- a/tools/dependencies/lz4.sh +++ b/tools/dependencies/lz4.sh @@ -18,14 +18,16 @@ # under the License. # This script builds the static library of lz4 that can be used as dependency of mxnet. +set -ex LZ4_VERSION=r130 if [[ ! -f $DEPS_PATH/lib/liblz4.a ]]; then # Download and build lz4 >&2 echo "Building lz4..." curl -s -L https://github.com/lz4/lz4/archive/$LZ4_VERSION.zip -o $DEPS_PATH/lz4.zip unzip -q $DEPS_PATH/lz4.zip -d $DEPS_PATH + pushd . cd $DEPS_PATH/lz4-$LZ4_VERSION - make - make PREFIX=$DEPS_PATH install - cd - + $MAKE + $MAKE PREFIX=$DEPS_PATH install + popd fi diff --git a/tools/dependencies/make_shared_dependencies.sh b/tools/dependencies/make_shared_dependencies.sh index d678fddcc02d..16508586169a 100755 --- a/tools/dependencies/make_shared_dependencies.sh +++ b/tools/dependencies/make_shared_dependencies.sh @@ -38,3 +38,5 @@ source $DIR/protobuf.sh source $DIR/cityhash.sh source $DIR/zmq.sh source $DIR/lz4.sh + +export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(dirname $(find $DEPS_PATH -type f -name 'libprotoc*' | grep protobuf | head -n 1)):$DEPS_PATH/lib diff --git a/tools/dependencies/openblas.sh b/tools/dependencies/openblas.sh index 9463e3325e0d..27f473919a5b 100755 --- a/tools/dependencies/openblas.sh +++ b/tools/dependencies/openblas.sh @@ -18,6 +18,8 @@ # under the License. # This script builds the static library of openblas that can be used as dependency of mxnet. +set +e # This script throws an error but otherwise works +set -x OPENBLAS_VERSION=0.3.3 if [[ ! -e $DEPS_PATH/lib/libopenblas.a ]]; then # download and build openblas @@ -25,11 +27,12 @@ if [[ ! -e $DEPS_PATH/lib/libopenblas.a ]]; then curl -s -L https://github.com/xianyi/OpenBLAS/archive/v$OPENBLAS_VERSION.zip -o $DEPS_PATH/openblas.zip unzip -q $DEPS_PATH/openblas.zip -d $DEPS_PATH + pushd . cd $DEPS_PATH/OpenBLAS-$OPENBLAS_VERSION - make DYNAMIC_ARCH=1 NO_SHARED=1 USE_OPENMP=1 - make PREFIX=$DEPS_PATH install - cd - + $MAKE DYNAMIC_ARCH=1 NO_SHARED=1 USE_OPENMP=1 + $MAKE PREFIX=$DEPS_PATH install + popd ln -s libopenblas.a $DEPS_PATH/lib/libcblas.a ln -s libopenblas.a $DEPS_PATH/lib/liblapack.a fi diff --git a/tools/dependencies/opencv.sh b/tools/dependencies/opencv.sh index 99d0ecb71c36..11c9c2155f91 100755 --- a/tools/dependencies/opencv.sh +++ b/tools/dependencies/opencv.sh @@ -19,6 +19,7 @@ # This script builds the static library of opencv that can be used as dependency of mxnet. # It expects openblas, libjpeg, libpng, libtiff, eigen, etc., to be in $DEPS_PATH. +set -ex OPENCV_VERSION=3.4.2 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" if [[ $PLATFORM == 'linux' ]]; then @@ -41,6 +42,7 @@ if [[ ! -f $DEPS_PATH/lib/libopencv_core.a ]] || [[ ! -f $DEPS_PATH/lib/libopenc curl -s -L https://github.com/opencv/opencv/archive/$OPENCV_VERSION.zip -o $DEPS_PATH/opencv.zip unzip -q $DEPS_PATH/opencv.zip -d $DEPS_PATH mkdir -p $DEPS_PATH/opencv-$OPENCV_VERSION/build + pushd . cd $DEPS_PATH/opencv-$OPENCV_VERSION/build cmake \ -D OPENCV_ENABLE_NONFREE=OFF \ @@ -184,9 +186,9 @@ if [[ ! -f $DEPS_PATH/lib/libopencv_core.a ]] || [[ ! -f $DEPS_PATH/lib/libopenc if [[ $PLATFORM == 'linux' ]]; then cp $DIR/patch/opencv_lapack.h ./ fi - make - make install - cd - + $MAKE + $MAKE install + popd # @szha: compatibility header cat $DEPS_PATH/include/opencv2/imgcodecs/imgcodecs_c.h >> $DEPS_PATH/include/opencv2/imgcodecs.hpp fi diff --git a/tools/dependencies/openssl.sh b/tools/dependencies/openssl.sh index b7e4317d4a89..93284db3e39f 100755 --- a/tools/dependencies/openssl.sh +++ b/tools/dependencies/openssl.sh @@ -18,6 +18,7 @@ # under the License. # This script builds the static library of openssl that can be used as dependency of mxnet. +set -ex OPENSSL_VERSION=1.0.2l if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f $DEPS_PATH/lib/libcrypto.a ]]; then # download and build openssl @@ -25,6 +26,7 @@ if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f $DEPS_PATH/lib/libcrypto.a ]]; OPENSSL_VERSION=$(echo $OPENSSL_VERSION | sed 's/\./_/g') curl -s -L https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION.zip -o $DEPS_PATH/openssl.zip unzip -q $DEPS_PATH/openssl.zip -d $DEPS_PATH + pushd . cd $DEPS_PATH/openssl-OpenSSL_$OPENSSL_VERSION if [[ $PLATFORM == 'linux' ]]; then TARGET=linux-x86_64 @@ -32,7 +34,7 @@ if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f $DEPS_PATH/lib/libcrypto.a ]]; TARGET=darwin64-x86_64-cc fi ./Configure no-shared no-zlib --prefix=$DEPS_PATH --openssldir=$DEPS_PATH/ssl $TARGET - make - make install - cd - + $MAKE + $MAKE install + popd fi diff --git a/tools/dependencies/protobuf.sh b/tools/dependencies/protobuf.sh index 1564701042af..76ce1de8e822 100755 --- a/tools/dependencies/protobuf.sh +++ b/tools/dependencies/protobuf.sh @@ -18,6 +18,7 @@ # under the License. # This script builds the static library of protobuf along with protoc, that can be used as dependency of mxnet. +set -ex PROTOBUF_VERSION=3.5.1 if [[ $PLATFORM == 'darwin' ]]; then DY_EXT="dylib" @@ -32,12 +33,12 @@ if [[ ! -e $LIBPROTOBUF ]] || [[ ! -e $LIBPROTOC ]]; then >&2 echo "Building protobuf..." curl -s -L https://github.com/google/protobuf/archive/v$PROTOBUF_VERSION.zip -o $DEPS_PATH/protobuf.zip unzip -q $DEPS_PATH/protobuf.zip -d $DEPS_PATH + pushd . cd $DEPS_PATH/protobuf-$PROTOBUF_VERSION ./autogen.sh ./configure -prefix=$DEPS_PATH - make - make install - cd - + $MAKE + $MAKE install + popd fi -export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(dirname $(find $DEPS_PATH -type f -name 'libprotoc*' | grep protobuf | head -n 1)):$DEPS_PATH/lib diff --git a/tools/dependencies/zmq.sh b/tools/dependencies/zmq.sh index 55e17798c2d3..0042d6bcd073 100755 --- a/tools/dependencies/zmq.sh +++ b/tools/dependencies/zmq.sh @@ -18,6 +18,7 @@ # under the License. # This script builds the static library of zeroMQ that can be used as dependency of mxnet. +set -ex ZEROMQ_VERSION=4.2.2 if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then # Download and build zmq @@ -25,14 +26,14 @@ if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then curl -s -L https://github.com/zeromq/libzmq/archive/v$ZEROMQ_VERSION.zip -o $DEPS_PATH/zeromq.zip unzip -q $DEPS_PATH/zeromq.zip -d $DEPS_PATH mkdir -p $DEPS_PATH/libzmq-$ZEROMQ_VERSION/build + pushd . cd $DEPS_PATH/libzmq-$ZEROMQ_VERSION/build cmake \ -D CMAKE_BUILD_TYPE=RELEASE \ -D CMAKE_INSTALL_PREFIX=$DEPS_PATH \ -D WITH_LIBSODIUM=OFF \ -D BUILD_SHARED_LIBS=OFF .. - make - make install - cp $DEPS_PATH/lib/x86_64-linux-gnu/libzmq.a $DEPS_PATH/lib/libzmq.a - cd - + $MAKE + $MAKE install + popd fi diff --git a/tools/setup_gpu_build_tools.sh b/tools/setup_gpu_build_tools.sh index 44b44c574114..167d4c6a6e13 100755 --- a/tools/setup_gpu_build_tools.sh +++ b/tools/setup_gpu_build_tools.sh @@ -246,6 +246,8 @@ if [[ ! -d $DEPS_PATH/usr/local/cuda-${CUDA_MAJOR_VERSION} ]]; then rm package.deb done + mkdir -p ${prefix}/include + mkdir -p ${prefix}/lib cp ${prefix}/usr/include/x86_64-linux-gnu/cudnn_v${LIBCUDNN_MAJOR}.h ${prefix}/include/cudnn.h ln -s libcudnn_static_v${LIBCUDNN_MAJOR}.a ${prefix}/usr/lib/x86_64-linux-gnu/libcudnn.a cp ${prefix}/usr/local/cuda-${CUDA_MAJOR_VERSION}/lib64/*.a ${prefix}/lib/ diff --git a/tools/staticbuild/build.sh b/tools/staticbuild/build.sh new file mode 100755 index 000000000000..4f8a78d70490 --- /dev/null +++ b/tools/staticbuild/build.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if [ $# -lt 1 ]; then + >&2 echo "Usage: build.sh " +fi + +export CURDIR=$PWD +export DEPS_PATH=$PWD/staticdeps +export VARIANT=$(echo $1 | tr '[:upper:]' '[:lower:]') +export STATIC_BUILD_TARGET=$(echo $2 | tr '[:upper:]' '[:lower:]') +export PLATFORM=$(uname | tr '[:upper:]' '[:lower:]') + +if [[ $VARIANT == darwin* ]]; then + export VARIANT="darwin" +fi + +NUM_PROC=1 +if [[ ! -z $(command -v nproc) ]]; then + NUM_PROC=$(nproc) +elif [[ ! -z $(command -v sysctl) ]]; then + NUM_PROC=$(sysctl -n hw.ncpu) +else + >&2 echo "Can't discover number of cores." +fi +export NUM_PROC +>&2 echo "Using $NUM_PROC parallel jobs in building." + +if [[ $DEBUG -eq 1 ]]; then + export ADD_MAKE_FLAG="-j $NUM_PROC" +else + export ADD_MAKE_FLAG="--quiet -j $NUM_PROC" +fi +export MAKE="make $ADD_MAKE_FLAG" + +export CC="gcc -fPIC" +export CXX="g++ -fPIC" +export FC="gfortran" +export PKG_CONFIG_PATH=$DEPS_PATH/lib/pkgconfig:$DEPS_PATH/lib64/pkgconfig:$DEPS_PATH/lib/x86_64-linux-gnu/pkgconfig:$PKG_CONFIG_PATH +export CPATH=$DEPS_PATH/include:$CPATH + +if [[ $PLATFORM == 'linux' && $VARIANT == cu* ]]; then + source tools/setup_gpu_build_tools.sh $VARIANT $DEPS_PATH +fi + +mkdir -p $DEPS_PATH + +# Build Dependencies +source tools/dependencies/make_shared_dependencies.sh + +# Build mxnet +source tools/staticbuild/build_lib.sh diff --git a/tools/build/build_lib.sh b/tools/staticbuild/build_lib.sh similarity index 77% rename from tools/build/build_lib.sh rename to tools/staticbuild/build_lib.sh index 032fcb92045f..dc6bd4e8b766 100755 --- a/tools/build/build_lib.sh +++ b/tools/staticbuild/build_lib.sh @@ -18,22 +18,20 @@ # under the License. # This script builds the libraries of mxnet. -make_config=config/pip_${PLATFORM}_${VARIANT}.mk +make_config=make/${STATIC_BUILD_TARGET}/${STATIC_BUILD_TARGET}_${PLATFORM}_${VARIANT}.mk if [[ ! -f $make_config ]]; then >&2 echo "Couldn't find make config $make_config for the current settings." exit 1 fi -git clone --recursive https://github.com/apache/incubator-mxnet mxnet-build - >&2 echo "Now building mxnet modules..." -cp $make_config mxnet-build/config.mk +cp $make_config config.mk -cd mxnet-build +git submodule update --init --recursive || true -make DEPS_PATH=$DEPS_PATH DMLCCORE -make DEPS_PATH=$DEPS_PATH $PWD/3rdparty/tvm/nnvm/lib/libnnvm.a -make DEPS_PATH=$DEPS_PATH PSLITE +$MAKE DEPS_PATH=$DEPS_PATH DMLCCORE +$MAKE DEPS_PATH=$DEPS_PATH $PWD/3rdparty/tvm/nnvm/lib/libnnvm.a +$MAKE DEPS_PATH=$DEPS_PATH PSLITE if [[ $VARIANT == *mkl ]]; then MKLDNN_LICENSE='license.txt' @@ -46,7 +44,11 @@ if [[ $VARIANT == *mkl ]]; then MKLML_LIBFILE='libmklml.dylib' MKLDNN_LIBFILE='libmkldnn.0.dylib' fi - make DEPS_PATH=$DEPS_PATH mkldnn + $MAKE DEPS_PATH=$DEPS_PATH mkldnn + cp 3rdparty/mkldnn/build/install/lib/$IOMP_LIBFILE lib + cp 3rdparty/mkldnn/build/install/lib/$MKLML_LIBFILE lib + cp 3rdparty/mkldnn/build/install/lib/$MKLDNN_LIBFILE lib + cp 3rdparty/mkldnn/build/install/$MKLDNN_LICENSE lib cp 3rdparty/mkldnn/LICENSE ./MKLML_LICENSE fi @@ -58,7 +60,7 @@ if [[ $VARIANT == *mkl ]]; then fi >&2 echo "Now building mxnet..." -make DEPS_PATH=$DEPS_PATH || exit 1; +$MAKE DEPS_PATH=$DEPS_PATH if [[ $PLATFORM == 'linux' ]]; then cp -L /usr/lib/gcc/x86_64-linux-gnu/4.8/libgfortran.so lib/libgfortran.so.3 @@ -77,4 +79,4 @@ else >&2 echo "Not available" fi -cd ../ +ln -s staticdeps/ deps diff --git a/tools/build/build_wheel.sh b/tools/staticbuild/build_wheel.sh similarity index 100% rename from tools/build/build_wheel.sh rename to tools/staticbuild/build_wheel.sh