From 53534ccc2b71826369fccb68f045520007cbce0f Mon Sep 17 00:00:00 2001 From: Qing Date: Wed, 9 Jan 2019 10:02:01 -0800 Subject: [PATCH 1/5] clean up build --- ci/docker/runtime_functions.sh | 17 +++++++------- ci/jenkins/Jenkins_steps.groovy | 22 +++++++++++++++---- ci/jenkins/Jenkinsfile_unix_cpu | 1 + contrib/clojure-package/README.md | 4 ++-- .../CNNClassifierExampleSuite.scala | 3 +-- .../ImageClassifierExampleSuite.scala | 1 - .../multitask/MultiTaskSuite.scala | 3 +-- .../profiler/ProfilerSuite.scala | 3 +-- scala-package/spark/README.md | 3 ++- tests/jenkins/run_test.sh | 4 ++-- tests/jenkins/run_test_amzn_linux_gpu.sh | 4 ++-- tests/jenkins/run_test_ubuntu.sh | 4 ++-- 12 files changed, 41 insertions(+), 28 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 4ec6b76d6267..e5b8632af88e 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -847,23 +847,23 @@ unittest_ubuntu_python3_quantization_gpu() { unittest_ubuntu_cpu_scala() { set -ex scala_prepare - make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 - make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 + cd scala-package + mvn -B install } unittest_centos7_cpu_scala() { set -ex cd /work/mxnet scala_prepare - make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 - make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 + cd scala-package + mvn -B install } unittest_ubuntu_cpu_clojure() { set -ex scala_prepare - make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 - make scalainstall USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 + cd scala-package + mvn -B install ./contrib/clojure-package/ci-test.sh } @@ -1008,8 +1008,9 @@ integrationtest_ubuntu_cpu_dist_kvstore() { integrationtest_ubuntu_gpu_scala() { set -ex scala_prepare - make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1 CI=1 - make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1 + cd scala-package + export SCALA_TEST_ON_GPU=1 + mvn -B integration-test -DskipTests=false } integrationtest_ubuntu_gpu_dist_kvstore() { diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 57454b9ee44c..e575d2472cac 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -825,7 +825,21 @@ def test_unix_scala_cpu() { node(NODE_LINUX_CPU) { ws('workspace/ut-scala-cpu') { timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('cpu', mx_dist_lib, true) + utils.unpack_and_init('cpu', mx_lib, true) + utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_scala', false) + utils.publish_test_coverage() + } + } + } + }] +} + +def test_unix_scala_mkldnn_cpu(){ + return ['Scala: CPU': { + node(NODE_LINUX_CPU) { + ws('workspace/ut-scala-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.unpack_and_init('mkldnn_cpu', mx_mkldnn_lib, true) utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_scala', false) utils.publish_test_coverage() } @@ -839,7 +853,7 @@ def test_unix_scala_gpu() { node(NODE_LINUX_GPU) { ws('workspace/ut-scala-gpu') { timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('gpu', mx_dist_lib, true) + utils.unpack_and_init('gpu', mx_lib, true) utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_scala', true) utils.publish_test_coverage() } @@ -853,7 +867,7 @@ def test_unix_clojure_cpu() { node(NODE_LINUX_CPU) { ws('workspace/ut-clojure-cpu') { timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('cpu', mx_dist_lib, true) + utils.unpack_and_init('cpu', mx_lib, true) utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_clojure', false) utils.publish_test_coverage() } @@ -1029,7 +1043,7 @@ def test_centos7_scala_cpu() { node(NODE_LINUX_CPU) { ws('workspace/ut-scala-centos7-cpu') { timeout(time: max_time, unit: 'MINUTES') { - utils.unpack_and_init('centos7_cpu', mx_dist_lib, true) + utils.unpack_and_init('centos7_cpu', mx_lib, true) utils.docker_run('centos7_cpu', 'unittest_centos7_cpu_scala', false) utils.publish_test_coverage() } diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu index 8f41fa69f91c..1cbb6a72e3a7 100644 --- a/ci/jenkins/Jenkinsfile_unix_cpu +++ b/ci/jenkins/Jenkinsfile_unix_cpu @@ -50,6 +50,7 @@ core_logic: { custom_steps.test_unix_python3_mkldnn_cpu(), custom_steps.test_unix_python3_mkldnn_mkl_cpu(), custom_steps.test_unix_scala_cpu(), + custom_steps.test_unix_scala_mkldnn_cpu(), custom_steps.test_unix_clojure_cpu(), custom_steps.test_unix_r_cpu(), custom_steps.test_unix_julia_cpu(), diff --git a/contrib/clojure-package/README.md b/contrib/clojure-package/README.md index 7930befc61aa..ba6160aed5c8 100644 --- a/contrib/clojure-package/README.md +++ b/contrib/clojure-package/README.md @@ -193,8 +193,8 @@ The outcome of this step will be a shared library `lib/libmxnet.so` that is used - Build and install the Scala package in your local Maven directory using the following commands: ```bash - make scalapkg - make scalainstall + cd scala-package + mvn install ``` #### Building the Clojure jar diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala index 44025c0459ad..a7e36dfc3a11 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala @@ -29,8 +29,7 @@ import org.slf4j.LoggerFactory import scala.sys.process.Process /** - * Integration test for imageClassifier example. - * This will run as a part of "make scalatest" + * Integration test for CNN example. */ class CNNClassifierExampleSuite extends FunSuite with BeforeAndAfterAll { private val logger = LoggerFactory.getLogger(classOf[CNNClassifierExampleSuite]) diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala index 27d9bb4c8fe9..c5308ac37512 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala @@ -27,7 +27,6 @@ import sys.process.Process /** * Integration test for imageClassifier example. - * This will run as a part of "make scalatest" */ class ImageClassifierExampleSuite extends FunSuite with BeforeAndAfterAll { private val logger = LoggerFactory.getLogger(classOf[ImageClassifierExampleSuite]) diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/multitask/MultiTaskSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/multitask/MultiTaskSuite.scala index 983978dbaec4..65902c7ad391 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/multitask/MultiTaskSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/multitask/MultiTaskSuite.scala @@ -25,8 +25,7 @@ import org.scalatest.FunSuite /** - * Integration test for imageClassifier example. - * This will run as a part of "make scalatest" + * Integration test for Multi-task example. */ class MultiTaskSuite extends FunSuite { test("Multitask Test") { diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/profiler/ProfilerSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/profiler/ProfilerSuite.scala index 859a8c06493f..67638b8fac0f 100644 --- a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/profiler/ProfilerSuite.scala +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/profiler/ProfilerSuite.scala @@ -25,8 +25,7 @@ import org.apache.mxnet.Profiler import org.apache.mxnet.Context /** - * Integration test for imageClassifier example. - * This will run as a part of "make scalatest" + * Integration test for profiler example. */ class ProfilerSuite extends FunSuite with BeforeAndAfterAll { private val logger = LoggerFactory.getLogger(classOf[ProfilerSuite]) diff --git a/scala-package/spark/README.md b/scala-package/spark/README.md index 06106648c059..d390afac1b77 100644 --- a/scala-package/spark/README.md +++ b/scala-package/spark/README.md @@ -16,7 +16,8 @@ Checkout the [Installation Guide](http://mxnet.io/get_started/setup.html) contai Compile the Scala Package by ```bash -make scalapkg +cd scala-package +mvn install ``` This will automatically build the `spark` submodule. Now you can submit Spark job with these built jars. diff --git a/tests/jenkins/run_test.sh b/tests/jenkins/run_test.sh index bc69ca1d7f39..5ef8c1ac01ef 100755 --- a/tests/jenkins/run_test.sh +++ b/tests/jenkins/run_test.sh @@ -56,8 +56,8 @@ nosetests3 --verbose tests/python/train || exit -1 echo "BUILD scala_test" export PATH=$PATH:/opt/apache-maven/bin -make scalapkg || exit -1 -make scalatest || exit -1 +cd scala-package +mvn install || exit -1 # echo "BUILD julia_test" # export MXNET_HOME="${PWD}" diff --git a/tests/jenkins/run_test_amzn_linux_gpu.sh b/tests/jenkins/run_test_amzn_linux_gpu.sh index ecfb5211b9e6..7724dd619911 100755 --- a/tests/jenkins/run_test_amzn_linux_gpu.sh +++ b/tests/jenkins/run_test_amzn_linux_gpu.sh @@ -65,5 +65,5 @@ nosetests3 --verbose tests/python/train #julia -e 'try Pkg.clone("MXNet"); catch end; Pkg.checkout("MXNet"); Pkg.build("MXNet"); Pkg.test("MXNet")' || exit -1 echo "BUILD scala_test" -make scalapkg -make scalatest +cd scala-package +mvn install diff --git a/tests/jenkins/run_test_ubuntu.sh b/tests/jenkins/run_test_ubuntu.sh index cdddd2865ddc..16180ba1f6b3 100755 --- a/tests/jenkins/run_test_ubuntu.sh +++ b/tests/jenkins/run_test_ubuntu.sh @@ -68,6 +68,6 @@ nosetests3 --verbose tests/python/train || exit 1 echo "BUILD scala_test" export PATH=$PATH:/opt/apache-maven/bin -make scalapkg || exit 1 -make scalatest || exit 1 +cd scala-package +mvn install || exit 1 From dfa99d65dea7d6dfda70b47b11812a6d50d6dc81 Mon Sep 17 00:00:00 2001 From: Qing Date: Wed, 9 Jan 2019 11:23:18 -0800 Subject: [PATCH 2/5] fix minor issue and add mkldnn --- ci/docker/runtime_functions.sh | 7 ++++--- ci/jenkins/Jenkins_steps.groovy | 6 ++---- scala-package/spark/README.md | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index e5b8632af88e..be48e9c5fb73 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -848,7 +848,7 @@ unittest_ubuntu_cpu_scala() { set -ex scala_prepare cd scala-package - mvn -B install + mvn -B integration-test } unittest_centos7_cpu_scala() { @@ -856,14 +856,15 @@ unittest_centos7_cpu_scala() { cd /work/mxnet scala_prepare cd scala-package - mvn -B install + mvn -B integration-test } unittest_ubuntu_cpu_clojure() { set -ex scala_prepare cd scala-package - mvn -B install + mvn -B integration-test + cd .. ./contrib/clojure-package/ci-test.sh } diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index e575d2472cac..15d8a4c609e1 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -28,8 +28,6 @@ mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdpart // Python wheels mx_pip = 'build/*.whl' -// for scala build, need to pass extra libs when run with dist_kvstore -mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. @@ -835,9 +833,9 @@ def test_unix_scala_cpu() { } def test_unix_scala_mkldnn_cpu(){ - return ['Scala: CPU': { + return ['Scala: MKLDNN-CPU': { node(NODE_LINUX_CPU) { - ws('workspace/ut-scala-cpu') { + ws('workspace/ut-scala-mkldnn-cpu') { timeout(time: max_time, unit: 'MINUTES') { utils.unpack_and_init('mkldnn_cpu', mx_mkldnn_lib, true) utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_scala', false) diff --git a/scala-package/spark/README.md b/scala-package/spark/README.md index d390afac1b77..503c279038a5 100644 --- a/scala-package/spark/README.md +++ b/scala-package/spark/README.md @@ -17,7 +17,7 @@ Compile the Scala Package by ```bash cd scala-package -mvn install +mvn package ``` This will automatically build the `spark` submodule. Now you can submit Spark job with these built jars. From 13d76a5deb1ccf4b545a19987c45adeb0ed8e350 Mon Sep 17 00:00:00 2001 From: Qing Date: Wed, 9 Jan 2019 11:44:12 -0800 Subject: [PATCH 3/5] fix mx_dist problem --- ci/jenkins/Jenkins_steps.groovy | 6 +++--- tests/jenkins/run_test_amzn_linux_gpu.sh | 2 +- tests/jenkins/run_test_ubuntu.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 15d8a4c609e1..33d76aa1668a 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -98,7 +98,7 @@ def compile_unix_cpu_openblas() { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas', false) - utils.pack_lib('cpu', mx_dist_lib, true) + utils.pack_lib('cpu', mx_lib, true) } } } @@ -126,7 +126,7 @@ def compile_unix_mkl_cpu() { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkl', false) - utils.pack_lib('cpu_mkl', mx_dist_lib, true) + utils.pack_lib('cpu_mkl', mx_mkldnn_lib, true) } } } @@ -252,7 +252,7 @@ def compile_centos7_cpu() { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() utils.docker_run('centos7_cpu', 'build_centos7_cpu', false) - utils.pack_lib('centos7_cpu', mx_dist_lib, true) + utils.pack_lib('centos7_cpu', mx_lib, true) } } } diff --git a/tests/jenkins/run_test_amzn_linux_gpu.sh b/tests/jenkins/run_test_amzn_linux_gpu.sh index 7724dd619911..20d6bcb3991a 100755 --- a/tests/jenkins/run_test_amzn_linux_gpu.sh +++ b/tests/jenkins/run_test_amzn_linux_gpu.sh @@ -66,4 +66,4 @@ nosetests3 --verbose tests/python/train echo "BUILD scala_test" cd scala-package -mvn install +mvn integration-test -Dskip.test=false diff --git a/tests/jenkins/run_test_ubuntu.sh b/tests/jenkins/run_test_ubuntu.sh index 16180ba1f6b3..28e00331b47e 100755 --- a/tests/jenkins/run_test_ubuntu.sh +++ b/tests/jenkins/run_test_ubuntu.sh @@ -69,5 +69,5 @@ nosetests3 --verbose tests/python/train || exit 1 echo "BUILD scala_test" export PATH=$PATH:/opt/apache-maven/bin cd scala-package -mvn install || exit 1 +mvn integration-test || exit 1 From 14f09ed119e22efeaff22cf6c5ad497e12d3aae3 Mon Sep 17 00:00:00 2001 From: Qing Date: Wed, 9 Jan 2019 13:30:55 -0800 Subject: [PATCH 4/5] fix clojure build --- ci/docker/runtime_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index be48e9c5fb73..8c4fabeb6317 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -863,7 +863,7 @@ unittest_ubuntu_cpu_clojure() { set -ex scala_prepare cd scala-package - mvn -B integration-test + mvn -B install cd .. ./contrib/clojure-package/ci-test.sh } From 46c8d57108fae5b4b472a8a333bf2f7de35be42b Mon Sep 17 00:00:00 2001 From: Qing Date: Wed, 9 Jan 2019 16:45:05 -0800 Subject: [PATCH 5/5] fix skip test --- tests/jenkins/run_test_amzn_linux_gpu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/jenkins/run_test_amzn_linux_gpu.sh b/tests/jenkins/run_test_amzn_linux_gpu.sh index 20d6bcb3991a..57d9c7884088 100755 --- a/tests/jenkins/run_test_amzn_linux_gpu.sh +++ b/tests/jenkins/run_test_amzn_linux_gpu.sh @@ -66,4 +66,4 @@ nosetests3 --verbose tests/python/train echo "BUILD scala_test" cd scala-package -mvn integration-test -Dskip.test=false +mvn integration-test