Add new Maven build for Scala package (#13819)

* clean up build * fix minor issue and add mkldnn * fix mx_dist problem * fix clojure build * fix skip test
apache · Jan 10, 2019 · 5fe2172 · 5fe2172
1 parent 30b552d
commit 5fe2172
Show file tree

Hide file tree

Showing 12 changed files with 45 additions and 33 deletions.
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
@@ -847,23 +847,24 @@ unittest_ubuntu_python3_quantization_gpu() {
 unittest_ubuntu_cpu_scala() {
     set -ex
     scala_prepare
-    make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
-    make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
+    cd scala-package
+    mvn -B integration-test
 }
 
 unittest_centos7_cpu_scala() {
     set -ex
     cd /work/mxnet
     scala_prepare
-    make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
-    make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
+    cd scala-package
+    mvn -B integration-test
 }
 
 unittest_ubuntu_cpu_clojure() {
     set -ex
     scala_prepare
-    make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
-    make scalainstall USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
+    cd scala-package
+    mvn -B install
+    cd ..
     ./contrib/clojure-package/ci-test.sh
 }
 
@@ -1008,8 +1009,9 @@ integrationtest_ubuntu_cpu_dist_kvstore() {
 integrationtest_ubuntu_gpu_scala() {
     set -ex
     scala_prepare
-    make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1 CI=1
-    make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
+    cd scala-package
+    export SCALA_TEST_ON_GPU=1
+    mvn -B integration-test -DskipTests=false
 }
 
 integrationtest_ubuntu_gpu_dist_kvstore() {

diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
@@ -28,8 +28,6 @@ mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdpart
 // Python wheels
 mx_pip = 'build/*.whl'
 
-// for scala build, need to pass extra libs when run with dist_kvstore
-mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
 mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
 // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
@@ -100,7 +98,7 @@ def compile_unix_cpu_openblas() {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.init_git()
             utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas', false)
-            utils.pack_lib('cpu', mx_dist_lib, true)
+            utils.pack_lib('cpu', mx_lib, true)
           }
         }
       }
@@ -128,7 +126,7 @@ def compile_unix_mkl_cpu() {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.init_git()
             utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkl', false)
-            utils.pack_lib('cpu_mkl', mx_dist_lib, true)
+            utils.pack_lib('cpu_mkl', mx_mkldnn_lib, true)
           }
         }
       }
@@ -254,7 +252,7 @@ def compile_centos7_cpu() {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.init_git()
             utils.docker_run('centos7_cpu', 'build_centos7_cpu', false)
-            utils.pack_lib('centos7_cpu', mx_dist_lib, true)
+            utils.pack_lib('centos7_cpu', mx_lib, true)
           }
         }
       }
@@ -825,7 +823,21 @@ def test_unix_scala_cpu() {
       node(NODE_LINUX_CPU) {
         ws('workspace/ut-scala-cpu') {
           timeout(time: max_time, unit: 'MINUTES') {
-            utils.unpack_and_init('cpu', mx_dist_lib, true)
+            utils.unpack_and_init('cpu', mx_lib, true)
+            utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_scala', false)
+            utils.publish_test_coverage()
+          }
+        }
+      }
+    }]
+}
+
+def test_unix_scala_mkldnn_cpu(){
+  return ['Scala: MKLDNN-CPU': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/ut-scala-mkldnn-cpu') {
+          timeout(time: max_time, unit: 'MINUTES') {
+            utils.unpack_and_init('mkldnn_cpu', mx_mkldnn_lib, true)
             utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_scala', false)
             utils.publish_test_coverage()
           }
@@ -839,7 +851,7 @@ def test_unix_scala_gpu() {
       node(NODE_LINUX_GPU) {
         ws('workspace/ut-scala-gpu') {
           timeout(time: max_time, unit: 'MINUTES') {
-            utils.unpack_and_init('gpu', mx_dist_lib, true)
+            utils.unpack_and_init('gpu', mx_lib, true)
             utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_scala', true)
             utils.publish_test_coverage()
           }
@@ -853,7 +865,7 @@ def test_unix_clojure_cpu() {
       node(NODE_LINUX_CPU) {
         ws('workspace/ut-clojure-cpu') {
           timeout(time: max_time, unit: 'MINUTES') {
-            utils.unpack_and_init('cpu', mx_dist_lib, true)
+            utils.unpack_and_init('cpu', mx_lib, true)
             utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_clojure', false)
             utils.publish_test_coverage()
           }
@@ -1029,7 +1041,7 @@ def test_centos7_scala_cpu() {
       node(NODE_LINUX_CPU) {
         ws('workspace/ut-scala-centos7-cpu') {
           timeout(time: max_time, unit: 'MINUTES') {
-            utils.unpack_and_init('centos7_cpu', mx_dist_lib, true)
+            utils.unpack_and_init('centos7_cpu', mx_lib, true)
             utils.docker_run('centos7_cpu', 'unittest_centos7_cpu_scala', false)
             utils.publish_test_coverage()
           }

diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu
@@ -50,6 +50,7 @@ core_logic: {
     custom_steps.test_unix_python3_mkldnn_cpu(),
     custom_steps.test_unix_python3_mkldnn_mkl_cpu(),
     custom_steps.test_unix_scala_cpu(),
+    custom_steps.test_unix_scala_mkldnn_cpu(),
     custom_steps.test_unix_clojure_cpu(),
     custom_steps.test_unix_r_cpu(),
     custom_steps.test_unix_julia_cpu(),

diff --git a/contrib/clojure-package/README.md b/contrib/clojure-package/README.md
@@ -193,8 +193,8 @@ The outcome of this step will be a shared library `lib/libmxnet.so` that is used
 - Build and install the Scala package in your local Maven directory using the following commands:
 
   ```bash
-  make scalapkg
-  make scalainstall
+  cd scala-package
+  mvn install
   ```
 
 #### Building the Clojure jar

diff --git a/...test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala b/...test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala
@@ -29,8 +29,7 @@ import org.slf4j.LoggerFactory
 import scala.sys.process.Process
 
 /**
-  * Integration test for imageClassifier example.
-  * This will run as a part of "make scalatest"
+  * Integration test for CNN example.
   */
 class CNNClassifierExampleSuite extends FunSuite with BeforeAndAfterAll {
   private val logger = LoggerFactory.getLogger(classOf[CNNClassifierExampleSuite])

diff --git a/...st/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala b/...st/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala
@@ -27,7 +27,6 @@ import sys.process.Process
 
 /**
   * Integration test for imageClassifier example.
-  * This will run as a part of "make scalatest"
   */
 class ImageClassifierExampleSuite extends FunSuite with BeforeAndAfterAll {
   private val logger = LoggerFactory.getLogger(classOf[ImageClassifierExampleSuite])

diff --git a/...a-package/examples/src/test/scala/org/apache/mxnetexamples/multitask/MultiTaskSuite.scala b/...a-package/examples/src/test/scala/org/apache/mxnetexamples/multitask/MultiTaskSuite.scala
@@ -25,8 +25,7 @@ import org.scalatest.FunSuite
 
 
 /**
-  * Integration test for imageClassifier example.
-  * This will run as a part of "make scalatest"
+  * Integration test for Multi-task example.
   */
 class MultiTaskSuite extends FunSuite {
   test("Multitask Test") {

diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/profiler/ProfilerSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/profiler/ProfilerSuite.scala
@@ -25,8 +25,7 @@ import org.apache.mxnet.Profiler
 import org.apache.mxnet.Context
 
 /**
-  * Integration test for imageClassifier example.
-  * This will run as a part of "make scalatest"
+  * Integration test for profiler example.
   */
 class ProfilerSuite extends FunSuite with BeforeAndAfterAll {
   private val logger = LoggerFactory.getLogger(classOf[ProfilerSuite])

diff --git a/scala-package/spark/README.md b/scala-package/spark/README.md
@@ -16,7 +16,8 @@ Checkout the [Installation Guide](http://mxnet.io/get_started/setup.html) contai
 Compile the Scala Package by
 
 ```bash
-make scalapkg
+cd scala-package
+mvn package
 ```
 
 This will automatically build the `spark` submodule. Now you can submit Spark job with these built jars.

diff --git a/tests/jenkins/run_test.sh b/tests/jenkins/run_test.sh
@@ -56,8 +56,8 @@ nosetests3 --verbose tests/python/train || exit -1
 
 echo "BUILD scala_test"
 export PATH=$PATH:/opt/apache-maven/bin
-make scalapkg || exit -1
-make scalatest || exit -1
+cd scala-package
+mvn install || exit -1
 
 # echo "BUILD julia_test"
 # export MXNET_HOME="${PWD}"

diff --git a/tests/jenkins/run_test_amzn_linux_gpu.sh b/tests/jenkins/run_test_amzn_linux_gpu.sh
@@ -65,5 +65,5 @@ nosetests3 --verbose tests/python/train
 #julia -e 'try Pkg.clone("MXNet"); catch end; Pkg.checkout("MXNet"); Pkg.build("MXNet"); Pkg.test("MXNet")' || exit -1
 
 echo "BUILD scala_test"
-make scalapkg
-make scalatest
+cd scala-package
+mvn integration-test
diff --git a/tests/jenkins/run_test_ubuntu.sh b/tests/jenkins/run_test_ubuntu.sh
@@ -68,6 +68,6 @@ nosetests3 --verbose tests/python/train || exit 1
 
 echo "BUILD scala_test"
 export PATH=$PATH:/opt/apache-maven/bin
-make scalapkg || exit 1
-make scalatest || exit 1
+cd scala-package
+mvn integration-test || exit 1