Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Add new Maven build for Scala package #13819

Merged
merged 5 commits into from
Jan 10, 2019
Merged

Add new Maven build for Scala package #13819

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
53534cc
clean up build
lanking520 Jan 9, 2019
dfa99d6
fix minor issue and add mkldnn
lanking520 Jan 9, 2019
13d76a5
fix mx_dist problem
lanking520 Jan 9, 2019
14f09ed
fix clojure build
lanking520 Jan 9, 2019
46c8d57
fix skip test
lanking520 Jan 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 10 additions & 8 deletions ci/docker/runtime_functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -847,23 +847,24 @@ unittest_ubuntu_python3_quantization_gpu() {
unittest_ubuntu_cpu_scala() {
set -ex
scala_prepare
make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
cd scala-package
mvn -B integration-test
}

unittest_centos7_cpu_scala() {
set -ex
cd /work/mxnet
scala_prepare
make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
cd scala-package
mvn -B integration-test
}

unittest_ubuntu_cpu_clojure() {
set -ex
scala_prepare
make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
make scalainstall USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
cd scala-package
mvn -B install
cd ..
./contrib/clojure-package/ci-test.sh
}

Expand Down Expand Up @@ -1008,8 +1009,9 @@ integrationtest_ubuntu_cpu_dist_kvstore() {
integrationtest_ubuntu_gpu_scala() {
set -ex
scala_prepare
make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1 CI=1
make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 CI=1
cd scala-package
export SCALA_TEST_ON_GPU=1
mvn -B integration-test -DskipTests=false
}

integrationtest_ubuntu_gpu_dist_kvstore() {
Expand Down
30 changes: 21 additions & 9 deletions ci/jenkins/Jenkins_steps.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,6 @@ mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdpart
// Python wheels
mx_pip = 'build/*.whl'

// for scala build, need to pass extra libs when run with dist_kvstore
mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a'
// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
Expand Down Expand Up @@ -100,7 +98,7 @@ def compile_unix_cpu_openblas() {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas', false)
utils.pack_lib('cpu', mx_dist_lib, true)
utils.pack_lib('cpu', mx_lib, true)
}
}
}
Expand Down Expand Up @@ -128,7 +126,7 @@ def compile_unix_mkl_cpu() {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkl', false)
utils.pack_lib('cpu_mkl', mx_dist_lib, true)
utils.pack_lib('cpu_mkl', mx_mkldnn_lib, true)
}
}
}
Expand Down Expand Up @@ -254,7 +252,7 @@ def compile_centos7_cpu() {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
utils.docker_run('centos7_cpu', 'build_centos7_cpu', false)
utils.pack_lib('centos7_cpu', mx_dist_lib, true)
utils.pack_lib('centos7_cpu', mx_lib, true)
}
}
}
Expand Down Expand Up @@ -825,7 +823,21 @@ def test_unix_scala_cpu() {
node(NODE_LINUX_CPU) {
ws('workspace/ut-scala-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('cpu', mx_dist_lib, true)
utils.unpack_and_init('cpu', mx_lib, true)
lanking520 marked this conversation as resolved.
Show resolved Hide resolved
utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_scala', false)
utils.publish_test_coverage()
}
}
}
}]
}

def test_unix_scala_mkldnn_cpu(){
return ['Scala: MKLDNN-CPU': {
node(NODE_LINUX_CPU) {
ws('workspace/ut-scala-mkldnn-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('mkldnn_cpu', mx_mkldnn_lib, true)
utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_scala', false)
utils.publish_test_coverage()
}
Expand All @@ -839,7 +851,7 @@ def test_unix_scala_gpu() {
node(NODE_LINUX_GPU) {
ws('workspace/ut-scala-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('gpu', mx_dist_lib, true)
utils.unpack_and_init('gpu', mx_lib, true)
utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_scala', true)
utils.publish_test_coverage()
}
Expand All @@ -853,7 +865,7 @@ def test_unix_clojure_cpu() {
node(NODE_LINUX_CPU) {
ws('workspace/ut-clojure-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('cpu', mx_dist_lib, true)
utils.unpack_and_init('cpu', mx_lib, true)
utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_clojure', false)
utils.publish_test_coverage()
}
Expand Down Expand Up @@ -1029,7 +1041,7 @@ def test_centos7_scala_cpu() {
node(NODE_LINUX_CPU) {
ws('workspace/ut-scala-centos7-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('centos7_cpu', mx_dist_lib, true)
utils.unpack_and_init('centos7_cpu', mx_lib, true)
utils.docker_run('centos7_cpu', 'unittest_centos7_cpu_scala', false)
utils.publish_test_coverage()
}
Expand Down
1 change: 1 addition & 0 deletions ci/jenkins/Jenkinsfile_unix_cpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ core_logic: {
custom_steps.test_unix_python3_mkldnn_cpu(),
custom_steps.test_unix_python3_mkldnn_mkl_cpu(),
custom_steps.test_unix_scala_cpu(),
custom_steps.test_unix_scala_mkldnn_cpu(),
custom_steps.test_unix_clojure_cpu(),
custom_steps.test_unix_r_cpu(),
custom_steps.test_unix_julia_cpu(),
Expand Down
4 changes: 2 additions & 2 deletions contrib/clojure-package/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,8 +193,8 @@ The outcome of this step will be a shared library `lib/libmxnet.so` that is used
- Build and install the Scala package in your local Maven directory using the following commands:

```bash
make scalapkg
make scalainstall
cd scala-package
mvn install
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would rather make is : mvn clean install

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As a new user tutorial, I think it should be fine to just do install

```

#### Building the Clojure jar
Expand Down
3 changes: 1 addition & 2 deletions ...test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,7 @@ import org.slf4j.LoggerFactory
import scala.sys.process.Process

/**
* Integration test for imageClassifier example.
* This will run as a part of "make scalatest"
* Integration test for CNN example.
*/
class CNNClassifierExampleSuite extends FunSuite with BeforeAndAfterAll {
private val logger = LoggerFactory.getLogger(classOf[CNNClassifierExampleSuite])
Expand Down
1 change: 0 additions & 1 deletion ...st/scala/org/apache/mxnetexamples/infer/imageclassifier/ImageClassifierExampleSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import sys.process.Process

/**
* Integration test for imageClassifier example.
* This will run as a part of "make scalatest"
*/
class ImageClassifierExampleSuite extends FunSuite with BeforeAndAfterAll {
private val logger = LoggerFactory.getLogger(classOf[ImageClassifierExampleSuite])
Expand Down
3 changes: 1 addition & 2 deletions ...a-package/examples/src/test/scala/org/apache/mxnetexamples/multitask/MultiTaskSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,7 @@ import org.scalatest.FunSuite


/**
* Integration test for imageClassifier example.
* This will run as a part of "make scalatest"
* Integration test for Multi-task example.
*/
class MultiTaskSuite extends FunSuite {
test("Multitask Test") {
Expand Down
3 changes: 1 addition & 2 deletions scala-package/examples/src/test/scala/org/apache/mxnetexamples/profiler/ProfilerSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,7 @@ import org.apache.mxnet.Profiler
import org.apache.mxnet.Context

/**
* Integration test for imageClassifier example.
* This will run as a part of "make scalatest"
* Integration test for profiler example.
*/
class ProfilerSuite extends FunSuite with BeforeAndAfterAll {
private val logger = LoggerFactory.getLogger(classOf[ProfilerSuite])
Expand Down
3 changes: 2 additions & 1 deletion scala-package/spark/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ Checkout the [Installation Guide](http://mxnet.io/get_started/setup.html) contai
Compile the Scala Package by

```bash
make scalapkg
cd scala-package
mvn package
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why change it from install to package ?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@zachgk

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Discussed offline. This seems fine.

```

This will automatically build the `spark` submodule. Now you can submit Spark job with these built jars.
Expand Down
4 changes: 2 additions & 2 deletions tests/jenkins/run_test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@ nosetests3 --verbose tests/python/train || exit -1

echo "BUILD scala_test"
export PATH=$PATH:/opt/apache-maven/bin
make scalapkg || exit -1
make scalatest || exit -1
cd scala-package
mvn install || exit -1

# echo "BUILD julia_test"
# export MXNET_HOME="${PWD}"
Expand Down
4 changes: 2 additions & 2 deletions tests/jenkins/run_test_amzn_linux_gpu.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,5 +65,5 @@ nosetests3 --verbose tests/python/train
#julia -e 'try Pkg.clone("MXNet"); catch end; Pkg.checkout("MXNet"); Pkg.build("MXNet"); Pkg.test("MXNet")' || exit -1

echo "BUILD scala_test"
make scalapkg
make scalatest
cd scala-package
mvn integration-test -Dskip.test=false
lanking520 marked this conversation as resolved.
Show resolved Hide resolved
4 changes: 2 additions & 2 deletions tests/jenkins/run_test_ubuntu.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,6 @@ nosetests3 --verbose tests/python/train || exit 1

echo "BUILD scala_test"
export PATH=$PATH:/opt/apache-maven/bin
make scalapkg || exit 1
make scalatest || exit 1
cd scala-package
mvn integration-test || exit 1