From a0bf25c56004cb7b0efe697f44cf8267ddc5b5e7 Mon Sep 17 00:00:00 2001 From: Zach Kimberg Date: Mon, 22 Oct 2018 22:05:30 -0700 Subject: [PATCH] [MXNET-1155] Add scala packageTest utility --- Makefile | 8 + scala-package/core/pom.xml | 8 + scala-package/examples/pom.xml | 8 + scala-package/infer/pom.xml | 8 + scala-package/macros/pom.xml | 8 + scala-package/packageTest/Makefile | 67 ++++++++ scala-package/packageTest/README.md | 50 ++++++ scala-package/packageTest/core/pom.xml | 30 ++++ scala-package/packageTest/core/scripts | 1 + scala-package/packageTest/examples/pom.xml | 39 +++++ scala-package/packageTest/examples/scripts | 1 + scala-package/packageTest/infer/pom.xml | 29 ++++ scala-package/packageTest/pom.xml | 173 +++++++++++++++++++++ 13 files changed, 430 insertions(+) create mode 100644 scala-package/packageTest/Makefile create mode 100644 scala-package/packageTest/README.md create mode 100644 scala-package/packageTest/core/pom.xml create mode 120000 scala-package/packageTest/core/scripts create mode 100644 scala-package/packageTest/examples/pom.xml create mode 120000 scala-package/packageTest/examples/scripts create mode 100644 scala-package/packageTest/infer/pom.xml create mode 100644 scala-package/packageTest/pom.xml diff --git a/Makefile b/Makefile index 620679a44f80..6d6850d25fb5 100644 --- a/Makefile +++ b/Makefile @@ -602,6 +602,14 @@ scalaclean: (cd $(ROOTDIR)/scala-package; \ mvn clean -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE)) +scalatestcompile: + (cd $(ROOTDIR)/scala-package; \ + mvn test-compile -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ + -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ + -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ + -Dcurrent_libdir="$(ROOTDIR)/lib" \ + -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") + scalapkg: (cd $(ROOTDIR)/scala-package; \ mvn package -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml index e93169f08faa..985491c7624c 100644 --- a/scala-package/core/pom.xml +++ b/scala-package/core/pom.xml @@ -65,6 +65,14 @@ META-INF/*.RSA + + + + test-jar + + + + org.apache.maven.plugins diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml index 436f2992768b..2d10ddb4ee79 100644 --- a/scala-package/examples/pom.xml +++ b/scala-package/examples/pom.xml @@ -121,6 +121,14 @@ org.apache.maven.plugins maven-jar-plugin + + + + test-jar + + + + org.apache.maven.plugins diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml index e50100169328..e5d213dfc470 100644 --- a/scala-package/infer/pom.xml +++ b/scala-package/infer/pom.xml @@ -65,6 +65,14 @@ META-INF/*.RSA + + + + test-jar + + + + org.apache.maven.plugins diff --git a/scala-package/macros/pom.xml b/scala-package/macros/pom.xml index ae4c60a1e6d0..6b1ffd0fbcbe 100644 --- a/scala-package/macros/pom.xml +++ b/scala-package/macros/pom.xml @@ -90,6 +90,14 @@ META-INF/*.RSA + + + + test-jar + + + + org.apache.maven.plugins diff --git a/scala-package/packageTest/Makefile b/scala-package/packageTest/Makefile new file mode 100644 index 000000000000..e9f065fac7d3 --- /dev/null +++ b/scala-package/packageTest/Makefile @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +SCALA_VERSION_PROFILE := 2.11 +SCALA_VERSION := 2.11.8 +MXNET_VERSION := 1.3.1-SNAPSHOT + +ifeq ($(OS),Windows_NT) + UNAME_S := Windows +else + UNAME_S := $(shell uname -s) +endif + +ifeq ($(UNAME_S), Windows) + # TODO: currently scala package does not support windows + SCALA_PKG_PROFILE := windows +else + ifeq ($(UNAME_S), Darwin) + SCALA_PKG_PROFILE := osx-x86_64-cpu + else + SCALA_PKG_PROFILE := linux-x86_64 + ifeq ($(USE_CUDA), 1) + SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-gpu + else + SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-cpu + endif + endif +endif + +scalapkg: + (mvn package -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ + -Dmxnet.version=$(MXNET_VERSION) \ + -Dscala.version=$(SCALA_VERSION)) + +scalaclean: + (mvn clean -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ + -Dmxnet.version=$(MXNET_VERSION) \ + -Dscala.version=$(SCALA_VERSION)) + +scalaintegrationtest: + (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + -PwithExamples \ + -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ + -Dmxnet.version=$(MXNET_VERSION) \ + -Dscala.version=$(SCALA_VERSION)) + +scalaintegrationtestwithoutexamples: + (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ + -Dmxnet.version=$(MXNET_VERSION) \ + -Dscala.version=$(SCALA_VERSION)) diff --git a/scala-package/packageTest/README.md b/scala-package/packageTest/README.md new file mode 100644 index 000000000000..7cdb61bd7acd --- /dev/null +++ b/scala-package/packageTest/README.md @@ -0,0 +1,50 @@ +# MXNet Scala Package Test + +This is an project created to run the test suite on a fully packaged mxnet jar. + +## Setup + +### Install Package + +To run the test suite, first install the package. This can be done either by installing directly from a jar with `mvn install:install-file -Dfile=` or by running `make scalainstall` in the main mxnet folder. Note that if you use `mvn install:install-file`, you will be unable to run the example tests unless you also install the mxnetexamples jar. You can run all tests except for those examples with `make scalaintegrationtestwithoutexamples`. + +### Build + +Build the mxnet tests by running `make scalatestcompile` from the main mxnet directory. This is needed for test discovery. + +## Run + +To run, ensure the versions are correct in the `Makefile`. Then, just run `make scalaintegrationtest` to execute the test suite + +## Clean + +You can clean temporary files and target artifacts by running `make scalaclean`. + +## Troubleshooting + +### Missing Examples + +If you fail with the following error +``` +[ERROR] Failed to execute goal org.scalatest:scalatest-maven-plugin:1.0:test (test) on project mxnet-scala-packagetest-examples_2.11: There are test failures -> [Help 1] +[ERROR] +[ERROR] To see the full stack trace of the errors, re-run Maven with the -e switch. +[ERROR] Re-run Maven using the -X switch to enable full debug logging. +[ERROR] +[ERROR] For more information about the errors and possible solutions, please read the following articles: +[ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/MojoFailureException +[ERROR] +[ERROR] After correcting the problems, you can resume the build with the command +[ERROR] mvn -rf :mxnet-scala-packagetest-examples_2.11 +Makefile:57: recipe for target 'scalaintegrationtest' failed +make: *** [scalaintegrationtest] Error 1 +``` + +and stacktrace begins with the following, + +``` +*** RUN ABORTED *** + java.lang.NoClassDefFoundError: org/apache/mxnetexamples/Util$ +``` + +you are missing the mxnetexamples package. See the "Install Package" section for details. diff --git a/scala-package/packageTest/core/pom.xml b/scala-package/packageTest/core/pom.xml new file mode 100644 index 000000000000..71ace9bf58d7 --- /dev/null +++ b/scala-package/packageTest/core/pom.xml @@ -0,0 +1,30 @@ + + + 4.0.0 + + PackageTest + mxnet-scala-packagetest_2.11 + 1.0-SNAPSHOT + ../pom.xml + + + mxnet-scala-packagetest-core_2.11 + MXNet Scala Package Test + pom + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.outputDirectory},${project.build.testOutputDirectory},../../core/target/test-classes + + + + + + diff --git a/scala-package/packageTest/core/scripts b/scala-package/packageTest/core/scripts new file mode 120000 index 000000000000..f806668aa847 --- /dev/null +++ b/scala-package/packageTest/core/scripts @@ -0,0 +1 @@ +../../core/scripts \ No newline at end of file diff --git a/scala-package/packageTest/examples/pom.xml b/scala-package/packageTest/examples/pom.xml new file mode 100644 index 000000000000..722eb50762b8 --- /dev/null +++ b/scala-package/packageTest/examples/pom.xml @@ -0,0 +1,39 @@ + + + 4.0.0 + + PackageTest + mxnet-scala-packagetest_2.11 + 1.0-SNAPSHOT + ../pom.xml + + + mxnet-scala-packagetest-examples_2.11 + MXNet Scala Package Test + pom + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.outputDirectory},${project.build.testOutputDirectory},../../examples/target/test-classes + + + + + + + + org.apache.mxnet + mxnet-examples_${mxnet.scalaprofile} + ${mxnet.version} + test + + + + diff --git a/scala-package/packageTest/examples/scripts b/scala-package/packageTest/examples/scripts new file mode 120000 index 000000000000..2bba4eeece74 --- /dev/null +++ b/scala-package/packageTest/examples/scripts @@ -0,0 +1 @@ +../../examples/scripts \ No newline at end of file diff --git a/scala-package/packageTest/infer/pom.xml b/scala-package/packageTest/infer/pom.xml new file mode 100644 index 000000000000..409145ea2899 --- /dev/null +++ b/scala-package/packageTest/infer/pom.xml @@ -0,0 +1,29 @@ + + + 4.0.0 + + PackageTest + mxnet-scala-packagetest_2.11 + 1.0-SNAPSHOT + ../pom.xml + + + mxnet-scala-packagetest-infer_2.11 + MXNet Scala Package Test + pom + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.outputDirectory},${project.build.testOutputDirectory},../../infer/target/test-classes + + + + + diff --git a/scala-package/packageTest/pom.xml b/scala-package/packageTest/pom.xml new file mode 100644 index 000000000000..805bc685691e --- /dev/null +++ b/scala-package/packageTest/pom.xml @@ -0,0 +1,173 @@ + + + 4.0.0 + PackageTest + mxnet-scala-packagetest_2.11 + 1.0-SNAPSHOT + MXNet Scala Package Test + pom + + + core + infer + + + + + withExamples + + examples + + + + + + + org.apache.mxnet + mxnet-full_${mxnet.scalaprofile}-${mxnet.profile} + ${mxnet.version} + + + org.apache.mxnet + mxnet-core_${mxnet.scalaprofile} + ${mxnet.version} + tests + test-jar + test + + + org.scala-lang + scala-library + ${scala.version} + + + commons-io + commons-io + 2.4 + + + org.scalatest + scalatest_${mxnet.scalaprofile} + 3.0.4 + test + + + org.scalacheck + scalacheck_${mxnet.scalaprofile} + 1.13.5 + test + + + org.mockito + mockito-all + 1.10.19 + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.6 + 1.6 + UTF-8 + + + + maven-resources-plugin + 2.7 + + + org.apache.maven.plugins + maven-dependency-plugin + 2.9 + + + copy-dependencies + package + + copy-dependencies + + + ${project.build.outputDirectory}/lib + runtime + test,provided + false + false + true + + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.5 + + + package + + jar + test-jar + + + + **/* + + + + + + + net.alchim31.maven + scala-maven-plugin + 3.2.2 + + + compile + + compile + + compile + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.19 + + true + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${skipTests} + ${project.build.directory}/surefire-reports + . + F + WDF TestSuite.txt + + + + test + integration-test + + test + + + + + + + +