From 4f2bcb3366816a467f0ffbc8ad128f73eed25708 Mon Sep 17 00:00:00 2001 From: Zach Kimberg Date: Fri, 30 Nov 2018 15:12:13 -0800 Subject: [PATCH] Clean up utility --- Makefile | 9 +++++ scala-package/.gitignore | 1 + scala-package/core/pom.xml | 8 ----- scala-package/examples/pom.xml | 22 ++++++++----- scala-package/infer/pom.xml | 8 ----- scala-package/macros/pom.xml | 8 ----- scala-package/packageTest/Makefile | 36 ++++++++++++-------- scala-package/packageTest/README.md | 38 ++++++++++++++++------ scala-package/packageTest/core/pom.xml | 9 +++++ scala-package/packageTest/examples/pom.xml | 9 +++++ scala-package/packageTest/infer/pom.xml | 9 +++++ scala-package/packageTest/pom.xml | 22 +++++++++++-- 12 files changed, 122 insertions(+), 57 deletions(-) diff --git a/Makefile b/Makefile index 65a191b03e59..218bfa0cd607 100644 --- a/Makefile +++ b/Makefile @@ -663,6 +663,15 @@ scaladeploy: -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") +scaladeploylocal: + (cd $(ROOTDIR)/scala-package; \ + mvn deploy -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \ + -DaltDeploymentRepository=snapshot-repo::default::file:local-snapshot \ + -Dgpg.skip \ + -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ + -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ + -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") + jnilint: 3rdparty/dmlc-core/scripts/lint.py mxnet-jnicpp cpp scala-package/native/src diff --git a/scala-package/.gitignore b/scala-package/.gitignore index 6aa4da6b1cfc..8bc87f53e802 100644 --- a/scala-package/.gitignore +++ b/scala-package/.gitignore @@ -6,3 +6,4 @@ core/src/main/scala/org/apache/mxnet/SymbolAPIBase.scala core/src/main/scala/org/apache/mxnet/SymbolBase.scala examples/scripts/infer/images/ examples/scripts/infer/models/ +local-snapshot \ No newline at end of file diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml index ba24334d2e4d..3425bb15f62a 100644 --- a/scala-package/core/pom.xml +++ b/scala-package/core/pom.xml @@ -63,14 +63,6 @@ META-INF/*.RSA - - - - test-jar - - - - org.apache.maven.plugins diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml index e10d11e679d9..80d484aebb2e 100644 --- a/scala-package/examples/pom.xml +++ b/scala-package/examples/pom.xml @@ -42,6 +42,20 @@ linux-x86_64-gpu + + deployLocal + + + + org.apache.maven.plugins + maven-deploy-plugin + + false + + + + + release @@ -119,14 +133,6 @@ org.apache.maven.plugins maven-jar-plugin - - - - test-jar - - - - org.apache.maven.plugins diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml index 8c5f267c17e9..3e6980cb6f4b 100644 --- a/scala-package/infer/pom.xml +++ b/scala-package/infer/pom.xml @@ -63,14 +63,6 @@ META-INF/*.RSA - - - - test-jar - - - - org.apache.maven.plugins diff --git a/scala-package/macros/pom.xml b/scala-package/macros/pom.xml index 891652306457..d435e211ceeb 100644 --- a/scala-package/macros/pom.xml +++ b/scala-package/macros/pom.xml @@ -90,14 +90,6 @@ META-INF/*.RSA - - - - test-jar - - - - org.apache.maven.plugins diff --git a/scala-package/packageTest/Makefile b/scala-package/packageTest/Makefile index 315cf6f8a131..6073ff8a722f 100644 --- a/scala-package/packageTest/Makefile +++ b/scala-package/packageTest/Makefile @@ -17,7 +17,7 @@ SCALA_VERSION_PROFILE := 2.11 SCALA_VERSION := 2.11.8 -MXNET_VERSION := 1.3.1-SNAPSHOT +MXNET_VERSION := "[1.3.0-SNAPSHOT,)" MXNET_REPO = https://repository.apache.org/content/repositories/snapshots @@ -43,35 +43,45 @@ else endif endif -scalapkg: - (mvn package -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ - -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ - -Dmxnet.version=$(MXNET_VERSION) \ - -Dscala.version=$(SCALA_VERSION)) +PROFILES := -Ptest +ifeq ($(UNIT), 1) + PROFILES := "$(PROFILES),unittest" +endif +ifeq ($(INTEGRATION), 1) + PROFILES := "$(PROFILES),integrationtest" +endif -scalaclean: +ifneq ($(UNIT), 1) + ifneq ($(INTEGRATION), 1) + PROFILES := "$(PROFILES),unittest,integrationtest" + endif +endif + + +clean: (mvn clean -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ -Dmxnet.version=$(MXNET_VERSION) \ -Dscala.version=$(SCALA_VERSION)) -scalaintegrationtest: +testinstall: (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ - -PwithExamples \ + $(PROFILES) \ -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ -Dmxnet.version=$(MXNET_VERSION) \ -Dscala.version=$(SCALA_VERSION)) -scalaintegrationtestsnapshot: +testlocal: (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ - -PfromSnapshots \ + $(PROFILES),fromLocal \ -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ - -Dmxnet.repo=$(MXNET_REPO) \ -Dmxnet.version=$(MXNET_VERSION) \ -Dscala.version=$(SCALA_VERSION)) -scalaintegrationtestwithoutexamples: +testsnapshot: (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + $(PROFILES),fromSnapshots \ -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ + -Dmxnet.repo=$(MXNET_REPO) \ -Dmxnet.version=$(MXNET_VERSION) \ -Dscala.version=$(SCALA_VERSION)) diff --git a/scala-package/packageTest/README.md b/scala-package/packageTest/README.md index 0972ea35836b..6affb90d3602 100644 --- a/scala-package/packageTest/README.md +++ b/scala-package/packageTest/README.md @@ -1,22 +1,40 @@ # MXNet Scala Package Test -This is an project created to run the test suite on a fully packaged mxnet jar. +This is an project created to run the test suite on a fully packaged mxnet jar. The test suite is found locally but mxnet is from the target jarfile. -## Setup +## General Setup -### Install Package +To setup the packageTest, you must first build your tests. To build the tests, follow these steps from the mxnet main directory: -To run the test suite, first install the package. This can be done either by installing directly from a jar with `mvn install:install-file -Dfile=` or by running `make scalainstall` in the main mxnet folder. Note that if you use `mvn install:install-file`, you will be unable to run the example tests unless you also install the mxnetexamples jar. You can run all tests except for those examples with `make scalaintegrationtestwithoutexamples`. +1. Build MXNet and the scala package from source following the directions [here](https://mxnet.incubator.apache.org/install/scala_setup.html#source) +2. Build the tests by running `make scalatestcompile`. +3. Follow setup instructions below for your testing goal -### Build +## Running -Build the mxnet tests by running `make scalapkg` and then `make scalatestcompile` from the main mxnet directory. This is needed for test discovery. +There are three different modes of operation for testing based on the location of the jar and where it is coming from: -## Run +### Test Installed Jars -To run, ensure the versions are correct in the `Makefile`. Then, just run `make scalaintegrationtest` to execute the test suite +If you have a jar file, you can install it to your maven cache repository(`~/.m2/repository`) using `mvn install:install-file -Dfile=`. This might be useful if you acquire the .jar file from elsewhere. You can also run `make scalainstall` to install from a local build. Then, run `make testinstall` in the package test directory to run the tests. Note that unless you also install an additional mxnetexamples jar, you can only run the unit tests. -## Clean +### Test Local Deployment + +To test the jars that would be produced by a deployment, you can run `make scaladeploylocal` from the main mxnet directory. This produces a local snapshot located at `scala-package/local-snapshot`. To test this local snapshot, run `make testlocal`. + +### Remote Repository Snapshot + +This mode is to test a jar located in a remote repository. The default repository is the apache snapshot repisotory located at `https://repository.apache.org/content/repositories/snapshots`. Note that the actual jar in a repisotory should be located at `$repoUrl/org/apache/mxnet/mxnet-full_$scalaVersion-$osMode/$version/*.jar`. + +Test the snapshot repo using `make testsnapshot` or a different repo using `make testsnapshot MXNET_REPO=$NEW_REPO_URL`. + +### Options + +You are able to run unit tests, integration tests, or both using this utility. To run the unit tests, add the flag `UNIT=1` to make (e.g. `make testsnapshot UNIT=1`). Use `INTEGRATION=1` for integration tests. The default behavior is to run both the unit and integration tests. However, the integration tests require that the mxnet examples be installed in addition to the full mxnet package (see test mode instructions above). + +An additional option, you can specify the mxnet version with `MXNET_VERSION=1.3.1-SNAPSHOT`. + +## Cleaning Up You can clean temporary files and target artifacts by running `make scalaclean`. @@ -47,4 +65,4 @@ and stacktrace begins with the following, java.lang.NoClassDefFoundError: org/apache/mxnetexamples/Util$ ``` -you are missing the mxnetexamples package. See the "Install Package" section for details. +you are missing the mxnetexamples package. See your test mode installation section for details. diff --git a/scala-package/packageTest/core/pom.xml b/scala-package/packageTest/core/pom.xml index 71ace9bf58d7..bdcd7662f082 100644 --- a/scala-package/packageTest/core/pom.xml +++ b/scala-package/packageTest/core/pom.xml @@ -14,6 +14,15 @@ MXNet Scala Package Test pom + + + unittest + + false + + + + diff --git a/scala-package/packageTest/examples/pom.xml b/scala-package/packageTest/examples/pom.xml index 722eb50762b8..e11be657e225 100644 --- a/scala-package/packageTest/examples/pom.xml +++ b/scala-package/packageTest/examples/pom.xml @@ -14,6 +14,15 @@ MXNet Scala Package Test pom + + + integrationtest + + false + + + + diff --git a/scala-package/packageTest/infer/pom.xml b/scala-package/packageTest/infer/pom.xml index 409145ea2899..7c5a096d6e14 100644 --- a/scala-package/packageTest/infer/pom.xml +++ b/scala-package/packageTest/infer/pom.xml @@ -14,6 +14,15 @@ MXNet Scala Package Test pom + + + unittest + + false + + + + diff --git a/scala-package/packageTest/pom.xml b/scala-package/packageTest/pom.xml index 1c16448c0692..9c5c11cf2779 100644 --- a/scala-package/packageTest/pom.xml +++ b/scala-package/packageTest/pom.xml @@ -16,7 +16,10 @@ - withExamples + test + + + integrationtest examples @@ -34,8 +37,24 @@ + + fromLocal + + + local-snapshot + file://${basedir}/../local-snapshot + + true + + + + + + true + + org.apache.mxnet @@ -119,7 +138,6 @@ package jar - test-jar