Skip to content

Commit

Permalink
[MXNET-1155] Add scala packageTest utility (apache#13046)
Browse files Browse the repository at this point in the history
* [MXNET-1155] Add scala packageTest utility

* Clean up utility

* Safe change directory in Makefile for scala

* mvn install file instructions with details
  • Loading branch information
zachgk authored and lanking520 committed Apr 26, 2019
1 parent 19d78e5 commit b2b49a5
Show file tree
Hide file tree
Showing 11 changed files with 523 additions and 9 deletions.
35 changes: 26 additions & 9 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -600,61 +600,78 @@ rpkgtest:
Rscript -e 'res<-covr:::package_coverage("R-package");fileConn<-file(paste("r-package_coverage_",toString(runif(1)),".json"));writeLines(covr:::to_codecov(res), fileConn);close(fileConn)'

scalaclean:
(cd $(ROOTDIR)/scala-package; \
(cd $(ROOTDIR)/scala-package && \
mvn clean -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE))

scalatestcompile:
(cd $(ROOTDIR)/scala-package && \
mvn test-compile -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \
-Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-Dcurrent_libdir="$(ROOTDIR)/lib" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")

scalapkg:
(cd $(ROOTDIR)/scala-package; \
(cd $(ROOTDIR)/scala-package && \
mvn package -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \
-Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-Dcurrent_libdir="$(ROOTDIR)/lib" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")

scalaunittest:
(cd $(ROOTDIR)/scala-package; \
(cd $(ROOTDIR)/scala-package && \
mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),unittest -Dcxx="$(CXX)" \
-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS))

scalaintegrationtest:
(cd $(ROOTDIR)/scala-package; \
(cd $(ROOTDIR)/scala-package && \
mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),integrationtest -Dcxx="$(CXX)" \
-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS))

scalainstall:
(cd $(ROOTDIR)/scala-package; \
(cd $(ROOTDIR)/scala-package && \
mvn install -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests=true -Dcxx="$(CXX)" \
-Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")

scalarelease-dryrun:
(cd $(ROOTDIR)/scala-package; \
(cd $(ROOTDIR)/scala-package && \
mvn release:clean release:prepare -DdryRun=true -DautoVersionSubmodules=true \
-Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \
-Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""")

scalarelease-prepare:
(cd $(ROOTDIR)/scala-package; \
(cd $(ROOTDIR)/scala-package && \
mvn release:clean release:prepare -DautoVersionSubmodules=true \
-Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \
-Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""")

scalarelease-perform:
(cd $(ROOTDIR)/scala-package; \
(cd $(ROOTDIR)/scala-package && \
mvn release:perform -DautoVersionSubmodules=true \
-Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \
-Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""")

scaladeploy:
(cd $(ROOTDIR)/scala-package; \
(cd $(ROOTDIR)/scala-package && \
mvn deploy -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
-Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")

scaladeploylocal:
(cd $(ROOTDIR)/scala-package && \
mvn deploy -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
-DaltDeploymentRepository=snapshot-repo::default::file:local-snapshot \
-Dgpg.skip \
-Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")

jnilint:
3rdparty/dmlc-core/scripts/lint.py mxnet-jnicpp cpp scala-package/native/src

Expand Down
1 change: 1 addition & 0 deletions scala-package/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,4 @@ core/src/main/scala/org/apache/mxnet/SymbolAPIBase.scala
core/src/main/scala/org/apache/mxnet/SymbolBase.scala
examples/scripts/infer/images/
examples/scripts/infer/models/
local-snapshot
14 changes: 14 additions & 0 deletions scala-package/examples/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,20 @@
<platform>linux-x86_64-gpu</platform>
</properties>
</profile>
<profile>
<id>deployLocal</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>release</id>
<build>
Expand Down
87 changes: 87 additions & 0 deletions scala-package/packageTest/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

SCALA_VERSION_PROFILE := 2.11
SCALA_VERSION := 2.11.8
MXNET_VERSION := "[1.3.0-SNAPSHOT,)"

MXNET_REPO = https://repository.apache.org/content/repositories/snapshots

ifeq ($(OS),Windows_NT)
UNAME_S := Windows
else
UNAME_S := $(shell uname -s)
endif

ifeq ($(UNAME_S), Windows)
# TODO: currently scala package does not support windows
SCALA_PKG_PROFILE := windows
else
ifeq ($(UNAME_S), Darwin)
SCALA_PKG_PROFILE := osx-x86_64-cpu
else
SCALA_PKG_PROFILE := linux-x86_64
ifeq ($(USE_CUDA), 1)
SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-gpu
else
SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-cpu
endif
endif
endif

PROFILES := -Ptest
ifeq ($(UNIT), 1)
PROFILES := "$(PROFILES),unittest"
endif
ifeq ($(INTEGRATION), 1)
PROFILES := "$(PROFILES),integrationtest"
endif

ifneq ($(UNIT), 1)
ifneq ($(INTEGRATION), 1)
PROFILES := "$(PROFILES),unittest,integrationtest"
endif
endif


clean:
(mvn clean -Dmxnet.profile=$(SCALA_PKG_PROFILE) \
-Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \
-Dmxnet.version=$(MXNET_VERSION) \
-Dscala.version=$(SCALA_VERSION))

testinstall:
(mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \
$(PROFILES) \
-Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \
-Dmxnet.version=$(MXNET_VERSION) \
-Dscala.version=$(SCALA_VERSION))

testlocal:
(mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \
$(PROFILES),fromLocal \
-Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \
-Dmxnet.version=$(MXNET_VERSION) \
-Dscala.version=$(SCALA_VERSION))

testsnapshot:
(mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \
$(PROFILES),fromSnapshots \
-Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \
-Dmxnet.repo=$(MXNET_REPO) \
-Dmxnet.version=$(MXNET_VERSION) \
-Dscala.version=$(SCALA_VERSION))
72 changes: 72 additions & 0 deletions scala-package/packageTest/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
# MXNet Scala Package Test

This is an project created to run the test suite on a fully packaged mxnet jar. The test suite is found locally but mxnet is from the target jarfile.

## General Setup

To setup the packageTest, you must first build your tests. To build the tests, follow these steps from the mxnet main directory:

1. Build MXNet and the scala package from source following the directions [here](https://mxnet.incubator.apache.org/install/scala_setup.html#source)
2. Build the tests by running `make scalatestcompile`.
3. Follow setup instructions below for your testing goal

## Running

There are three different modes of operation for testing based on the location of the jar and where it is coming from:

### Test Installed Jars

If you have a jar file, you can install it to your maven cache repository(`~/.m2/repository`). This might be useful if you acquire the .jar file from elsewhere. To install, it is easiest to use `mvn install:install-file -Dfile=<path-to-file> -DpomFile=<path-to-pomfile>`. If the pom file is not available, you can also run `mvn install:install-file -Dfile=<path-to-file> -DgroupId=<group-id> -DartifactId=<artifact-id> -Dversion=<version> -Dpackaging=<packaging>`. With the full mxnet jar, this might look like `mvn install:install-file -Dfile=<path-to-file> -DgroupId=org.apache.mxnet -DartifactId=mxnet-full_2.11-linux-x86_64-cpu -Dversion=1.3.0 -Dpackaging=jar`.

You can also run `make scalainstall` to install from a local build.

After installing, run `make testinstall` in the package test directory to run the tests. Note that unless you also install an additional mxnetexamples jar, you can only run the unit tests.

### Test Local Deployment

To test the jars that would be produced by a deployment, you can run `make scaladeploylocal` from the main mxnet directory. This produces a local snapshot located at `scala-package/local-snapshot`. To test this local snapshot, run `make testlocal`.

### Remote Repository Snapshot

This mode is to test a jar located in a remote repository. The default repository is the apache snapshot repisotory located at `https://repository.apache.org/content/repositories/snapshots`. Note that the actual jar in a repisotory should be located at `$repoUrl/org/apache/mxnet/mxnet-full_$scalaVersion-$osMode/$version/*.jar`.

Test the snapshot repo using `make testsnapshot` or a different repo using `make testsnapshot MXNET_REPO=$NEW_REPO_URL`.

### Options

You are able to run unit tests, integration tests, or both using this utility. To run the unit tests, add the flag `UNIT=1` to make (e.g. `make testsnapshot UNIT=1`). Use `INTEGRATION=1` for integration tests. The default behavior is to run both the unit and integration tests. However, the integration tests require that the mxnet examples be installed in addition to the full mxnet package (see test mode instructions above).

An additional option, you can specify the mxnet version with `MXNET_VERSION=1.3.1-SNAPSHOT`.

## Cleaning Up

You can clean temporary files and target artifacts by running `make scalaclean`.

## Troubleshooting

### Missing Examples

If you fail with the following error
```
[ERROR] Failed to execute goal org.scalatest:scalatest-maven-plugin:1.0:test (test) on project mxnet-scala-packagetest-examples_2.11: There are test failures -> [Help 1]
[ERROR]
[ERROR] To see the full stack trace of the errors, re-run Maven with the -e switch.
[ERROR] Re-run Maven using the -X switch to enable full debug logging.
[ERROR]
[ERROR] For more information about the errors and possible solutions, please read the following articles:
[ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/MojoFailureException
[ERROR]
[ERROR] After correcting the problems, you can resume the build with the command
[ERROR] mvn <goals> -rf :mxnet-scala-packagetest-examples_2.11
Makefile:57: recipe for target 'scalaintegrationtest' failed
make: *** [scalaintegrationtest] Error 1
```

and stacktrace begins with the following,

```
*** RUN ABORTED ***
java.lang.NoClassDefFoundError: org/apache/mxnetexamples/Util$
```

you are missing the mxnetexamples package. See your test mode installation section for details.
39 changes: 39 additions & 0 deletions scala-package/packageTest/core/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>PackageTest</groupId>
<artifactId>mxnet-scala-packagetest_2.11</artifactId>
<version>1.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

<artifactId>mxnet-scala-packagetest-core_2.11</artifactId>
<name>MXNet Scala Package Test</name>
<packaging>pom</packaging>

<profiles>
<profile>
<id>unittest</id>
<properties>
<skipTests>false</skipTests>
</properties>
</profile>
</profiles>

<build>
<plugins>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>1.0</version>
<configuration>
<runpath>${project.build.outputDirectory},${project.build.testOutputDirectory},../../core/target/test-classes</runpath>
</configuration>
</plugin>
</plugins>
</build>

</project>
1 change: 1 addition & 0 deletions scala-package/packageTest/core/scripts
48 changes: 48 additions & 0 deletions scala-package/packageTest/examples/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>PackageTest</groupId>
<artifactId>mxnet-scala-packagetest_2.11</artifactId>
<version>1.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

<artifactId>mxnet-scala-packagetest-examples_2.11</artifactId>
<name>MXNet Scala Package Test</name>
<packaging>pom</packaging>

<profiles>
<profile>
<id>integrationtest</id>
<properties>
<skipTests>false</skipTests>
</properties>
</profile>
</profiles>

<build>
<plugins>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>1.0</version>
<configuration>
<runpath>${project.build.outputDirectory},${project.build.testOutputDirectory},../../examples/target/test-classes</runpath>
</configuration>
</plugin>
</plugins>
</build>

<dependencies>
<dependency>
<groupId>org.apache.mxnet</groupId>
<artifactId>mxnet-examples_${mxnet.scalaprofile}</artifactId>
<version>${mxnet.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

</project>
1 change: 1 addition & 0 deletions scala-package/packageTest/examples/scripts
Loading

0 comments on commit b2b49a5

Please sign in to comment.