Clean up utility

apache · Dec 6, 2018 · 4f2bcb3 · 4f2bcb3
1 parent d6ad0e4
commit 4f2bcb3
Show file tree

Hide file tree

Showing 12 changed files with 122 additions and 57 deletions.
diff --git a/Makefile b/Makefile
@@ -663,6 +663,15 @@ scaladeploy:
 			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
 			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
 
+scaladeploylocal:
+	(cd $(ROOTDIR)/scala-package; \
+		mvn deploy -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \
+		  -DaltDeploymentRepository=snapshot-repo::default::file:local-snapshot \
+		  -Dgpg.skip \
+		  -Dbuild.platform="$(SCALA_PKG_PROFILE)" \
+			-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
+			-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
+
 jnilint:
 	3rdparty/dmlc-core/scripts/lint.py mxnet-jnicpp cpp scala-package/native/src
 

diff --git a/scala-package/.gitignore b/scala-package/.gitignore
@@ -6,3 +6,4 @@ core/src/main/scala/org/apache/mxnet/SymbolAPIBase.scala
 core/src/main/scala/org/apache/mxnet/SymbolBase.scala
 examples/scripts/infer/images/
 examples/scripts/infer/models/
+local-snapshot
diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml
@@ -63,14 +63,6 @@
             <exclude>META-INF/*.RSA</exclude>
           </excludes>
         </configuration>
-        <executions>
-          <execution>
-            <goals>
-              <goal>test-jar
-              </goal>
-            </goals>
-          </execution>
-        </executions>
       </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>

diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml
@@ -42,6 +42,20 @@
         <platform>linux-x86_64-gpu</platform>
       </properties>
     </profile>
+    <profile>
+      <id>deployLocal</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-deploy-plugin</artifactId>
+            <configuration>
+              <skip>false</skip>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
     <profile>
       <id>release</id>
       <build>
@@ -119,14 +133,6 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
-        <executions>
-          <execution>
-            <goals>
-              <goal>test-jar
-              </goal>
-            </goals>
-          </execution>
-        </executions>
       </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>

diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml
@@ -63,14 +63,6 @@
                         <exclude>META-INF/*.RSA</exclude>
                     </excludes>
                 </configuration>
-                <executions>
-                  <execution>
-                    <goals>
-                      <goal>test-jar
-                      </goal>
-                    </goals>
-                  </execution>
-                </executions>
             </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>

diff --git a/scala-package/macros/pom.xml b/scala-package/macros/pom.xml
@@ -90,14 +90,6 @@
             <exclude>META-INF/*.RSA</exclude>
           </excludes>
         </configuration>
-        <executions>
-          <execution>
-            <goals>
-              <goal>test-jar
-              </goal>
-            </goals>
-          </execution>
-        </executions>
       </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>

diff --git a/scala-package/packageTest/Makefile b/scala-package/packageTest/Makefile
@@ -17,7 +17,7 @@
 
 SCALA_VERSION_PROFILE := 2.11
 SCALA_VERSION := 2.11.8
-MXNET_VERSION := 1.3.1-SNAPSHOT
+MXNET_VERSION := "[1.3.0-SNAPSHOT,)"
 
 MXNET_REPO = https://repository.apache.org/content/repositories/snapshots
 
@@ -43,35 +43,45 @@ else
 	endif
 endif
 
-scalapkg:
-	(mvn package -Dmxnet.profile=$(SCALA_PKG_PROFILE) \
-		-Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \
-		-Dmxnet.version=$(MXNET_VERSION) \
-		-Dscala.version=$(SCALA_VERSION))
+PROFILES := -Ptest
+ifeq ($(UNIT), 1)
+  PROFILES := "$(PROFILES),unittest"
+endif
+ifeq ($(INTEGRATION), 1)
+  PROFILES := "$(PROFILES),integrationtest"
+endif
 
-scalaclean:
+ifneq ($(UNIT), 1)
+	ifneq ($(INTEGRATION), 1)
+		PROFILES := "$(PROFILES),unittest,integrationtest"
+	endif
+endif
+
+
+clean:
 	(mvn clean -Dmxnet.profile=$(SCALA_PKG_PROFILE) \
 		-Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \
 		-Dmxnet.version=$(MXNET_VERSION) \
 		-Dscala.version=$(SCALA_VERSION))
 
-scalaintegrationtest:
+testinstall:
 	(mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \
-    -PwithExamples \
+    $(PROFILES) \
 		-Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \
 		-Dmxnet.version=$(MXNET_VERSION) \
 		-Dscala.version=$(SCALA_VERSION))
 
-scalaintegrationtestsnapshot:
+testlocal:
 	(mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \
-    -PfromSnapshots \
+    $(PROFILES),fromLocal \
 		-Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \
-		-Dmxnet.repo=$(MXNET_REPO) \
 		-Dmxnet.version=$(MXNET_VERSION) \
 		-Dscala.version=$(SCALA_VERSION))
 
-scalaintegrationtestwithoutexamples:
+testsnapshot:
 	(mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \
+    $(PROFILES),fromSnapshots \
 		-Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \
+		-Dmxnet.repo=$(MXNET_REPO) \
 		-Dmxnet.version=$(MXNET_VERSION) \
 		-Dscala.version=$(SCALA_VERSION))
diff --git a/scala-package/packageTest/README.md b/scala-package/packageTest/README.md
@@ -1,22 +1,40 @@
 # MXNet Scala Package Test
 
-This is an project created to run the test suite on a fully packaged mxnet jar.
+This is an project created to run the test suite on a fully packaged mxnet jar. The test suite is found locally but mxnet is from the target jarfile.
 
-## Setup
+## General Setup
 
-### Install Package
+To setup the packageTest, you must first build your tests. To build the tests, follow these steps from the mxnet main directory:
 
-To run the test suite, first install the package.  This can be done either by installing directly from a jar with `mvn install:install-file -Dfile=<path-to-file>` or by running `make scalainstall` in the main mxnet folder.  Note that if you use `mvn install:install-file`, you will be unable to run the example tests unless you also install the mxnetexamples jar. You can run all tests except for those examples with `make scalaintegrationtestwithoutexamples`.
+1. Build MXNet and the scala package from source following the directions [here](https://mxnet.incubator.apache.org/install/scala_setup.html#source)
+2. Build the tests by running `make scalatestcompile`.
+3. Follow setup instructions below for your testing goal
 
-### Build
+## Running
 
-Build the mxnet tests by running `make scalapkg` and then `make scalatestcompile` from the main mxnet directory.  This is needed for test discovery.
+There are three different modes of operation for testing based on the location of the jar and where it is coming from:
 
-## Run
+### Test Installed Jars
 
-To run, ensure the versions are correct in the `Makefile`.  Then, just run `make scalaintegrationtest` to execute the test suite
+If you have a jar file, you can install it to your maven cache repository(`~/.m2/repository`) using `mvn install:install-file -Dfile=<path-to-file>`.  This might be useful if you acquire the .jar file from elsewhere. You can also run `make scalainstall` to install from a local build. Then, run `make testinstall` in the package test directory to run the tests.  Note that unless you also install an additional mxnetexamples jar, you can only run the unit tests.
 
-## Clean
+### Test Local Deployment
+
+To test the jars that would be produced by a deployment, you can run `make scaladeploylocal` from the main mxnet directory. This produces a local snapshot located at `scala-package/local-snapshot`. To test this local snapshot, run `make testlocal`.
+
+### Remote Repository Snapshot
+
+This mode is to test a jar located in a remote repository. The default repository is the apache snapshot repisotory located at `https://repository.apache.org/content/repositories/snapshots`. Note that the actual jar in a repisotory should be located at `$repoUrl/org/apache/mxnet/mxnet-full_$scalaVersion-$osMode/$version/*.jar`.
+
+Test the snapshot repo using `make testsnapshot` or a different repo using `make testsnapshot MXNET_REPO=$NEW_REPO_URL`.
+
+### Options
+
+You are able to run unit tests, integration tests, or both using this utility. To run the unit tests, add the flag `UNIT=1` to make (e.g. `make testsnapshot UNIT=1`). Use `INTEGRATION=1` for integration tests. The default behavior is to run both the unit and integration tests. However, the integration tests require that the mxnet examples be installed in addition to the full mxnet package (see test mode instructions above).
+
+An additional option, you can specify the mxnet version with `MXNET_VERSION=1.3.1-SNAPSHOT`.
+
+## Cleaning Up
 
 You can clean temporary files and target artifacts by running `make scalaclean`.
 
@@ -47,4 +65,4 @@ and stacktrace begins with the following,
   java.lang.NoClassDefFoundError: org/apache/mxnetexamples/Util$
 ```
 
-you are missing the mxnetexamples package.  See the "Install Package" section for details.
+you are missing the mxnetexamples package.  See your test mode installation section for details.
diff --git a/scala-package/packageTest/core/pom.xml b/scala-package/packageTest/core/pom.xml
@@ -14,6 +14,15 @@
     <name>MXNet Scala Package Test</name>
     <packaging>pom</packaging>
 
+    <profiles>
+      <profile>
+        <id>unittest</id>
+        <properties>
+          <skipTests>false</skipTests>
+        </properties>
+      </profile>
+    </profiles>
+
     <build>
         <plugins>
             <plugin>

diff --git a/scala-package/packageTest/examples/pom.xml b/scala-package/packageTest/examples/pom.xml
@@ -14,6 +14,15 @@
     <name>MXNet Scala Package Test</name>
     <packaging>pom</packaging>
 
+    <profiles>
+      <profile>
+        <id>integrationtest</id>
+        <properties>
+          <skipTests>false</skipTests>
+        </properties>
+      </profile>
+    </profiles>
+
     <build>
         <plugins>
             <plugin>

diff --git a/scala-package/packageTest/infer/pom.xml b/scala-package/packageTest/infer/pom.xml
@@ -14,6 +14,15 @@
     <name>MXNet Scala Package Test</name>
     <packaging>pom</packaging>
 
+    <profiles>
+      <profile>
+        <id>unittest</id>
+        <properties>
+          <skipTests>false</skipTests>
+        </properties>
+      </profile>
+    </profiles>
+
     <build>
         <plugins>
             <plugin>

diff --git a/scala-package/packageTest/pom.xml b/scala-package/packageTest/pom.xml
@@ -16,7 +16,10 @@
 
     <profiles>
       <profile>
-        <id>withExamples</id>
+        <id>test</id>
+      </profile>
+      <profile>
+        <id>integrationtest</id>
         <modules>
           <module>examples</module>
         </modules>
@@ -34,8 +37,24 @@
           </repository>
         </repositories>
       </profile>
+      <profile>
+        <id>fromLocal</id>
+        <repositories>
+          <repository>
+            <id>local-snapshot</id>
+            <url>file://${basedir}/../local-snapshot</url>
+            <snapshots>
+              <enabled>true</enabled>
+            </snapshots>
+          </repository>
+        </repositories>
+      </profile>
     </profiles>
 
+    <properties>
+      <skipTests>true</skipTests>
+    </properties>
+
     <dependencies>
         <dependency>
             <groupId>org.apache.mxnet</groupId>
@@ -119,7 +138,6 @@
                         <phase>package</phase>
                         <goals>
                             <goal>jar</goal>
-                            <goal>test-jar</goal>
                         </goals>
                         <configuration>
                             <includes>