Java2.0 proto

CMakeLists.txt

@@ -91,6 +91,7 @@ cmake_dependent_option(ENABLE_TESTCOVERAGE "Enable compilation with test coverag
 option(BUILD_EXTENSION_PATH "Path to extension to build" "")
 option(BUILD_CYTHON_MODULES "Build cython modules." OFF)
 option(LOG_FATAL_THROW "Log exceptions but do not abort" ON)
+option(BUILD_JAVA_NATIVE "Skip signal handler registration for Java Binding" OFF)
 cmake_dependent_option(USE_SPLIT_ARCH_DLL "Build a separate DLL for each Cuda arch (Windows only)." ON "MSVC" OFF)
 cmake_dependent_option(USE_CCACHE "Attempt using CCache to wrap the compilation" ON "UNIX" OFF)
 cmake_dependent_option(MXNET_FORCE_SHARED_CRT "Build with dynamic CRT on Windows (/MD)" ON "MXNET_BUILD_SHARED_LIBS" OFF)
@@ -970,6 +971,10 @@ if(USE_CPP_PACKAGE)
   target_compile_definitions(mxnet PUBLIC MXNET_USE_CPP_PACKAGE=1)
 endif()
 
+if(BUILD_JAVA_NATIVE)
+  add_definitions(-DSKIP_SIGNAL_HANDLER_REGISTRATION=1)
+endif()
+
 if(NOT CMAKE_BUILD_TYPE STREQUAL "Distribution")
   # Staticbuild applies linker version script to hide private symbols, breaking unit tests
   add_subdirectory(tests)

ci/docker/runtime_functions.sh

@@ -310,6 +310,23 @@ build_ubuntu_cpu() {
     build_ubuntu_cpu_openblas
 }
 
+build_ubuntu_cpu_and_test_java() {
+    build_ubuntu_cpu_openblas_java
+    java_package_integration_test
+}
+
+java_package_integration_test() {
+    # make sure you are using java 11
+    # build java project
+    cd /work/mxnet/java-package
+    ./gradlew build -x javadoc
+    # generate native library
+    ./gradlew :native:buildLocalLibraryJarDefault
+    ./gradlew :native:mkl-linuxJar
+    # run integration
+    ./gradlew :integration:run
+}
+
 build_ubuntu_cpu_openblas() {
     set -ex
     cd /work/build
@@ -327,6 +344,24 @@ build_ubuntu_cpu_openblas() {
     ninja
 }
 
+build_ubuntu_cpu_openblas_java() {
+    set -ex
+    cd /work/build
+    CXXFLAGS="-Wno-error=strict-overflow" CC=gcc-7 CXX=g++-7 cmake \
+        -DCMAKE_BUILD_TYPE="RelWithDebInfo" \
+        -DENABLE_TESTCOVERAGE=ON \
+        -DUSE_TVM_OP=ON \
+        -DUSE_BLAS=Open \
+        -DUSE_ONEDNN=OFF \
+        -DUSE_CUDA=OFF \
+        -DUSE_DIST_KVSTORE=ON \
+        -DBUILD_CYTHON_MODULES=ON \
+        -DBUILD_EXTENSION_PATH=/work/mxnet/example/extensions/lib_external_ops \
+        -DBUILD_JAVA_NATIVE=ON \
+        -G Ninja /work/mxnet
+    ninja
+}
+
 build_ubuntu_cpu_mkl() {
     set -ex
     cd /work/build

java-package/Develop.md

@@ -0,0 +1,109 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+# Development Tips
+
+## Set up the Project
+### Step 1. Obtain MXNet Library
+The first step is to obtain the mxnet library. We recommend you build it from source. Also, you can download the library 
+from 
+#### Build from source
+Refer to [Build From Source](https://mxnet.apache.org/get_started/build_from_source#building-mxnet)   
+For MacOS users:
+- Prepare  
+```shell
+# Install OS X Developer Tools
+$ xcode-select --install
+
+# Install Homebrew
+$ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+
+# Install dependencies
+$ brew install cmake ninja ccache opencv
+```
+- Clone 3rd party projects
+```shell
+# Clone 3rd dependency for mxnet. It's necessary
+$ git submodule update --init --recursive
+```
+- Build MXNet  
+```shell
+# select and copy cmake configure files for macos
+$ cp config/darwin.cmake config.cmake
+
+# create build directory for prpject
+$ mkdir build; cd build
+
+# cmake
+$ cmake  ..
+$ cmake --build .
+```
+Libraries will be generated under the directory _build/_.
+
+For Linux users:  
+Docker might help you build libraries on different platforms. You can get help from [README for CI](../ci/README.md).  
+For example, you can build mxnet on Ubuntu with  by the following command.  
+```shell
+$ python3 ci/build.py -p ubuntu_cpu
+```
+##### Download Pre-built library
+You can find the mxnet library from installed packages for mxnet, like python module. However, mxnet 2.0 is not released
+yet, that's why we recommend you build it from source.  
+```shell
+# download python module for mxnet (have to mention that mxnet 2.0 hasn't been released by now)
+$ pip3 install  mxnet==1.7.0.post2
+# find the location of the installed module
+$ python
+Python 3.6.8 |Anaconda, Inc.| (default, Dec 29 2018, 19:04:46)
+>>> import mxnet
+>>> mxnet
+<module 'mxnet' from '/Users/xxx/anaconda3/lib/python3.6/site-packages/mxnet/__init__.py'>
+>>> quit()
+# you can locate the module under /Users/xxx/anaconda3/lib/python3.6/site-packages/mxnet/
+$ ls /Users/xxx/anaconda3/lib/python3.6/site-packages/mxnet/ | grep libmxnet
+libmxnet.dylib
+```
+The compiled library is the file with the name of _libmxnet.*_. For MacOS, you will receive the file with suffix 
+_.dylib_; For Linux, the lib file have the suffix ".so"; For Windows, the suffix is "." 
+
+### Step 2. Build MXNet Native Lib for Java
+The project uses gradle to manage dependencies. You can build the project using gradle. We have to encapsulate the mxnet
+library into a jar file so that we can load it into JVM.
+```shell
+$ cd java-package
+# Build the project
+$ ./gradlew build 
+# Create gradle tasks to package mxnet library into jar
+# The task name is in this form {$favor}-{$platform}Jar
+# MacOS -> mkl-osxJar
+# Linux -> mkl-linuxJar
+# Windows -> mkl-winJar
+$ ./gradlew :native:buildLocalLibraryJarDefault
+# Build native lib for macos
+$ ./gradlew mkl-osxJar
+# Check the lib for osx
+$ ls native/build/libs | grep osx
+native-2.0.0-SNAPSHOT-osx-x86_64.jar
+```
+The jar file _native-2.0.0-SNAPSHOT-osx-x86_64.jar_ is the output lib file. 
+
+### Step 3. Run Integration Test
+When we execute the task for integration test, the built mxnet native lib will be added into classpath automatically. 
+```shell
+$ ./gradlew :integration:run
+
+```

java-package/README.md

@@ -0,0 +1,58 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+# Java Package for MXNet 2.0
+
+## Requirements
+
+## Install
+
+## Scripts
+- customize mxnet library path  
+```bash
+export MXNET_LIBRARY_PATH=//anaconda3/lib/python3.8/site-packages/mxnet/
+```
+
+
+## Tests  
+Test case for a rough inference run with MXNet model  
+```bash
+./gradlew :integration:run  
+```
+
+## Example
+
+```java
+try (MxResource base = BaseMxResource.getSystemMxResource())
+        {
+        Model model = Model.loadModel(Item.MLP);
+//            Model model = Model.loadModel("test", Paths.get("/Users/cspchen/mxnet.java_package/cache/repo/test-models/mlp.tar.gz/mlp/"));
+        Predictor<NDList, NDList> predictor = model.newPredictor();
+        NDArray input = NDArray.create(base, new Shape(1, 28, 28)).ones();
+        NDList inputs = new NDList();
+        inputs.add(input);
+        NDList result = predictor.predict(inputs);
+        NDArray expected =  NDArray.create(
+        base,
+        new float[]{4.93476f, -0.76084447f, 0.37713608f, 0.6605506f, -1.3485785f, -0.8736369f
+        , 0.018061712f, -1.3274033f, 1.0609543f, 0.24042489f}, new Shape(1, 10));
+        Assertions.assertAlmostEquals(result.get(0), expected);
+
+        } catch (IOException e) {
+        logger.error(e.getMessage(), e);
+        }
+```

java-package/build.gradle

@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins {
+    id "com.github.spotbugs" version "4.2.0" apply true
+}
+
+defaultTasks 'build'
+
+allprojects {
+    group 'org.apache.mxnet'
+    boolean isRelease = project.hasProperty("release") || project.hasProperty("staging")
+    version = "${java_package_version}" + (isRelease ? "" : "-SNAPSHOT")
+
+    repositories {
+//        maven {
+//            url "https://mlrepo.djl.ai/maven/"
+//        }
+        mavenCentral()
+        maven {
+            url 'https://oss.sonatype.org/content/repositories/snapshots/'
+        }
+    }
+
+    apply plugin: 'idea'
+    idea {
+        module {
+            outputDir = file('build/classes/java/main')
+            testOutputDir = file('build/classes/java/test')
+            // inheritOutputDirs = true
+        }
+    }
+}
+
+def javaProjects() {
+    return subprojects.findAll { new File(it.projectDir, "src/main").exists() }
+}
+
+configure(javaProjects()) {
+    apply plugin: 'java-library'
+    sourceCompatibility = 1.8
+    targetCompatibility = 1.8
+    compileJava.options.encoding = "UTF-8"
+    compileTestJava.options.encoding = "UTF-8"
+    if (JavaVersion.current() != JavaVersion.VERSION_1_8) {
+        compileJava.options.compilerArgs.addAll(["--release", "8"])
+    }
+
+    apply plugin: 'eclipse'
+
+    eclipse {
+        jdt.file.withProperties { props ->
+            props.setProperty "org.eclipse.jdt.core.circularClasspath", "warning"
+        }
+        classpath {
+            sourceSets.test.java {
+                srcDirs = ["src/test/java"]
+                exclude "**/package-info.java"
+            }
+        }
+    }
+
+    apply from: file("${rootProject.projectDir}/tools/gradle/java-formatter.gradle")
+    apply from: file("${rootProject.projectDir}/tools/gradle/check.gradle")
+
+    test {
+        // tensorflow mobilenet and resnet require more cpu memory
+        maxHeapSize = "4096m"
+        doFirst {
+            if (JavaVersion.current() != JavaVersion.VERSION_1_8) {
+                jvmArgs = [
+                        '--add-opens', "java.base/jdk.internal.loader=ALL-UNNAMED"
+                ]
+            }
+        }
+
+        useTestNG() {
+//             suiteXmlFiles << new File(rootDir, "testng.xml") //This is how to add custom testng.xml
+        }
+
+        testLogging {
+            showStandardStreams = true
+            events "passed", "skipped", "failed", "standardOut", "standardError"
+        }
+
+        doFirst {
+            systemProperties System.getProperties()
+            systemProperties.remove("user.dir")
+            systemProperty "org.apache.mxnet.logging.level", "debug"
+            systemProperty "org.slf4j.simpleLogger.defaultLogLevel", "debug"
+            systemProperty "org.slf4j.simpleLogger.log.org.mortbay.log", "warn"
+            systemProperty "disableProgressBar", "true"
+            systemProperty "nightly", System.getProperty("nightly", "false")
+//            systemProperty "java.library.path", "/Users/cspchen/Work/incubator-mxnet/build"
+            if (gradle.startParameter.offline) {
+                systemProperty "offline", "true"
+            }
+        }
+    }
+
+    compileJava {
+        options.compilerArgs << "-proc:none" << "-Xlint:all,-options,-static" << "-Werror"
+    }
+
+    compileTestJava {
+        options.compilerArgs << "-proc:none" << "-Xlint:all,-options,-static" << "-Werror"
+    }
+
+    jar {
+        manifest {
+            attributes("Automatic-Module-Name": "org.apach.mxnet.${project.name.replace('-', '_')}")
+        }
+    }
+}
+
+apply from: file("${rootProject.projectDir}/tools/gradle/jacoco.gradle")
+apply from: file("${rootProject.projectDir}/tools/gradle/stats.gradle")