Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Java2.0 proto #20461

Merged
merged 63 commits into from
Oct 5, 2021
Merged
Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
63 commits
Select commit Hold shift + click to select a range
3aa0a24
ADD: first commit
Jun 28, 2021
735320d
ADD: load local libraries
Jun 29, 2021
67e2988
UPDATE: use header files of MXNet 2.0
Jun 30, 2021
563bdcd
ADD: load binaries from environment variable, java properties or jar …
Jun 30, 2021
5829a44
ADD: add symbol loading and closing
Jul 4, 2021
5250246
ADD: [WIP] Component MxNDArray
Jul 14, 2021
8a666f7
ADD: [WIP] Component MxNDArray
Jul 14, 2021
aba88ab
ADD: Component MxNDArray. Pass static compilation check
Jul 15, 2021
b1037ed
ADD: Component CachedOp
Jul 15, 2021
edbf730
REMOVE: module api which is no use
Jul 15, 2021
df2d0c7
FIX: dependency missing
Jul 15, 2021
6c4e7d7
ADD: [WIP] add test cases for NdArray and CachedOp
Jul 16, 2021
b9b9021
ADD: [WIP] add test cases for NdArray and CachedOp
Jul 18, 2021
a458c54
Merge branch 'master' into jave2.0-proto
Jul 18, 2021
64b443d
ADD: implement of the forward function for MxSymbolblock
Jul 19, 2021
92e4814
ADD: implement of the forward function for MxSymbolblock
Jul 19, 2021
f99f813
ADD: Sample model downloader for MLP
Jul 20, 2021
8020d69
ADD: doc
Jul 20, 2021
dbb6d83
ADD: Front-end module for inference, class MxModel, Predictor and so on.
Jul 21, 2021
36a5f4e
FIX: Mxnet crash when process exits.
Jul 22, 2021
d097675
FIX: remove and initialize 3rdparty directory
Jul 22, 2021
8756743
FIX: revert version of submodules: dlpack, dmlc-core, googletest, ps-…
Jul 22, 2021
fd3cef6
Merge branch 'java2.0-proto' into jave2.0-proto
Jul 22, 2021
a687482
Revert "FIX: remove and initialize 3rdparty directory"
Jul 22, 2021
22dc991
FIX: redownload files in 3rdparty
Jul 22, 2021
83907da
FIX: reset --hard the version of a few submodules
Jul 22, 2021
a3beddd
FIX: reset --hard the version of a few submodules
Jul 22, 2021
d1bc141
FIX: reset --hard the version of a few submodules
Jul 22, 2021
963568c
PERF: [WIP] optimize code structure and memory management and
Jul 22, 2021
aeec7f1
ADD: add copyright; remove Mx prefix for some classes
Jul 27, 2021
a7da58f
ADD: add copyright
Jul 27, 2021
62d560f
FIX: group name, path to find header files
Jul 27, 2021
216fb8f
UPDATE: README.md
Jul 27, 2021
f47045f
ADD: copyright
Jul 27, 2021
8776eab
ADD: copyright
Jul 27, 2021
ae90d8d
ADD: package-info
Jul 28, 2021
77ea37d
FIX: ci config file
Jul 30, 2021
85f0c64
UPDATE: remove ParameterStore and some scripts
Jul 30, 2021
be4afd0
UPDATE: ci for java-package
Aug 2, 2021
240841c
FIX: loading on Linux platform
Aug 11, 2021
1ea18ed
# This is a combination of 18 commits.
Aug 17, 2021
24bfc34
# This is a combination of 27 commits.
Aug 17, 2021
87130e2
MERGE: resolve conflicts
Aug 17, 2021
1a3a644
MERGE: resolve conflicts
Aug 17, 2021
fa5bbbd
# This is a combination of 35 commits.
Aug 17, 2021
bfc1f5d
parent 1ea18edce197b402cbfbaaaa54a94347501d92ab
Aug 17, 2021
720a14d
FIX: solve problems before merge
Aug 31, 2021
1d568ae
UPDATE: remove useless files
channel960608 Sep 4, 2021
277a51d
FIX: licence to apache
channel960608 Sep 6, 2021
e05a8ef
FIX: sanity check
channel960608 Sep 6, 2021
bd8c3df
FIX: sanity check
channel960608 Sep 6, 2021
4f31840
FIX: sanity check
channel960608 Sep 6, 2021
28335bc
FIX: remove unused files
channel960608 Sep 11, 2021
d6c2c2e
FIX: remove unused files
channel960608 Sep 12, 2021
8b2984b
DOC: add document
channel960608 Sep 13, 2021
d30f077
FIX: doesn't work on osx
channel960608 Sep 13, 2021
5817850
FIX: clang static check
channel960608 Sep 13, 2021
1b1f190
FIX: sanity
channel960608 Sep 13, 2021
6955539
FIX: skip signal handler registration when building java package
channel960608 Sep 25, 2021
4915e06
Merge pull request #2 from channel960608/feature/java2.0-proto-09252021
channel960608 Sep 25, 2021
db3fe64
FIX: remove DataType String
channel960608 Sep 29, 2021
6a16984
Merge branch 'java2.0-doc' into jave2.0-proto
channel960608 Sep 29, 2021
9b5899b
FIX: add license
channel960608 Sep 29, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion 3rdparty/dmlc-core
Submodule dmlc-core updated 120 files
2 changes: 1 addition & 1 deletion 3rdparty/googletest
Submodule googletest updated 355 files
36 changes: 36 additions & 0 deletions java-package/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# Java Package for MXNet 2.0

## Requirements

## Install

## Scripts
- customize mxnet library path
```bash
export MXNET_LIBRARY_PATH=//anaconda3/lib/python3.8/site-packages/mxnet/
```


## Tests
Test case for a rough inference run with MXNet model
```bash
./gradlew :integration:run
```

## Example

```java
try (MxResource base = BaseMxResource.getSystemMxResource())
{
MxModel mxModel = MxModel.loadModel(Item.MLP);
// MxModel.loadModel(Item.MLP.getName(), Paths.get(Item.MLP.getUrl());
Predictor<MxNDList, MxNDList> predictor = mxModel.newPredictor();
MxNDArray input = MxNDArray.create(base, new Shape(1, 28, 28)).ones();
MxNDList inputs = new MxNDList();
inputs.add(input);
MxNDList result = predictor.predict(inputs);
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
}
```
113 changes: 113 additions & 0 deletions java-package/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
plugins {
id "com.github.spotbugs" version "4.2.0" apply true
}

defaultTasks 'build'

allprojects {
group 'incubator-mxnet.java-package'
channel960608 marked this conversation as resolved.
Show resolved Hide resolved
boolean isRelease = project.hasProperty("release") || project.hasProperty("staging")
version = "${java_package_version}" + (isRelease ? "" : "-SNAPSHOT")

repositories {
// maven {
// url "https://mlrepo.djl.ai/maven/"
// }
mavenCentral()
maven {
url 'https://oss.sonatype.org/content/repositories/snapshots/'
}
}

apply plugin: 'idea'
idea {
module {
outputDir = file('build/classes/java/main')
testOutputDir = file('build/classes/java/test')
// inheritOutputDirs = true
}
}
}

def javaProjects() {
return subprojects.findAll { new File(it.projectDir, "src/main").exists() }
}

configure(javaProjects()) {
apply plugin: 'java-library'
sourceCompatibility = 1.8
targetCompatibility = 1.8
compileJava.options.encoding = "UTF-8"
compileTestJava.options.encoding = "UTF-8"
if (JavaVersion.current() != JavaVersion.VERSION_1_8) {
compileJava.options.compilerArgs.addAll(["--release", "8"])
}

apply plugin: 'eclipse'

eclipse {
jdt.file.withProperties { props ->
props.setProperty "org.eclipse.jdt.core.circularClasspath", "warning"
}
classpath {
sourceSets.test.java {
srcDirs = ["src/test/java"]
exclude "**/package-info.java"
}
}
}

apply from: file("${rootProject.projectDir}/tools/gradle/java-formatter.gradle")
apply from: file("${rootProject.projectDir}/tools/gradle/check.gradle")

test {
// tensorflow mobilenet and resnet require more cpu memory
maxHeapSize = "4096m"
doFirst {
if (JavaVersion.current() != JavaVersion.VERSION_1_8) {
jvmArgs = [
'--add-opens', "java.base/jdk.internal.loader=ALL-UNNAMED"
]
}
}

useTestNG() {
// suiteXmlFiles << new File(rootDir, "testng.xml") //This is how to add custom testng.xml
}

testLogging {
showStandardStreams = true
events "passed", "skipped", "failed", "standardOut", "standardError"
}

doFirst {
systemProperties System.getProperties()
systemProperties.remove("user.dir")
systemProperty "incubator-mxnet.java-package.logging.level", "debug"
systemProperty "org.slf4j.simpleLogger.defaultLogLevel", "debug"
systemProperty "org.slf4j.simpleLogger.log.org.mortbay.log", "warn"
systemProperty "disableProgressBar", "true"
systemProperty "nightly", System.getProperty("nightly", "false")
if (gradle.startParameter.offline) {
systemProperty "offline", "true"
}
}
}

compileJava {
options.compilerArgs << "-proc:none" << "-Xlint:all,-options,-static" << "-Werror"
}

compileTestJava {
options.compilerArgs << "-proc:none" << "-Xlint:all,-options,-static" << "-Werror"
}

jar {
manifest {
attributes("Automatic-Module-Name": "incubator-mxnet.java-package.${project.name.replace('-', '_')}")
}
}
}

apply from: file("${rootProject.projectDir}/tools/gradle/jacoco.gradle")
apply from: file("${rootProject.projectDir}/tools/gradle/stats.gradle")
19 changes: 19 additions & 0 deletions java-package/example/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
plugins {
id 'java'
}

group 'incubator-mxnet.java-package'
version '0.0.1-SNAPSHOT'

repositories {
mavenCentral()
}

dependencies {
testImplementation 'org.junit.jupiter:junit-jupiter-api:5.7.0'
testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.7.0'
}

test {
useJUnitPlatform()
}
27 changes: 27 additions & 0 deletions java-package/gradle.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
org.gradle.daemon=true
org.gradle.jvmargs=-Xmx2048M

systemProp.org.gradle.internal.http.socketTimeout=120000
systemProp.org.gradle.internal.http.connectionTimeout=60000

# FIXME: Workaround gradle publish issue: https://github.com/gradle/gradle/issues/11308
systemProp.org.gradle.internal.publish.checksums.insecure=true

java_package_version=0.0.1
mxnet_version=2.0.0
api_version=0.0.1
jnarator_version=0.0.1

antlr_version=4.7.2
commons_cli_version=1.4
commons_compress_version=1.20
commons_csv_version=1.8
gson_version=2.8.6
jna_version=5.3.0
netty_version=4.1.51.Final
slf4j_version=1.7.30
log4j_slf4j_version=2.13.3
testng_version=7.1.0
powermock_version=2.0.7

MXNET_LIBRARY_PATH = /Users/cspchen/Work/refer/incubator-mxnet/build
47 changes: 47 additions & 0 deletions java-package/integration/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
plugins {
id 'application'
id 'java'
}

group 'incubator-mxnet.java-package'
version '0.0.1-SNAPSHOT'

repositories {
mavenCentral()
}

application {
mainClassName = System.getProperty("main", "org.apache.mxnet.integration.IntegrationTest")
}

dependencies {
api "commons-cli:commons-cli:${commons_cli_version}"
api "org.apache.logging.log4j:log4j-slf4j-impl:${log4j_slf4j_version}"

api project(":mxnet-engine")
implementation 'org.testng:testng:7.1.0'
// testImplementation(":mxnet-engine")
testImplementation("org.testng:testng:${testng_version}") {
exclude group: "junit", module: "junit"
}
testImplementation "org.slf4j:slf4j-simple:${slf4j_version}"
}

run {
environment("TF_CPP_MIN_LOG_LEVEL", "1") // turn off TensorFlow print out
channel960608 marked this conversation as resolved.
Show resolved Hide resolved
systemProperties System.getProperties()
systemProperties.remove("user.dir")
systemProperty("file.encoding", "UTF-8")
jvmArgs "-Xverify:none"
args("-p=org.apache.mxnet.integration.tests.", "-m=modelLoadAndPredictTest")

}

test {

useTestNG()
filter {
includeTestsMatching "org.apache.mxnet.integration.tests.engine.*"
}

}
Loading