From 6ff919ecac227642498730b5155af3676e53e6e2 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Fri, 15 Feb 2019 18:05:19 +0800 Subject: [PATCH 001/115] Refine JNI bazel script to make it suitable for more systems --- BUILD.bazel | 23 ++++++++++++++--------- bazel/BUILD.plasma | 30 +++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index b2cda35ddf38..9d608d756008 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -353,25 +353,30 @@ pyx_library( cc_binary( name = "raylet_library_java.so", srcs = [ - "src/ray/id.h", - "src/ray/raylet/lib/java/org_ray_runtime_raylet_RayletClientImpl.cc", "src/ray/raylet/lib/java/org_ray_runtime_raylet_RayletClientImpl.h", + "src/ray/raylet/lib/java/org_ray_runtime_raylet_RayletClientImpl.cc", + "src/ray/id.h", "src/ray/raylet/raylet_client.h", "src/ray/util/logging.h", - "@bazel_tools//tools/jdk:current_java_runtime", - ], + "@bazel_tools//tools/jdk:jni_header", + ] + select({ + "@bazel_tools//src/conditions:windows": ["@bazel_tools//tools/jdk:jni_md_header-windows"] , + "@bazel_tools//src/conditions:darwin": ["@bazel_tools//tools/jdk:jni_md_header-darwin"], + "//conditions:default": ["@bazel_tools//tools/jdk:jni_md_header-linux"], + }), includes = [ "src", - "external/local_jdk/include", + "external/bazel_tools/tools/jdk/include", ] + select({ - "@bazel_tools//src/conditions:darwin": ["external/local_jdk/include/darwin"], - "//conditions:default": ["external/local_jdk/include/linux"], + "@bazel_tools//src/conditions:windows": ["external/bazel_tools/tools/jdk/include/windows" ], + "@bazel_tools//src/conditions:darwin": ["external/bazel_tools/tools/jdk/include/darwin"], + "//conditions:default": ["external/bazel_tools/tools/jdk/include/linux"], }), linkshared = 1, - linkstatic = 1, + linkstatic = 0, deps = [ - "//:raylet_lib", "@plasma//:plasma_client", + "//:raylet_lib" ], ) diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index edb92a3ac649..d2bfe1841f5a 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -69,20 +69,36 @@ cc_binary( srcs = [ "cpp/src/plasma/lib/java/org_apache_arrow_plasma_PlasmaClientJNI.cc", "cpp/src/plasma/lib/java/org_apache_arrow_plasma_PlasmaClientJNI.h", - "@bazel_tools//tools/jdk:current_java_runtime", + ":jni.h", + ":jni_md.h", ], includes = [ "cpp/src", - "../../external/local_jdk/include", - ] + select({ - "@bazel_tools//src/conditions:darwin": ["../../external/local_jdk/include/darwin"], - "//conditions:default": ["../../external/local_jdk/include/linux"], - }), + ".", + ], linkshared = 1, - linkstatic = 1, + linkstatic = 0, deps = [":plasma_client"], ) +genrule( + name = "copy_jni_h", + srcs = ["@bazel_tools//tools/jdk:jni_header"], + outs = ["jni.h"], + cmd = "cp -f $< $@", +) + +genrule( + name = "copy_jni_md_h", + srcs = select({ + "@bazel_tools//src/conditions:windows": ["@bazel_tools//tools/jdk:jni_md_header-windows"] , + "@bazel_tools//src/conditions:darwin": ["@bazel_tools//tools/jdk:jni_md_header-darwin"], + "//conditions:default": ["@bazel_tools//tools/jdk:jni_md_header-linux"], + }), + outs = ["jni_md.h"], + cmd = "cp -f $< $@", +) + genrule( name = "plasma-jni-darwin-compat", srcs = [":plasma_client_java.so"], From ce314ffca4b51754f69207d27e77fe134b6899fb Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Wed, 6 Mar 2019 02:47:14 +0800 Subject: [PATCH 002/115] java bazel config --- BUILD.bazel | 45 +- WORKSPACE | 4 + bazel/BUILD.plasma | 31 +- bazel/ray.bzl | 12 + bazel/ray_java.bzl | 1577 ++++++++++++++++++++++++++++++++++++++++++++ build.sh | 23 +- java/BUILD.bazel | 211 ++++++ java/build.sh | 6 +- java/test.sh | 53 +- java/testng.xml | 11 + 10 files changed, 1931 insertions(+), 42 deletions(-) create mode 100644 bazel/ray_java.bzl create mode 100644 java/BUILD.bazel create mode 100644 java/testng.xml diff --git a/BUILD.bazel b/BUILD.bazel index 61651b2f0a18..52887f47b052 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -372,7 +372,7 @@ pyx_library( ) cc_binary( - name = "raylet_library_java.so", + name = "libraylet_library_java.so", srcs = [ "src/ray/raylet/lib/java/org_ray_runtime_raylet_RayletClientImpl.h", "src/ray/raylet/lib/java/org_ray_runtime_raylet_RayletClientImpl.cc", @@ -403,8 +403,8 @@ cc_binary( genrule( name = "raylet-jni-darwin-compat", - srcs = [":raylet_library_java.so"], - outs = ["raylet_library_java.dylib"], + srcs = [":libraylet_library_java.so"], + outs = ["libraylet_library_java.dylib"], cmd = "cp $< $@", output_to_bindir = 1, ) @@ -412,16 +412,22 @@ genrule( filegroup( name = "raylet_library_java", srcs = select({ - "@bazel_tools//src/conditions:darwin": [":raylet_library_java.dylib"], - "//conditions:default": [":raylet_library_java.so"], + "@bazel_tools//src/conditions:darwin": [":libraylet_library_java.dylib"], + "//conditions:default": [":libraylet_library_java.so"], }), visibility = ["//visibility:public"], ) +filegroup( + name = "gcs_fbs_file", + srcs = ["src/ray/gcs/format/gcs.fbs"], + visibility = ["//visibility:public"], +) + flatbuffer_py_library( name = "python_gcs_fbs", srcs = [ - "src/ray/gcs/format/gcs.fbs", + ":gcs_fbs_file", ], outs = [ "ActorCheckpointIdData.py", @@ -579,3 +585,30 @@ genrule( mv python $(location ray_pkg) """, ) + +genrule( + name = "ray_java_pkg", + srcs = [ + "//:redis-server", + "//:ray_redis_module.so", + "//:raylet", + "//:raylet_library_java", + "@plasma//:plasma_store_server", + "@plasma//:plasma_client_java", + ], + outs = ["ray_java_pkg"], + cmd = """ + set -x && + mkdir -p java/ray/src/plasma/ && + cp $(location @plasma//:plasma_client_java) java/ray/src/plasma/ && + cp $(location @plasma//:plasma_store_server) java/ray/src/plasma/ && + mkdir -p java/ray/src/ray/raylet/ && + cp $(location //:raylet) java/ray/src/ray/raylet/ && + cp $(location //:raylet_library_java) java/ray/src/ray/raylet/ && + mkdir -p java/ray/src/ray/gcs/redis_module/ && + cp $(location //:ray_redis_module.so) java/ray/src/ray/gcs/redis_module/libray_redis_module.so && + mkdir -p java/ray/src/ray/thirdparty/redis/src/ && + cp $(location //:redis-server) java/ray/src/ray/thirdparty/redis/src/ && + mv java $(location ray_java_pkg) + """, +) diff --git a/WORKSPACE b/WORKSPACE index 00f77852e2c1..c517dcee4d43 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,4 +1,8 @@ +workspace(name = "com_github_riselab_ray") + load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository") +load("//bazel:ray_java.bzl", "generated_maven_jars") +generated_maven_jars() git_repository( name = "com_github_nelhage_rules_boost", diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index 33353caaf6f7..16960674726b 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -1,4 +1,25 @@ load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library") +load("@com_github_riselab_ray//bazel:ray_java.bzl", "generated_java_libraries") + +generated_java_libraries() + +java_library( + name = "org_apache_arrow_arrow_plasma", + srcs = glob(["java/plasma/src/main/java/**/*.java"]), + visibility = ["//visibility:public"], + data = [":plasma_client_java"], + deps = [ + "@org_slf4j_slf4j_api//jar", + ], +) + +java_binary( + name = "org_apache_arrow_arrow_plasma_test", + srcs = ["java/plasma/src/test/java/org/apache/arrow/plasma/PlasmaClientTest.java"], + main_class = "org.apache.arrow.plasma.PlasmaClientTest", + visibility = ["//visibility:public"], + deps = [":org_apache_arrow_arrow_plasma", "@junit_junit//jar",], +) cc_library( name = "arrow", @@ -65,7 +86,7 @@ cc_library( ) cc_binary( - name = "plasma_client_java.so", + name = "libplasma_java.so", srcs = [ "cpp/src/plasma/lib/java/org_apache_arrow_plasma_PlasmaClientJNI.cc", "cpp/src/plasma/lib/java/org_apache_arrow_plasma_PlasmaClientJNI.h", @@ -101,8 +122,8 @@ genrule( genrule( name = "plasma-jni-darwin-compat", - srcs = [":plasma_client_java.so"], - outs = ["plasma_client_java.dylib"], + srcs = [":libplasma_java.so"], + outs = ["libplasma_java.dylib"], cmd = "cp $< $@", output_to_bindir = 1, ) @@ -110,8 +131,8 @@ genrule( filegroup( name = "plasma_client_java", srcs = select({ - "@bazel_tools//src/conditions:darwin": [":plasma_client_java.dylib"], - "//conditions:default": [":plasma_client_java.so"], + "@bazel_tools//src/conditions:darwin": [":libplasma_java.dylib"], + "//conditions:default": [":libplasma_java.so"], }), visibility = ["//visibility:public"], ) diff --git a/bazel/ray.bzl b/bazel/ray.bzl index 4a1e4f1063e4..2f68b25127b6 100644 --- a/bazel/ray.bzl +++ b/bazel/ray.bzl @@ -10,3 +10,15 @@ def flatbuffer_py_library(name, srcs, outs, out_prefix, includes = [], include_p include_paths = include_paths, includes = includes, ) + +def flatbuffer_java_library(name, srcs, outs, out_prefix, includes = [], include_paths = []): + flatbuffer_library_public( + name = name, + srcs = srcs, + outs = outs, + language_flag = "-j", + out_prefix = out_prefix, + include_paths = include_paths, + includes = includes, + ) + diff --git a/bazel/ray_java.bzl b/bazel/ray_java.bzl new file mode 100644 index 000000000000..c66feda3f227 --- /dev/null +++ b/bazel/ray_java.bzl @@ -0,0 +1,1577 @@ +# The following dependencies were calculated from: +# +# generate_workspace --maven_project=/home/admin/ruifang.crf/code/X/arrow/java --repositories=http://mvn.dev.alipay.net:8080/artifactory/repo + + +def generated_maven_jars(): + # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT + # pom.xml got requested version + # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 wanted version 1.1.2 + # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 wanted version 1.1.2 + # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "ch_qos_logback_logback_classic", + artifact = "ch.qos.logback:logback-classic:1.2.3", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "7c4f3c474fb2c041d8028740440937705ebb473a", + ) + + + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "com_google_code_findbugs_jsr305", + artifact = "com.google.code.findbugs:jsr305:3.0.2", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "25ea2e8b0c338a877313bd4672d3fe056ea78f0d", + ) + + + # io.grpc:grpc-protobuf:jar:1.14.0 + native.maven_jar( + name = "io_grpc_grpc_protobuf_lite", + artifact = "io.grpc:grpc-protobuf-lite:1.14.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "2ac8c28ca927f954eaa228a931d9c163cf3d860f", + ) + + + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT wanted version 2.10 + native.maven_jar( + name = "joda_time_joda_time", + artifact = "joda-time:joda-time:2.9.9", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "f7b520c458572890807d143670c9b24f4de90897", + ) + + + # org.mockito:mockito-core:jar:2.7.22 + native.maven_jar( + name = "net_bytebuddy_byte_buddy_agent", + artifact = "net.bytebuddy:byte-buddy-agent:1.6.11", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "0200d9c012befccd211ff91082a151257b1dc084", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 + native.maven_jar( + name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_converter_classic", + artifact = "de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:0.9.44", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "6f08f9bfd65869a51edc922432f46024d94f3ccd", + ) + + + # io.netty:netty-handler-proxy:jar:4.1.22.Final got requested version + # io.netty:netty-handler:jar:4.1.22.Final got requested version + # io.netty:netty-codec:jar:4.1.22.Final + native.maven_jar( + name = "io_netty_netty_transport", + artifact = "io.netty:netty-transport:4.1.22.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "3bd455cd9e5e5fb2e08fd9cd0acfa54c079ca989", + ) + + + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "com_carrotsearch_hppc", + artifact = "com.carrotsearch:hppc:0.7.2", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "710398361f2ae8fd594a133e3619045c16b24137", + ) + + + # io.netty:netty-codec-http2:jar:4.1.22.Final + native.maven_jar( + name = "io_netty_netty_handler", + artifact = "io.netty:netty-handler:4.1.22.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "a3a16b17d5a5ed6f784b0daba95e28d940356109", + ) + + + # io.grpc:grpc-netty:jar:1.14.0 + native.maven_jar( + name = "io_netty_netty_handler_proxy", + artifact = "io.netty:netty-handler-proxy:4.1.27.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "1a822ce7760bc6eb4937b7e448c9e081fedcc807", + ) + + + # org.apache.arrow:arrow-java-root:pom:0.12.0HOT + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "org_mockito_mockito_core", + artifact = "org.mockito:mockito-core:2.7.22", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "fcf63bc8010ca77991e3cadd8d33ad1a40603404", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-core:jar:0.9.44 got requested version + # de.huxhorn.lilith:de.huxhorn.lilith.data.logging.protobuf:jar:0.9.44 got requested version + # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 + # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 got requested version + native.maven_jar( + name = "de_huxhorn_sulky_de_huxhorn_sulky_codec", + artifact = "de.huxhorn.sulky:de.huxhorn.sulky.codec:0.9.17", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "4a2a917f1cc4b75d71aadeef4f4c178f229febc2", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.data.logging:jar:0.9.44 + native.maven_jar( + name = "de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", + artifact = "de.huxhorn.lilith:de.huxhorn.lilith.data.eventsource:0.9.44", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "618f853d75d59855c955b3a1bf0f447476f8027f", + ) + + + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "com_google_flatbuffers_flatbuffers_java", + artifact = "com.google.flatbuffers:flatbuffers-java:1.9.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "460bc8f6411768659c1ffb529592e251a808b9f2", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version + # de.huxhorn.lilith:de.huxhorn.lilith.data.logging.protobuf:jar:0.9.44 wanted version 2.5.0 + # io.grpc:grpc-protobuf:jar:1.14.0 + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT wanted version 2.5.0 + native.maven_jar( + name = "com_google_protobuf_protobuf_java", + artifact = "com.google.protobuf:protobuf-java:3.5.1", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "8c3492f7662fa1cbf8ca76a0f5eb1146f7725acd", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "io_netty_netty_tcnative_boringssl_static", + artifact = "io.netty:netty-tcnative-boringssl-static:2.0.12.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "b884be1450a7fd0854b98743836b8ccb0dfd75a4", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "com_h2database_h2", + artifact = "com.h2database:h2:1.4.196", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "dd0034398d593aa3588c6773faac429bbd9aea0e", + ) + + + # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "org_slf4j_jul_to_slf4j", + artifact = "org.slf4j:jul-to-slf4j:1.7.25", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "0af5364cd6679bfffb114f0dec8a157aaa283b76", + ) + + + native.maven_jar( + name = "net_java_dev_jna_jna", + artifact = "net.java.dev.jna:jna:4.1.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "1c12d070e602efd8021891cdd7fd18bc129372d4", + ) + + + native.maven_jar( + name = "org_mockito_mockito_all", + artifact = "org.mockito:mockito-all:1.10.19", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "539df70269cc254a58cccc5d8e43286b4a73bf30", + ) + + + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "commons_codec_commons_codec", + artifact = "commons-codec:commons-codec:1.10", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "4b95f4897fa13f2cd904aee711aeafc0c5295cd8", + ) + + + native.maven_jar( + name = "com_beust_jcommander", + artifact = "com.beust:jcommander:1.72", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "6375e521c1e11d6563d4f25a07ce124ccf8cd171", + ) + + + # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "org_slf4j_log4j_over_slf4j", + artifact = "org.slf4j:log4j-over-slf4j:1.7.25", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "a87bb47468f47ee7aabbd54f93e133d4215769c3", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "commons_cli_commons_cli", + artifact = "commons-cli:commons-cli:1.4", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "c51c00206bb913cd8612b24abd9fa98ae89719b1", + ) + + + # io.grpc:grpc-core:jar:1.14.0 + native.maven_jar( + name = "com_google_errorprone_error_prone_annotations", + artifact = "com.google.errorprone:error_prone_annotations:2.1.2", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "6dcc08f90f678ac33e5ef78c3c752b6f59e63e0c", + ) + + + # io.netty:netty-transport:jar:4.1.22.Final + native.maven_jar( + name = "io_netty_netty_resolver", + artifact = "io.netty:netty-resolver:4.1.22.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "b5484d17a97cb57b07d2a1ac092c249e47234c17", + ) + + + # io.grpc:grpc-core:jar:1.14.0 + native.maven_jar( + name = "io_opencensus_opencensus_contrib_grpc_metrics", + artifact = "io.opencensus:opencensus-contrib-grpc-metrics:0.12.3", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "a4c7ff238a91b901c8b459889b6d0d7a9d889b4d", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "io_grpc_grpc_stub", + artifact = "io.grpc:grpc-stub:1.14.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "74bfe83c0dc69bf903fff8df3568cbeb8b387d35", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "io_grpc_grpc_protobuf", + artifact = "io.grpc:grpc-protobuf:1.14.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "d429fdc2e0d288b34ea7588bb762eb458f385bd5", + ) + + + # io.netty:netty-handler-proxy:jar:4.1.22.Final + native.maven_jar( + name = "io_netty_netty_codec_socks", + artifact = "io.netty:netty-codec-socks:4.1.22.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "d077b39da2dedc5dc5db50a44e5f4c30353e86f3", + ) + + + # io.netty:netty-codec-http:jar:4.1.22.Final + # io.netty:netty-handler:jar:4.1.22.Final got requested version + # io.netty:netty-codec-socks:jar:4.1.22.Final got requested version + native.maven_jar( + name = "io_netty_netty_codec", + artifact = "io.netty:netty-codec:4.1.22.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "239c0af275952e70bb4adf7cf8c03d88ddc394c9", + ) + + + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version + # io.netty:netty-transport:jar:4.1.22.Final got requested version + # io.netty:netty-handler:jar:4.1.22.Final got requested version + native.maven_jar( + name = "io_netty_netty_buffer", + artifact = "io.netty:netty-buffer:4.1.22.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "15e964a2095031364f534a6e21977f5ee9ca32a9", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 + native.maven_jar( + name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_core", + artifact = "de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-core:0.9.44", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "636cc11e0e09d363af87766550c1deef8ab0c5bf", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "io_grpc_grpc_netty", + artifact = "io.grpc:grpc-netty:1.14.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "21c6edadd45b6869384f8aa0df1663d62c503617", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-core:jar:0.9.44 wanted version 1.1.2 + # ch.qos.logback:logback-classic:jar:1.2.3 + native.maven_jar( + name = "ch_qos_logback_logback_core", + artifact = "ch.qos.logback:logback-core:1.2.3", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "864344400c3d4d92dfeb0a305dc87d953677c03c", + ) + + + # org.mockito:mockito-core:jar:2.7.22 + native.maven_jar( + name = "net_bytebuddy_byte_buddy", + artifact = "net.bytebuddy:byte-buddy:1.6.11", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "8a8f9409e27f1d62c909c7eef2aa7b3a580b4901", + ) + + + # io.opencensus:opencensus-contrib-grpc-metrics:jar:0.12.3 got requested version + # io.grpc:grpc-core:jar:1.14.0 + native.maven_jar( + name = "io_opencensus_opencensus_api", + artifact = "io.opencensus:opencensus-api:0.12.3", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "743f074095f29aa985517299545e72cc99c87de0", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-core:jar:0.9.44 + native.maven_jar( + name = "de_huxhorn_lilith_de_huxhorn_lilith_sender", + artifact = "de.huxhorn.lilith:de.huxhorn.lilith.sender:0.9.44", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "fdeb2f1aaba42cad76c460ac4fd7072b8642520f", + ) + + + native.maven_jar( + name = "org_javassist_javassist", + artifact = "org.javassist:javassist:3.19.0-GA", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "50120f69224dd8684b445a6f3a5b08fe9b5c60f6", + ) + + + # org.mockito:mockito-core:jar:2.7.22 + native.maven_jar( + name = "org_objenesis_objenesis", + artifact = "org.objenesis:objenesis:2.5", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "612ecb799912ccf77cba9b3ed8c813da086076e9", + ) + + + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT got requested version + # io.netty:netty-resolver:jar:4.1.22.Final got requested version + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT got requested version + # io.netty:netty-buffer:jar:4.1.22.Final + native.maven_jar( + name = "io_netty_netty_common", + artifact = "io.netty:netty-common:4.1.22.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "56ff4deca53fc791ed59ac2b72eb6718714a4de9", + ) + + + # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT wanted version 4.12 + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "junit_junit", + artifact = "junit:junit:4.11", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "4e031bb61df09069aeb2bffb4019e7a5034a4ee0", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 + native.maven_jar( + name = "de_huxhorn_lilith_de_huxhorn_lilith_data_logging_protobuf", + artifact = "de.huxhorn.lilith:de.huxhorn.lilith.data.logging.protobuf:0.9.44", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "56a037b365c7ff9b1c348e6d663fd3d778e2b0be", + ) + + + # de.huxhorn.sulky:de.huxhorn.sulky.codec:jar:0.9.17 + # de.huxhorn.lilith:de.huxhorn.lilith.sender:jar:0.9.44 got requested version + native.maven_jar( + name = "de_huxhorn_sulky_de_huxhorn_sulky_io", + artifact = "de.huxhorn.sulky:de.huxhorn.sulky.io:0.9.17", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "6baf1e17f80ba40f9a4b4b97235e72488047db02", + ) + + + native.maven_jar( + name = "log4j_log4j", + artifact = "log4j:log4j:1.2.17", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "5af35056b4d257e4b64b9e8069c0746e8b08629f", + ) + + + native.maven_jar( + name = "org_slf4j_slf4j_log4j12", + artifact = "org.slf4j:slf4j-log4j12:1.7.25", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "110cefe2df103412849d72ef7a67e4e91e4266b4", + ) + + + # org.slf4j:jcl-over-slf4j:jar:1.7.25 got requested version + # org.slf4j:jul-to-slf4j:jar:1.7.25 + # de.huxhorn.lilith:de.huxhorn.lilith.sender:jar:0.9.44 wanted version 1.7.7 + # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 wanted version 1.7.7 + # org.slf4j:log4j-over-slf4j:jar:1.7.25 got requested version + # ch.qos.logback:logback-classic:jar:1.2.3 got requested version + # de.huxhorn.sulky:de.huxhorn.sulky.codec:jar:0.9.17 wanted version 1.7.7 + # de.huxhorn.sulky:de.huxhorn.sulky.formatting:jar:0.9.17 wanted version 1.7.7 + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "org_slf4j_slf4j_api", + artifact = "org.slf4j:slf4j-api:1.7.25", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "da76ca59f6a57ee3102f8f9bd9cee742973efa8a", + ) + + + native.maven_jar( + name = "com_typesafe_config", + artifact = "com.typesafe:config:1.3.2", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "d6ac0ce079f114adce620f2360c92a70b2cb36dc", + ) + + + native.maven_jar( + name = "org_apache_commons_commons_lang3", + artifact = "org.apache.commons:commons-lang3:3.4", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "5fe28b9518e58819180a43a850fbc0dd24b7c050", + ) + + + # io.netty:netty-handler-proxy:jar:4.1.22.Final got requested version + # io.netty:netty-codec-http2:jar:4.1.22.Final + native.maven_jar( + name = "io_netty_netty_codec_http", + artifact = "io.netty:netty-codec-http:4.1.22.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "3805f3ca0d57630200defc7f9bb6ed3382dcb10b", + ) + + + # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOTi + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_classic", + artifact = "de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:0.9.44", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "1ac6ffe5fc63f7900f8af54b0dfebf2508b41db7", + ) + + + native.maven_jar( + name = "commons_collections_commons_collections", + artifact = "commons-collections:commons-collections:3.2.2", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "8ad72fe39fa8c91eaaf12aadb21e0c3661fe26d5", + ) + + + # junit:junit:jar:4.11 + native.maven_jar( + name = "org_hamcrest_hamcrest_core", + artifact = "org.hamcrest:hamcrest-core:1.3", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "42a25dc3219429f0e5d060061f71acb49bf010a0", + ) + + + # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "org_jmockit_jmockit", + artifact = "org.jmockit:jmockit:1.33", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "8a2746af90706d8d68f3919e1a87d5e47b8c94d7", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.data.logging:jar:0.9.44 + native.maven_jar( + name = "de_huxhorn_sulky_de_huxhorn_sulky_formatting", + artifact = "de.huxhorn.sulky:de.huxhorn.sulky.formatting:0.9.17", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "0501b143ee18ca9e7807438cc1b0acfa144f657f", + ) + + + # io.grpc:grpc-netty:jar:1.14.0 + native.maven_jar( + name = "io_netty_netty_codec_http2", + artifact = "io.netty:netty-codec-http2:4.1.27.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "3769790a2033667d663f9a526d5b63cfecdbdf4e", + ) + + + native.maven_jar( + name = "net_lingala_zip4j_zip4j", + artifact = "net.lingala.zip4j:zip4j:1.3.2", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "4ba84e98ee017b74cb52f45962f929a221f3074c", + ) + + + native.maven_jar( + name = "org_testng_testng", + artifact = "org.testng:testng:6.9.9", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "1bf509349476d6a48978cc2b04af9caa907781ab", + ) + + native.maven_jar( + name = "org_ini4j_ini4j", + artifact = "org.ini4j:ini4j:0.5.4", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "4a3ee4146a90c619b20977d65951825f5675b560", + ) + + + native.maven_jar( + name = "org_ow2_asm_asm", + artifact = "org.ow2.asm:asm:6.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "bc6fa6b19424bb9592fe43bbc20178f92d403105", + ) + + + # io.grpc:grpc-core:jar:1.14.0 + native.maven_jar( + name = "com_google_code_gson_gson", + artifact = "com.google.code.gson:gson:2.7",#2.8.5 + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "751f548c85fa49f330cecbb1875893f971b33c4e", + ) + + + # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "org_slf4j_jcl_over_slf4j", + artifact = "org.slf4j:jcl-over-slf4j:1.7.25", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "f8c32b13ff142a513eeb5b6330b1588dcb2c0461", + ) + + + # pom.xml got requested version + # com.fasterxml.jackson.core:jackson-databind:bundle:2.7.9 wanted version 2.7.0 + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version + native.maven_jar( + name = "com_fasterxml_jackson_core_jackson_annotations", + artifact = "com.fasterxml.jackson.core:jackson-annotations:2.7.9", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "eb356e825cb73da42f7c902a3fe0276fe32b26c8", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.data.logging.protobuf:jar:0.9.44 got requested version + # de.huxhorn.lilith:de.huxhorn.lilith.logback.classic:jar:0.9.44 + # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 got requested version + native.maven_jar( + name = "de_huxhorn_lilith_de_huxhorn_lilith_data_logging", + artifact = "de.huxhorn.lilith:de.huxhorn.lilith.data.logging:0.9.44", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "3574a72628f59d78ff17210f10de057188043a4c", + ) + + + # io.grpc:grpc-core:jar:1.14.0 + native.maven_jar( + name = "io_grpc_grpc_context", + artifact = "io.grpc:grpc-context:1.14.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "77252b5f926875891aaae5629e6ab2ef968cd6c6", + ) + + + # io.grpc:grpc-protobuf:jar:1.14.0 + native.maven_jar( + name = "com_google_api_grpc_proto_google_common_protos", + artifact = "com.google.api.grpc:proto-google-common-protos:1.0.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "86f070507e28b930e50d218ee5b6788ef0dd05e6", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 + native.maven_jar( + name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_classic", + artifact = "de.huxhorn.lilith:de.huxhorn.lilith.logback.classic:0.9.44", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "f84da242a1988240222f6a06b511a18ff9c64344", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "com_fasterxml_jackson_core_jackson_databind", + artifact = "com.fasterxml.jackson.core:jackson-databind:2.7.9", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "a4c0b14c7dd85bdf4d25da074e90a10fa4b9b88b", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "com_fasterxml_jackson_dataformat_jackson_dataformat_yaml", + artifact = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.7.9", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "163a1bd63ace592d2a449023a1c07056b885ff12", + ) + + + native.maven_jar( + name = "com_github_davidmoten_flatbuffers_java", + artifact = "com.github.davidmoten:flatbuffers-java:1.9.0.1", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "0eaa1b27095bb5127879458bc2ee1b78e00d4a20", + ) + + + native.maven_jar( + name = "redis_clients_jedis", + artifact = "redis.clients:jedis:2.8.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "a9486e200b2195f31ce9b8779a70a5e2450a73ba", + ) + + + native.maven_jar( + name = "org_apache_commons_commons_pool2", + artifact = "org.apache.commons:commons-pool2:2.3", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "62a559a025fd890c30364296ece14643ba9c8c5b", + ) + + + native.maven_jar( + name = "commons_io_commons_io", + artifact = "commons-io:commons-io:2.5", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "2852e6e05fbb95076fc091f6d1780f1f8fe35e0f", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + # io.grpc:grpc-protobuf:jar:1.14.0 got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version + # io.grpc:grpc-protobuf-lite:jar:1.14.0 got requested version + # io.grpc:grpc-core:jar:1.14.0 + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT wanted version 23.0 + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "com_google_guava_guava", + artifact = "com.google.guava:guava:20.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "89507701249388e1ed5ddcf8c41f4ce1be7831ef", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 + native.maven_jar( + name = "de_huxhorn_lilith_de_huxhorn_lilith_data_converter", + artifact = "de.huxhorn.lilith:de.huxhorn.lilith.data.converter:0.9.44", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "31e4994f9bc768c43491c3c0d12237ae478d21e3", + ) + + + native.maven_jar( + name = "de_ruedigermoeller_fst", + artifact = "de.ruedigermoeller:fst:2.47", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "281d390ebed24a3621d3053825affc13d0eead8b", + ) + + + # pom.xml got requested version + # com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:bundle:2.7.9 got requested version + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # com.fasterxml.jackson.core:jackson-databind:bundle:2.7.9 got requested version + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "com_fasterxml_jackson_core_jackson_core", + artifact = "com.fasterxml.jackson.core:jackson-core:2.7.9", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "09b530cec4fd2eb841ab8e79f19fc7cf0ec487b2", + ) + + + # pom.xml got requested version + # io.grpc:grpc-protobuf:jar:1.14.0 got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version + # io.grpc:grpc-protobuf-lite:jar:1.14.0 got requested version + # io.grpc:grpc-stub:jar:1.14.0 got requested version + # io.grpc:grpc-netty:jar:1.14.0 + native.maven_jar( + name = "io_grpc_grpc_core", + artifact = "io.grpc:grpc-core:1.14.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "526e5be291c96e248789d769c108a084febda07f", + ) + + + # com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:bundle:2.7.9 + native.maven_jar( + name = "org_yaml_snakeyaml", + artifact = "org.yaml:snakeyaml:1.15", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "3b132bea69e8ee099f416044970997bde80f4ea6", + ) + + + + +def generated_java_libraries(): + native.java_library( + name = "log4j_log4j", + visibility = ["//visibility:public"], + exports = ["@log4j_log4j//jar"], + ) + + + native.java_library( + name = "org_slf4j_slf4j_log4j12", + visibility = ["//visibility:public"], + exports = ["@org_slf4j_slf4j_log4j12//jar"], + runtime_deps = [ + ":log4j_log4j", + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "com_typesafe_config", + visibility = ["//visibility:public"], + exports = ["@com_typesafe_config//jar"], + ) + + + native.java_library( + name = "org_apache_commons_commons_lang3", + visibility = ["//visibility:public"], + exports = ["@org_apache_commons_commons_lang3//jar"], + ) + + + native.java_library( + name = "com_google_code_findbugs_jsr305", + visibility = ["//visibility:public"], + exports = ["@com_google_code_findbugs_jsr305//jar"], + ) + + + native.java_library( + name = "io_grpc_grpc_core", + visibility = ["//visibility:public"], + exports = ["@io_grpc_grpc_core//jar"], + runtime_deps = [ + ":com_google_code_findbugs_jsr305", + ":com_google_code_gson_gson", + ":com_google_errorprone_error_prone_annotations", + ":com_google_guava_guava", + ":io_grpc_grpc_context", + ":io_opencensus_opencensus_api", + ":io_opencensus_opencensus_contrib_grpc_metrics", + ], + ) + + + native.java_library( + name = "ch_qos_logback_logback_classic", + visibility = ["//visibility:public"], + exports = ["@ch_qos_logback_logback_classic//jar"], + runtime_deps = [ + ":ch_qos_logback_logback_core", + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "joda_time_joda_time", + visibility = ["//visibility:public"], + exports = ["@joda_time_joda_time//jar"], + ) + + + native.java_library( + name = "net_bytebuddy_byte_buddy_agent", + visibility = ["//visibility:public"], + exports = ["@net_bytebuddy_byte_buddy_agent//jar"], + ) + + + native.java_library( + name = "io_grpc_grpc_protobuf_lite", + visibility = ["//visibility:public"], + exports = ["@io_grpc_grpc_protobuf_lite//jar"], + runtime_deps = [ + ":com_google_guava_guava", + ":io_grpc_grpc_core", + ], + ) + + + native.java_library( + name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_converter_classic", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_logback_converter_classic//jar"], + runtime_deps = [ + ":ch_qos_logback_logback_classic", + ":de_huxhorn_lilith_de_huxhorn_lilith_data_converter", + ":de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", + ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging", + ":de_huxhorn_lilith_de_huxhorn_lilith_logback_classic", + ":de_huxhorn_sulky_de_huxhorn_sulky_codec", + ":de_huxhorn_sulky_de_huxhorn_sulky_formatting", + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "io_netty_netty_transport", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_transport//jar"], + runtime_deps = [ + ":io_netty_netty_buffer", + ":io_netty_netty_common", + ":io_netty_netty_resolver", + ], + ) + + + native.java_library( + name = "com_carrotsearch_hppc", + visibility = ["//visibility:public"], + exports = ["@com_carrotsearch_hppc//jar"], + ) + + + native.java_library( + name = "org_mockito_mockito_core", + visibility = ["//visibility:public"], + exports = ["@org_mockito_mockito_core//jar"], + runtime_deps = [ + ":net_bytebuddy_byte_buddy", + ":net_bytebuddy_byte_buddy_agent", + ":org_objenesis_objenesis", + ], + ) + + + native.java_library( + name = "io_netty_netty_handler_proxy", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_handler_proxy//jar"], + runtime_deps = [ + ":io_netty_netty_codec", + ":io_netty_netty_codec_http", + ":io_netty_netty_codec_socks", + ":io_netty_netty_transport", + ], + ) + + + native.java_library( + name = "de_huxhorn_sulky_de_huxhorn_sulky_codec", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_sulky_de_huxhorn_sulky_codec//jar"], + runtime_deps = [ + ":de_huxhorn_sulky_de_huxhorn_sulky_io", + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource//jar"], + ) + + + native.java_library( + name = "com_google_flatbuffers_flatbuffers_java", + visibility = ["//visibility:public"], + exports = ["@com_google_flatbuffers_flatbuffers_java//jar"], + ) + + + native.java_library( + name = "com_google_protobuf_protobuf_java", + visibility = ["//visibility:public"], + exports = ["@com_google_protobuf_protobuf_java//jar"], + ) + + + native.java_library( + name = "com_h2database_h2", + visibility = ["//visibility:public"], + exports = ["@com_h2database_h2//jar"], + ) + + + native.java_library( + name = "org_slf4j_jul_to_slf4j", + visibility = ["//visibility:public"], + exports = ["@org_slf4j_jul_to_slf4j//jar"], + runtime_deps = [ + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "commons_codec_commons_codec", + visibility = ["//visibility:public"], + exports = ["@commons_codec_commons_codec//jar"], + ) + + + native.java_library( + name = "org_slf4j_log4j_over_slf4j", + visibility = ["//visibility:public"], + exports = ["@org_slf4j_log4j_over_slf4j//jar"], + runtime_deps = [ + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "commons_cli_commons_cli", + visibility = ["//visibility:public"], + exports = ["@commons_cli_commons_cli//jar"], + ) + + + native.java_library( + name = "io_netty_netty_tcnative_boringssl_static", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_tcnative_boringssl_static//jar"], + ) + + + native.java_library( + name = "io_grpc_grpc_stub", + visibility = ["//visibility:public"], + exports = ["@io_grpc_grpc_stub//jar"], + runtime_deps = [ + ":io_grpc_grpc_core", + ], + ) + + + native.java_library( + name = "io_grpc_grpc_protobuf", + visibility = ["//visibility:public"], + exports = ["@io_grpc_grpc_protobuf//jar"], + runtime_deps = [ + ":com_google_api_grpc_proto_google_common_protos", + ":com_google_guava_guava", + ":com_google_protobuf_protobuf_java", + ":io_grpc_grpc_core", + ":io_grpc_grpc_protobuf_lite", + ], + ) + + + native.java_library( + name = "io_netty_netty_codec_socks", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_codec_socks//jar"], + runtime_deps = [ + ":io_netty_netty_codec", + ], + ) + + + native.java_library( + name = "io_netty_netty_codec", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_codec//jar"], + runtime_deps = [ + ":io_netty_netty_buffer", + ":io_netty_netty_common", + ":io_netty_netty_resolver", + ":io_netty_netty_transport", + ], + ) + + + native.java_library( + name = "io_netty_netty_buffer", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_buffer//jar"], + runtime_deps = [ + ":io_netty_netty_common", + ], + ) + + + native.java_library( + name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_core", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_core//jar"], + runtime_deps = [ + ":ch_qos_logback_logback_core", + ":de_huxhorn_lilith_de_huxhorn_lilith_sender", + ":de_huxhorn_sulky_de_huxhorn_sulky_codec", + ":de_huxhorn_sulky_de_huxhorn_sulky_io", + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "io_grpc_grpc_netty", + visibility = ["//visibility:public"], + exports = ["@io_grpc_grpc_netty//jar"], + runtime_deps = [ + ":com_google_code_findbugs_jsr305", + ":com_google_code_gson_gson", + ":com_google_errorprone_error_prone_annotations", + ":com_google_guava_guava", + ":io_grpc_grpc_context", + ":io_grpc_grpc_core", + ":io_netty_netty_buffer", + ":io_netty_netty_codec", + ":io_netty_netty_codec_http", + ":io_netty_netty_codec_http2", + ":io_netty_netty_codec_socks", + ":io_netty_netty_common", + ":io_netty_netty_handler", + ":io_netty_netty_handler_proxy", + ":io_netty_netty_resolver", + ":io_netty_netty_transport", + ":io_opencensus_opencensus_api", + ":io_opencensus_opencensus_contrib_grpc_metrics", + ], + ) + + + native.java_library( + name = "ch_qos_logback_logback_core", + visibility = ["//visibility:public"], + exports = ["@ch_qos_logback_logback_core//jar"], + ) + + + native.java_library( + name = "net_bytebuddy_byte_buddy", + visibility = ["//visibility:public"], + exports = ["@net_bytebuddy_byte_buddy//jar"], + ) + + + native.java_library( + name = "io_opencensus_opencensus_api", + visibility = ["//visibility:public"], + exports = ["@io_opencensus_opencensus_api//jar"], + ) + + + native.java_library( + name = "de_huxhorn_lilith_de_huxhorn_lilith_sender", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_sender//jar"], + runtime_deps = [ + ":de_huxhorn_sulky_de_huxhorn_sulky_io", + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "com_google_errorprone_error_prone_annotations", + visibility = ["//visibility:public"], + exports = ["@com_google_errorprone_error_prone_annotations//jar"], + ) + + + native.java_library( + name = "io_netty_netty_resolver", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_resolver//jar"], + runtime_deps = [ + ":io_netty_netty_common", + ], + ) + + + native.java_library( + name = "io_opencensus_opencensus_contrib_grpc_metrics", + visibility = ["//visibility:public"], + exports = ["@io_opencensus_opencensus_contrib_grpc_metrics//jar"], + runtime_deps = [ + ":io_opencensus_opencensus_api", + ], + ) + + + native.java_library( + name = "org_javassist_javassist", + visibility = ["//visibility:public"], + exports = ["@org_javassist_javassist//jar"], + ) + + + native.java_library( + name = "org_objenesis_objenesis", + visibility = ["//visibility:public"], + exports = ["@org_objenesis_objenesis//jar"], + ) + + + native.java_library( + name = "io_netty_netty_common", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_common//jar"], + ) + + + native.java_library( + name = "junit_junit", + visibility = ["//visibility:public"], + exports = ["@junit_junit//jar"], + runtime_deps = [ + ":org_hamcrest_hamcrest_core", + ], + ) + + + native.java_library( + name = "de_huxhorn_lilith_de_huxhorn_lilith_data_logging_protobuf", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_data_logging_protobuf//jar"], + runtime_deps = [ + ":com_google_protobuf_protobuf_java", + ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging", + ":de_huxhorn_sulky_de_huxhorn_sulky_codec", + ], + ) + + + native.java_library( + name = "de_huxhorn_sulky_de_huxhorn_sulky_io", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_sulky_de_huxhorn_sulky_io//jar"], + ) + + + native.java_library( + name = "org_slf4j_slf4j_api", + visibility = ["//visibility:public"], + exports = ["@org_slf4j_slf4j_api//jar"], + ) + + + native.java_library( + name = "io_netty_netty_codec_http", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_codec_http//jar"], + runtime_deps = [ + ":io_netty_netty_buffer", + ":io_netty_netty_codec", + ":io_netty_netty_common", + ":io_netty_netty_resolver", + ":io_netty_netty_transport", + ], + ) + + + native.java_library( + name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_classic", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_classic//jar"], + runtime_deps = [ + ":ch_qos_logback_logback_classic", + ":ch_qos_logback_logback_core", + ":com_google_protobuf_protobuf_java", + ":de_huxhorn_lilith_de_huxhorn_lilith_data_converter", + ":de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", + ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging", + ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging_protobuf", + ":de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_core", + ":de_huxhorn_lilith_de_huxhorn_lilith_logback_classic", + ":de_huxhorn_lilith_de_huxhorn_lilith_logback_converter_classic", + ":de_huxhorn_lilith_de_huxhorn_lilith_sender", + ":de_huxhorn_sulky_de_huxhorn_sulky_codec", + ":de_huxhorn_sulky_de_huxhorn_sulky_formatting", + ":de_huxhorn_sulky_de_huxhorn_sulky_io", + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "org_hamcrest_hamcrest_core", + visibility = ["//visibility:public"], + exports = ["@org_hamcrest_hamcrest_core//jar"], + ) + + + native.java_library( + name = "io_netty_netty_handler", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_handler//jar"], + runtime_deps = [ + ":io_netty_netty_buffer", + ":io_netty_netty_codec", + ":io_netty_netty_transport", + ], + ) + + + native.java_library( + name = "org_jmockit_jmockit", + visibility = ["//visibility:public"], + exports = ["@org_jmockit_jmockit//jar"], + ) + + + native.java_library( + name = "de_huxhorn_sulky_de_huxhorn_sulky_formatting", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_sulky_de_huxhorn_sulky_formatting//jar"], + runtime_deps = [ + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "io_netty_netty_codec_http2", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_codec_http2//jar"], + runtime_deps = [ + ":io_netty_netty_buffer", + ":io_netty_netty_codec", + ":io_netty_netty_codec_http", + ":io_netty_netty_common", + ":io_netty_netty_handler", + ":io_netty_netty_resolver", + ":io_netty_netty_transport", + ], + ) + + + native.java_library( + name = "net_lingala_zip4j_zip4j", + visibility = ["//visibility:public"], + exports = ["@net_lingala_zip4j_zip4j//jar"], + ) + + + native.java_library( + name = "org_ini4j_ini4j", + visibility = ["//visibility:public"], + exports = ["@org_ini4j_ini4j//jar"], + ) + + + native.java_library( + name = "org_ow2_asm_asm", + visibility = ["//visibility:public"], + exports = ["@org_ow2_asm_asm//jar"], + ) + + + native.java_library( + name = "com_google_code_gson_gson", + visibility = ["//visibility:public"], + exports = ["@com_google_code_gson_gson//jar"], + ) + + + native.java_library( + name = "org_slf4j_jcl_over_slf4j", + visibility = ["//visibility:public"], + exports = ["@org_slf4j_jcl_over_slf4j//jar"], + runtime_deps = [ + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "com_fasterxml_jackson_core_jackson_annotations", + visibility = ["//visibility:public"], + exports = ["@com_fasterxml_jackson_core_jackson_annotations//jar"], + ) + + + native.java_library( + name = "de_huxhorn_lilith_de_huxhorn_lilith_data_logging", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_data_logging//jar"], + runtime_deps = [ + ":de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", + ":de_huxhorn_sulky_de_huxhorn_sulky_formatting", + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_classic", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_logback_classic//jar"], + runtime_deps = [ + ":de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", + ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging", + ":de_huxhorn_sulky_de_huxhorn_sulky_formatting", + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "com_fasterxml_jackson_core_jackson_databind", + visibility = ["//visibility:public"], + exports = ["@com_fasterxml_jackson_core_jackson_databind//jar"], + runtime_deps = [ + ":com_fasterxml_jackson_core_jackson_annotations", + ":com_fasterxml_jackson_core_jackson_core", + ], + ) + + + native.java_library( + name = "com_fasterxml_jackson_dataformat_jackson_dataformat_yaml", + visibility = ["//visibility:public"], + exports = ["@com_fasterxml_jackson_dataformat_jackson_dataformat_yaml//jar"], + runtime_deps = [ + ":com_fasterxml_jackson_core_jackson_core", + ":org_yaml_snakeyaml", + ], + ) + + + native.java_library( + name = "io_grpc_grpc_context", + visibility = ["//visibility:public"], + exports = ["@io_grpc_grpc_context//jar"], + ) + + + native.java_library( + name = "com_google_api_grpc_proto_google_common_protos", + visibility = ["//visibility:public"], + exports = ["@com_google_api_grpc_proto_google_common_protos//jar"], + ) + + + native.java_library( + name = "com_github_davidmoten_flatbuffers_java", + visibility = ["//visibility:public"], + exports = ["@com_github_davidmoten_flatbuffers_java//jar"], + ) + + + native.java_library( + name = "redis_clients_jedis", + visibility = ["//visibility:public"], + exports = ["@redis_clients_jedis//jar"], + runtime_deps = [ + ":org_apache_commons_commons_pool2", + ], + ) + + + native.java_library( + name = "org_apache_commons_commons_pool2", + visibility = ["//visibility:public"], + exports = ["@org_apache_commons_commons_pool2//jar"], + ) + + + native.java_library( + name = "com_google_guava_guava", + visibility = ["//visibility:public"], + exports = ["@com_google_guava_guava//jar"], + ) + + + native.java_library( + name = "de_huxhorn_lilith_de_huxhorn_lilith_data_converter", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_data_converter//jar"], + ) + + + native.java_library( + name = "de_ruedigermoeller_fst", + visibility = ["//visibility:public"], + exports = ["@de_ruedigermoeller_fst//jar"], + runtime_deps = [ + ":com_fasterxml_jackson_core_jackson_core", + ":org_javassist_javassist", + ":org_objenesis_objenesis", + ], + ) + + + native.java_library( + name = "com_fasterxml_jackson_core_jackson_core", + visibility = ["//visibility:public"], + exports = ["@com_fasterxml_jackson_core_jackson_core//jar"], + ) + + + native.java_library( + name = "org_yaml_snakeyaml", + visibility = ["//visibility:public"], + exports = ["@org_yaml_snakeyaml//jar"], + ) + + + native.java_library( + name = "net_java_dev_jna_jna", + visibility = ["//visibility:public"], + exports = ["@net_java_dev_jna_jna//jar"], + ) + + + native.java_library( + name = "org_mockito_mockito_all", + visibility = ["//visibility:public"], + exports = ["@org_mockito_mockito_all//jar"], + ) + + + native.java_library( + name = "com_beust_jcommander", + visibility = ["//visibility:public"], + exports = ["@com_beust_jcommander//jar"], + ) + + native.java_library( + name = "org_testng_testng", + visibility = ["//visibility:public"], + exports = ["@org_testng_testng//jar"], + ) diff --git a/build.sh b/build.sh index cf137c959161..1b143b6da838 100755 --- a/build.sh +++ b/build.sh @@ -121,14 +121,21 @@ else $PYTHON_EXECUTABLE -m pip install \ --target=$ROOT_DIR/python/ray/pyarrow_files pyarrow==0.12.0.RAY \ --find-links https://s3-us-west-2.amazonaws.com/arrow-wheels/9357dc130789ee42f8181d8724bee1d5d1509060/index.html - bazel build //:ray_pkg -c opt - # Copy files and keep them writeable. This is a workaround, as Bazel - # marks all generated files non-writeable. If we would just copy them - # over without adding write permission, the copy would fail the next time. - # TODO(pcm): It would be great to have a solution here that does not - # require us to copy the files. - find $ROOT_DIR/bazel-genfiles/ray_pkg/ -exec chmod +w {} \; - cp -r $ROOT_DIR/bazel-genfiles/ray_pkg/ray $ROOT_DIR/python || true + if [ "$RAY_BUILD_PYTHON" == "YES" ]; then + bazel build //:ray_pkg -c opt + # Copy files and keep them writeable. This is a workaround, as Bazel + # marks all generated files non-writeable. If we would just copy them + # over without adding write permission, the copy would fail the next time. + # TODO(pcm): It would be great to have a solution here that does not + # require us to copy the files. + find $ROOT_DIR/bazel-genfiles/ray_pkg/ -exec chmod +w {} \; + cp -r $ROOT_DIR/bazel-genfiles/ray_pkg/ray $ROOT_DIR/python || true + fi + if [ "$RAY_BUILD_JAVA" == "YES" ]; then + bazel build //:ray_java_pkg -c opt + find $ROOT_DIR/bazel-genfiles/ray_java_pkg/ -exec chmod +w {} \; + cp -r $ROOT_DIR/bazel-genfiles/ray_java_pkg/ray/* $ROOT_DIR/build || true + fi fi popd diff --git a/java/BUILD.bazel b/java/BUILD.bazel new file mode 100644 index 000000000000..adb4bf3cff20 --- /dev/null +++ b/java/BUILD.bazel @@ -0,0 +1,211 @@ + +load("//bazel:ray.bzl", "flatbuffer_java_library") +load("//bazel:ray_java.bzl", "generated_java_libraries") + +generated_java_libraries() + +java_import( + name = "org_ray_ray_java_root", + jars = [ + "liborg_ray_ray_api.jar", + "liborg_ray_ray_api-src.jar", + "liborg_ray_ray_runtime.jar", + "liborg_ray_ray_runtime-src.jar", + "liborg_ray_ray_cli.jar", + "liborg_ray_ray_cli-src.jar", + "liborg_ray_ray_runtime_test.jar", + "liborg_ray_ray_runtime_test-src.jar", + "liborg_ray_ray_test.jar", + "liborg_ray_ray_test-src.jar", + "org_ray_ray_tutorial_deploy.jar", + "org_ray_ray_tutorial_deploy-src.jar", + "AllTests_deploy.jar", + "AllTests_deploy-src.jar", + ], + deps = [ + ":org_ray_ray_api", + ":org_ray_ray_cli", + ":org_ray_ray_runtime", + ":org_ray_ray_tutorial", + ":org_ray_ray_test", + ":org_ray_ray_runtime_test", + ], +) + +java_library( + name = "org_ray_ray_api", + srcs = glob(["api/src/main/java/**/*.java"]), + deps = [ + "@org_slf4j_slf4j_log4j12//jar", + "@log4j_log4j//jar", + ], +) + +java_library( + name = "org_ray_ray_cli", + srcs = glob(["cli/src/main/java/**/*.java"]), + visibility = ["//visibility:public"], + deps = [ + "@com_beust_jcommander//jar", + "@com_github_davidmoten_flatbuffers_java//jar", + "@de_ruedigermoeller_fst//jar", + ":org_ray_ray_api", + ":org_ray_ray_runtime", + "@redis_clients_jedis//jar", + "@org_slf4j_slf4j_api//jar", + ], +) + +java_library( + name = "org_ray_ray_runtime", + srcs = glob(["runtime/src/main/java/**/*.java"]), + resources = glob(["runtime/src/main/resources/**"]), + visibility = ["//visibility:public"], + data = ["//:raylet_library_java", ":copy_java_gcs_fbs"], + deps = [ + ":org_ray_ray_api", + "@com_typesafe_config//jar", + "@org_apache_commons_commons_lang3//jar", + "@de_ruedigermoeller_fst//jar", + "@com_github_davidmoten_flatbuffers_java//jar", + "@redis_clients_jedis//jar", + "@plasma//:org_apache_arrow_arrow_plasma", + "@commons_io_commons_io//jar", + "@com_google_guava_guava//jar", + "@net_lingala_zip4j_zip4j//jar", + "@org_slf4j_slf4j_log4j12//jar", + "@org_slf4j_slf4j_api//jar", + "@org_ini4j_ini4j//jar", + "@org_ow2_asm_asm//jar", + "@com_google_code_gson_gson//jar", + "@com_fasterxml_jackson_core_jackson_core//jar", + "@org_apache_commons_commons_pool2//jar", + "@org_javassist_javassist//jar", + "@org_objenesis_objenesis//jar", + ], +) + +java_binary( + name = "org_ray_ray_tutorial", + srcs = glob(["tutorial/src/main/java/**/*.java"]), + resources = glob(["tutorial/src/main/resources/**"]), + visibility = ["//visibility:public"], + deps = [ + ":org_ray_ray_api", + ":org_ray_ray_runtime", + "@com_google_guava_guava//jar", + ], +) + +java_library( + name = "org_ray_ray_test", + srcs = glob(["test/src/main/java/**/*.java"]), + deps = [ + "@com_google_guava_guava//jar", + "@commons_collections_commons_collections//jar", + ":org_ray_ray_api", + ":org_ray_ray_runtime", + "@org_apache_commons_commons_lang3//jar", + "@org_slf4j_slf4j_api//jar", + "@plasma//:org_apache_arrow_arrow_plasma", + "@org_testng_testng//jar", + ], +) + +java_library( + name = "org_ray_ray_runtime_test", + srcs = glob(["runtime/src/test/java/**/*.java"]), + deps = [ + ":org_ray_ray_api", + ":org_ray_ray_runtime", + "@org_testng_testng//jar", + "@org_apache_commons_commons_lang3//jar", + "@com_beust_jcommander//jar", + ], +) + +java_test( + name = "AllTests", + size = "small", + main_class = "org.testng.TestNG", + data = ["testng.xml"], + runtime_deps = [ + ":org_ray_ray_test", + ":org_ray_ray_runtime_test", + ], +) + +#use .py file to generate *.java, but be telled can't to find the generated files +#genrule( +# name = "generate_java_gcs_fbs", +# outs = ["generate_java_gcs_fbs.out"], +# cmd = """ +# python3 $$(pwd)/java/modify_generated_java_flatbuffers_files.py $$(pwd) +# echo $$(pwd) > $@ +# """, +# local = 1, +#) + +flatbuffer_java_library( + name = "java_gcs_fbs", + srcs = ["//:gcs_fbs_file"], + outs = [ + "ActorCheckpointData.java", + "ActorCheckpointIdData.java", + "ActorState.java", + "ActorTableData.java", + "Arg.java", + "ClassTableData.java", + "ClientTableData.java", + "ClientType.java", + "ConfigTableData.java", + "CustomSerializerData.java", + "DriverTableData.java", + "ErrorTableData.java", + "ErrorType.java", + "FunctionTableData.java", + "GcsTableEntry.java", + "GcsTableNotificationMode.java", + "HeartbeatBatchTableData.java", + "HeartbeatTableData.java", + "Language.java", + "ObjectTableData.java", + "PipeClientData.java", + "ProfileEvent.java", + "ProfileTableData.java", + "RayResource.java", + "ResourcePair.java", + "SchedulingState.java", + "TablePrefix.java", + "TablePubsub.java", + "TaskInfo.java", + "TaskLeaseData.java", + "TaskReconstructionData.java", + "TaskTableData.java", + "TaskTableTestAndUpdate.java", + ], + out_prefix = "java/runtime/src/main/java/org/ray/runtime/generated", +) + +genrule( + name = "copy_java_gcs_fbs", + srcs = [ + "//java:java_gcs_fbs", + ], + outs = [ + "java_gcs_fbs.out", + ], + cmd = """ + WORK_DIR=$$(pwd) + GENERATED_DIR=$$WORK_DIR/java/runtime/src/main/java/org/ray/runtime/generated + mkdir -p $$GENERATED_DIR + for f in $(locations //java:java_gcs_fbs); do + cp $$f $$GENERATED_DIR/; + file="$$GENERATED_DIR/$${f#*java/}"; + chmod +w $$file; + sed -i "2i\package org.ray.runtime.generated;" $$file; + done + echo $$(pwd) >> $@ + """, + local = 1, +) diff --git a/java/build.sh b/java/build.sh index 60b361ae8eb9..b9a25686e5e4 100755 --- a/java/build.sh +++ b/java/build.sh @@ -1,3 +1,7 @@ #!/usr/bin/env bash -mvn clean install -Dmaven.test.skip +if [ ! -z "$RAY_USE_CMAKE" ] ; then + mvn clean install -Dmaven.test.skip +else + bazel build //java:org_ray_ray_java_root +fi diff --git a/java/test.sh b/java/test.sh index 1c6370d1fe8e..7eeec4a071e9 100755 --- a/java/test.sh +++ b/java/test.sh @@ -9,29 +9,38 @@ set -x ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) $ROOT_DIR/../build.sh -l java -pushd $ROOT_DIR/../java -mvn clean install -Dmaven.test.skip -check_style=$(mvn checkstyle:check) -echo "${check_style}" -[[ ${check_style} =~ "BUILD FAILURE" ]] && exit 1 - -# test raylet -mvn test | tee mvn_test -if [ `grep -c "BUILD FAILURE" mvn_test` -eq '0' ]; then - rm mvn_test - echo "Tests passed under CLUSTER mode!" -else - rm mvn_test - exit 1 -fi -# test raylet under SINGLE_PROCESS mode -mvn test -Dray.run-mode=SINGLE_PROCESS | tee dev_mvn_test -if [ `grep -c "BUILD FAILURE" dev_mvn_test` -eq '0' ]; then - rm dev_mvn_test - echo "Tests passed under SINGLE_PROCESS mode!" +if [ ! -z "$RAY_USE_CMAKE" ] ; then + pushd $ROOT_DIR/../java + mvn clean install -Dmaven.test.skip + check_style=$(mvn checkstyle:check) + echo "${check_style}" + [[ ${check_style} =~ "BUILD FAILURE" ]] && exit 1 + + # test raylet + mvn test | tee mvn_test + if [ `grep -c "BUILD FAILURE" mvn_test` -eq '0' ]; then + rm mvn_test + echo "Tests passed under CLUSTER mode!" + else + rm mvn_test + exit 1 + fi + # test raylet under SINGLE_PROCESS mode + mvn test -Dray.run-mode=SINGLE_PROCESS | tee dev_mvn_test + if [ `grep -c "BUILD FAILURE" dev_mvn_test` -eq '0' ]; then + rm dev_mvn_test + echo "Tests passed under SINGLE_PROCESS mode!" + else + rm dev_mvn_test + exit 1 + fi else - rm dev_mvn_test - exit 1 + pushd $ROOT_DIR/.. + bazel build -c opt //java:all + pushd $ROOT_DIR/../java/test + java -jar -Dray.home=$ROOT_DIR/../ -Djava.library.path=$ROOT_DIR/../bazel-bin:$ROOT_DIR/../bazel-bin/externl/plasma:$ROOT_DIR/../bazel-genfiles $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml + + java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS -Djava.library.path=$ROOT_DIR/../bazel-bin:$ROOT_DIR/../bazel-bin/externl/plasma:$ROOT_DIR/../bazel-genfiles $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml fi popd diff --git a/java/testng.xml b/java/testng.xml new file mode 100644 index 000000000000..4dc31f8f99d2 --- /dev/null +++ b/java/testng.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + From c39a75b42568539755381cfbb183e0fe3aad2c1b Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Wed, 6 Mar 2019 03:17:33 +0800 Subject: [PATCH 003/115] java bazel config --- java/BUILD.bazel | 21 ------------------- .../org/ray/api/test/ClientExceptionTest.java | 4 ++-- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/java/BUILD.bazel b/java/BUILD.bazel index adb4bf3cff20..d0394f9aa24a 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -11,8 +11,6 @@ java_import( "liborg_ray_ray_api-src.jar", "liborg_ray_ray_runtime.jar", "liborg_ray_ray_runtime-src.jar", - "liborg_ray_ray_cli.jar", - "liborg_ray_ray_cli-src.jar", "liborg_ray_ray_runtime_test.jar", "liborg_ray_ray_runtime_test-src.jar", "liborg_ray_ray_test.jar", @@ -24,7 +22,6 @@ java_import( ], deps = [ ":org_ray_ray_api", - ":org_ray_ray_cli", ":org_ray_ray_runtime", ":org_ray_ray_tutorial", ":org_ray_ray_test", @@ -41,21 +38,6 @@ java_library( ], ) -java_library( - name = "org_ray_ray_cli", - srcs = glob(["cli/src/main/java/**/*.java"]), - visibility = ["//visibility:public"], - deps = [ - "@com_beust_jcommander//jar", - "@com_github_davidmoten_flatbuffers_java//jar", - "@de_ruedigermoeller_fst//jar", - ":org_ray_ray_api", - ":org_ray_ray_runtime", - "@redis_clients_jedis//jar", - "@org_slf4j_slf4j_api//jar", - ], -) - java_library( name = "org_ray_ray_runtime", srcs = glob(["runtime/src/main/java/**/*.java"]), @@ -157,7 +139,6 @@ flatbuffer_java_library( "Arg.java", "ClassTableData.java", "ClientTableData.java", - "ClientType.java", "ConfigTableData.java", "CustomSerializerData.java", "DriverTableData.java", @@ -165,12 +146,10 @@ flatbuffer_java_library( "ErrorType.java", "FunctionTableData.java", "GcsTableEntry.java", - "GcsTableNotificationMode.java", "HeartbeatBatchTableData.java", "HeartbeatTableData.java", "Language.java", "ObjectTableData.java", - "PipeClientData.java", "ProfileEvent.java", "ProfileTableData.java", "RayResource.java", diff --git a/java/test/src/main/java/org/ray/api/test/ClientExceptionTest.java b/java/test/src/main/java/org/ray/api/test/ClientExceptionTest.java index 0c0433299386..4e6c4cab3ccb 100644 --- a/java/test/src/main/java/org/ray/api/test/ClientExceptionTest.java +++ b/java/test/src/main/java/org/ray/api/test/ClientExceptionTest.java @@ -41,12 +41,12 @@ public void testWaitAndCrash() { Ray.wait(ImmutableList.of(notExisting), 1, 2000); Assert.fail("Should not reach here"); } catch (RayException e) { - LOGGER.debug(String.format("Expected runtime exception: {}", e)); + LOGGER.debug("Expected runtime exception: {}", e); } try { thread.join(); } catch (Exception e) { - LOGGER.error(String.format("Excpetion caught: {}", e)); + LOGGER.error("Excpetion caught: {}", e); } } } From 268e6949fa9008dfc1afc98f52c3a9193e88e292 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Wed, 6 Mar 2019 15:27:11 +0800 Subject: [PATCH 004/115] java bazel config --- BUILD.bazel | 15 ++------------- build.sh | 21 ++++++++++++++++----- java/BUILD.bazel | 24 +++++++++++------------- 3 files changed, 29 insertions(+), 31 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 69683a35a7d4..04656c645528 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -596,19 +596,8 @@ genrule( "@plasma//:plasma_store_server", "@plasma//:plasma_client_java", ], - outs = ["ray_java_pkg"], + outs = ["ray_java_pkg.out"], cmd = """ - set -x && - mkdir -p java/ray/src/plasma/ && - cp $(location @plasma//:plasma_client_java) java/ray/src/plasma/ && - cp $(location @plasma//:plasma_store_server) java/ray/src/plasma/ && - mkdir -p java/ray/src/ray/raylet/ && - cp $(location //:raylet) java/ray/src/ray/raylet/ && - cp $(location //:raylet_library_java) java/ray/src/ray/raylet/ && - mkdir -p java/ray/src/ray/gcs/redis_module/ && - cp $(location //:ray_redis_module.so) java/ray/src/ray/gcs/redis_module/libray_redis_module.so && - mkdir -p java/ray/src/ray/thirdparty/redis/src/ && - cp $(location //:redis-server) java/ray/src/ray/thirdparty/redis/src/ && - mv java $(location ray_java_pkg) + echo $$(pwd) > $@ """, ) diff --git a/build.sh b/build.sh index 1b143b6da838..590c0da1713f 100755 --- a/build.sh +++ b/build.sh @@ -121,6 +121,22 @@ else $PYTHON_EXECUTABLE -m pip install \ --target=$ROOT_DIR/python/ray/pyarrow_files pyarrow==0.12.0.RAY \ --find-links https://s3-us-west-2.amazonaws.com/arrow-wheels/9357dc130789ee42f8181d8724bee1d5d1509060/index.html + +if [ "$RAY_BUILD_JAVA" == "YES" ]; then + bazel build //:ray_java_pkg -c opt + #soft link + mkdir -p $ROOT_DIR/build/src/ray/raylet/ + mkdir -p $ROOT_DIR/build/src/ray/gcs/redis_module/ + mkdir -p $ROOT_DIR/build/src/ray/thirdparty/redis/src/ + mkdir -p $ROOT_DIR/build/src/plasma/ + ln -sf $ROOT_DIR/bazel-bin/* $ROOT_DIR/build/src/ray/raylet/ + ln -sf $ROOT_DIR/bazel-bin/external/plasma/* $ROOT_DIR/build/src/plasma/ + ln -sf $ROOT_DIR/bazel-genfiles/redis-server $ROOT_DIR/build/src/ray/thirdparty/redis/src/redis-server + ln -sf $ROOT_DIR/bazel-bin/ray_redis_module.so $ROOT_DIR/build/src/ray/gcs/redis_module/libray_redis_module.so + ln -sf $ROOT_DIR/bazel-bin/external/plasma/plasma_store_server $ROOT_DIR/build/src/plasma/plasma_store_server + ln -sf $ROOT_DIR/bazel-bin/raylet $ROOT_DIR/build/src/ray/raylet/raylet + fi + if [ "$RAY_BUILD_PYTHON" == "YES" ]; then bazel build //:ray_pkg -c opt # Copy files and keep them writeable. This is a workaround, as Bazel @@ -131,11 +147,6 @@ else find $ROOT_DIR/bazel-genfiles/ray_pkg/ -exec chmod +w {} \; cp -r $ROOT_DIR/bazel-genfiles/ray_pkg/ray $ROOT_DIR/python || true fi - if [ "$RAY_BUILD_JAVA" == "YES" ]; then - bazel build //:ray_java_pkg -c opt - find $ROOT_DIR/bazel-genfiles/ray_java_pkg/ -exec chmod +w {} \; - cp -r $ROOT_DIR/bazel-genfiles/ray_java_pkg/ray/* $ROOT_DIR/build || true - fi fi popd diff --git a/java/BUILD.bazel b/java/BUILD.bazel index d0394f9aa24a..72ee53778440 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -43,7 +43,7 @@ java_library( srcs = glob(["runtime/src/main/java/**/*.java"]), resources = glob(["runtime/src/main/resources/**"]), visibility = ["//visibility:public"], - data = ["//:raylet_library_java", ":copy_java_gcs_fbs"], + data = ["//:raylet_library_java", ":generate_java_gcs_fbs"], deps = [ ":org_ray_ray_api", "@com_typesafe_config//jar", @@ -118,15 +118,16 @@ java_test( ) #use .py file to generate *.java, but be telled can't to find the generated files -#genrule( -# name = "generate_java_gcs_fbs", -# outs = ["generate_java_gcs_fbs.out"], -# cmd = """ -# python3 $$(pwd)/java/modify_generated_java_flatbuffers_files.py $$(pwd) -# echo $$(pwd) > $@ -# """, -# local = 1, -#) +genrule( + name = "generate_java_gcs_fbs", + srcs = [":copy_java_gcs_fbs"], + outs = ["generate_java_gcs_fbs.out"], + cmd = """ + python3 $$(pwd)/java/modify_generated_java_flatbuffers_files.py $$(pwd) + echo $$(pwd) > $@ + """, + local = 1, +) flatbuffer_java_library( name = "java_gcs_fbs", @@ -180,9 +181,6 @@ genrule( mkdir -p $$GENERATED_DIR for f in $(locations //java:java_gcs_fbs); do cp $$f $$GENERATED_DIR/; - file="$$GENERATED_DIR/$${f#*java/}"; - chmod +w $$file; - sed -i "2i\package org.ray.runtime.generated;" $$file; done echo $$(pwd) >> $@ """, From c63eb0456039327a0eb9c3ecaeddd54b6893f6a6 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Wed, 6 Mar 2019 15:33:07 +0800 Subject: [PATCH 005/115] java bazel config --- java/BUILD.bazel | 1 - 1 file changed, 1 deletion(-) diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 72ee53778440..170b04b04cbf 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -117,7 +117,6 @@ java_test( ], ) -#use .py file to generate *.java, but be telled can't to find the generated files genrule( name = "generate_java_gcs_fbs", srcs = [":copy_java_gcs_fbs"], From c2bc0d88121a7ca0d182a98f238f2fbffcc6e50e Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 19:12:54 +0800 Subject: [PATCH 006/115] Update test.sh delete the setting of -Djava.library.path --- java/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/test.sh b/java/test.sh index 7eeec4a071e9..f7f846872b25 100755 --- a/java/test.sh +++ b/java/test.sh @@ -38,9 +38,9 @@ else pushd $ROOT_DIR/.. bazel build -c opt //java:all pushd $ROOT_DIR/../java/test - java -jar -Dray.home=$ROOT_DIR/../ -Djava.library.path=$ROOT_DIR/../bazel-bin:$ROOT_DIR/../bazel-bin/externl/plasma:$ROOT_DIR/../bazel-genfiles $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml + java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml - java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS -Djava.library.path=$ROOT_DIR/../bazel-bin:$ROOT_DIR/../bazel-bin/externl/plasma:$ROOT_DIR/../bazel-genfiles $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml + java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml fi popd From 1b8a5a450b3250a8d6f945c97868ca8e151414c3 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Wed, 6 Mar 2019 19:29:19 +0800 Subject: [PATCH 007/115] Update java/testng.xml Modified according to the recommendations of raulchen Co-Authored-By: alegithub111 --- java/testng.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/testng.xml b/java/testng.xml index 4dc31f8f99d2..5ccbb15a19b8 100644 --- a/java/testng.xml +++ b/java/testng.xml @@ -5,7 +5,7 @@ - + From 40c0b62bf09ef0b84abf4a4798106247f15026a5 Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 19:46:38 +0800 Subject: [PATCH 008/115] Update build.sh Modified according to the recommendations of raulchen --- java/build.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/java/build.sh b/java/build.sh index b9a25686e5e4..06a99d4f63de 100755 --- a/java/build.sh +++ b/java/build.sh @@ -1,7 +1,3 @@ #!/usr/bin/env bash -if [ ! -z "$RAY_USE_CMAKE" ] ; then - mvn clean install -Dmaven.test.skip -else - bazel build //java:org_ray_ray_java_root -fi +bazel build //java:all From 8eb90fa77d3b335d2e51e40649757faf1ef5523b Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 19:48:18 +0800 Subject: [PATCH 009/115] Update test.sh Modified according to the recommendations of raulchen --- java/test.sh | 39 +++++++-------------------------------- 1 file changed, 7 insertions(+), 32 deletions(-) diff --git a/java/test.sh b/java/test.sh index f7f846872b25..5c454cde3a16 100755 --- a/java/test.sh +++ b/java/test.sh @@ -9,38 +9,13 @@ set -x ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) $ROOT_DIR/../build.sh -l java -if [ ! -z "$RAY_USE_CMAKE" ] ; then - pushd $ROOT_DIR/../java - mvn clean install -Dmaven.test.skip - check_style=$(mvn checkstyle:check) - echo "${check_style}" - [[ ${check_style} =~ "BUILD FAILURE" ]] && exit 1 - - # test raylet - mvn test | tee mvn_test - if [ `grep -c "BUILD FAILURE" mvn_test` -eq '0' ]; then - rm mvn_test - echo "Tests passed under CLUSTER mode!" - else - rm mvn_test - exit 1 - fi - # test raylet under SINGLE_PROCESS mode - mvn test -Dray.run-mode=SINGLE_PROCESS | tee dev_mvn_test - if [ `grep -c "BUILD FAILURE" dev_mvn_test` -eq '0' ]; then - rm dev_mvn_test - echo "Tests passed under SINGLE_PROCESS mode!" - else - rm dev_mvn_test - exit 1 - fi -else - pushd $ROOT_DIR/.. - bazel build -c opt //java:all - pushd $ROOT_DIR/../java/test - java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml +pushd $ROOT_DIR/.. +bazel build -c opt //java:all +pushd $ROOT_DIR/../java/test +# test raylet +java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml - java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml -fi +# test raylet under SINGLE_PROCESS mode +java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml popd From bded0ae98ceb08555de63cd856766b0abb3d7d2f Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 19:49:02 +0800 Subject: [PATCH 010/115] Update build.sh --- java/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/build.sh b/java/build.sh index 06a99d4f63de..9fc0aa7ab742 100755 --- a/java/build.sh +++ b/java/build.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -bazel build //java:all +bazel build -c opt //java:all From 0263855d7025009c11af826c7b70ec96b79a7248 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Wed, 6 Mar 2019 19:54:13 +0800 Subject: [PATCH 011/115] delete the variable RAY_USR_CMAKE --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6e48d406ce28..23e77ca4ea4d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,6 @@ matrix: env: - JDK='Oracle JDK 8' - PYTHON=3.5 PYTHONWARNINGS=ignore - - RAY_USE_CMAKE=1 install: - ./ci/travis/install-dependencies.sh - export PATH="$HOME/miniconda/bin:$PATH" @@ -106,7 +105,7 @@ matrix: # Build Linux wheels. - os: linux dist: trusty - env: LINUX_WHEELS=1 PYTHONWARNINGS=ignore RAY_USE_CMAKE=1 + env: LINUX_WHEELS=1 PYTHONWARNINGS=ignore install: - ./ci/travis/install-dependencies.sh # This command should be kept in sync with ray/python/README-building-wheels.md. @@ -117,7 +116,7 @@ matrix: # Build MacOS wheels. - os: osx osx_image: xcode7 - env: MAC_WHEELS=1 PYTHONWARNINGS=ignore RAY_USE_CMAKE=1 + env: MAC_WHEELS=1 PYTHONWARNINGS=ignore install: - ./ci/travis/install-dependencies.sh # This command should be kept in sync with ray/python/README-building-wheels.md. @@ -132,7 +131,6 @@ matrix: - PYTHON=3.5 - RAY_USE_NEW_GCS=on - PYTHONWARNINGS=ignore - - RAY_USE_CMAKE=1 install: From 470518b435c7c39d33183f29147f399837e051dd Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 19:57:45 +0800 Subject: [PATCH 012/115] Update test.sh add the mvn ckeckstyle --- java/test.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/java/test.sh b/java/test.sh index 5c454cde3a16..b0af93e71052 100755 --- a/java/test.sh +++ b/java/test.sh @@ -9,6 +9,12 @@ set -x ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) $ROOT_DIR/../build.sh -l java +pushd $ROOT_DIR/../java +mvn clean install -Dmaven.test.skip +check_style=$(mvn checkstyle:check) +echo "${check_style}" +[[ ${check_style} =~ "BUILD FAILURE" ]] && exit 1 + pushd $ROOT_DIR/.. bazel build -c opt //java:all pushd $ROOT_DIR/../java/test From 2b694480b270e9e4d67cb6f552ffc077c34f60d3 Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 20:12:06 +0800 Subject: [PATCH 013/115] Update .travis.yml Revert accidentally deleted code --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 23e77ca4ea4d..eb2e2d9ec0c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -105,7 +105,7 @@ matrix: # Build Linux wheels. - os: linux dist: trusty - env: LINUX_WHEELS=1 PYTHONWARNINGS=ignore + env: LINUX_WHEELS=1 PYTHONWARNINGS=ignore RAY_USE_CMAKE=1 install: - ./ci/travis/install-dependencies.sh # This command should be kept in sync with ray/python/README-building-wheels.md. @@ -116,7 +116,7 @@ matrix: # Build MacOS wheels. - os: osx osx_image: xcode7 - env: MAC_WHEELS=1 PYTHONWARNINGS=ignore + env: MAC_WHEELS=1 PYTHONWARNINGS=ignore RAY_USE_CMAKE=1 install: - ./ci/travis/install-dependencies.sh # This command should be kept in sync with ray/python/README-building-wheels.md. @@ -131,7 +131,8 @@ matrix: - PYTHON=3.5 - RAY_USE_NEW_GCS=on - PYTHONWARNINGS=ignore - + - RAY_USE_CMAKE=1 + install: - ./ci/travis/install-bazel.sh From 2aa6ceb6cb1d619a686edd85257a90e718c9ae16 Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 20:12:37 +0800 Subject: [PATCH 014/115] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index eb2e2d9ec0c2..c6fd2abd8f29 100644 --- a/.travis.yml +++ b/.travis.yml @@ -132,7 +132,7 @@ matrix: - RAY_USE_NEW_GCS=on - PYTHONWARNINGS=ignore - RAY_USE_CMAKE=1 - + install: - ./ci/travis/install-bazel.sh From 172e8cc22554603aac940f74990c03892899d301 Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 20:16:53 +0800 Subject: [PATCH 015/115] Update testng.xml Delete useless comments --- java/testng.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/java/testng.xml b/java/testng.xml index 5ccbb15a19b8..81503edf4b85 100644 --- a/java/testng.xml +++ b/java/testng.xml @@ -1,7 +1,6 @@ - From 99eea6543614a9d71d6df67916c0fce2a52bfa2d Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 22:31:50 +0800 Subject: [PATCH 016/115] Update BUILD.bazel fix the error with target copy_java_gcs_fbs --- java/BUILD.bazel | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 170b04b04cbf..90ab4f8a53f6 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -122,7 +122,7 @@ genrule( srcs = [":copy_java_gcs_fbs"], outs = ["generate_java_gcs_fbs.out"], cmd = """ - python3 $$(pwd)/java/modify_generated_java_flatbuffers_files.py $$(pwd) + python $$(pwd)/java/modify_generated_java_flatbuffers_files.py $$(pwd) echo $$(pwd) > $@ """, local = 1, @@ -179,7 +179,8 @@ genrule( GENERATED_DIR=$$WORK_DIR/java/runtime/src/main/java/org/ray/runtime/generated mkdir -p $$GENERATED_DIR for f in $(locations //java:java_gcs_fbs); do - cp $$f $$GENERATED_DIR/; + chmod +w $$f + cp -f $$f $$GENERATED_DIR/ done echo $$(pwd) >> $@ """, From 1eec35f9418dabaf5602a4d3802fb29c5e34bce0 Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 22:40:38 +0800 Subject: [PATCH 017/115] Update ray_java.bzl add a TODO note --- bazel/ray_java.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/bazel/ray_java.bzl b/bazel/ray_java.bzl index c66feda3f227..aefbc1df7466 100644 --- a/bazel/ray_java.bzl +++ b/bazel/ray_java.bzl @@ -1,6 +1,7 @@ # The following dependencies were calculated from: # # generate_workspace --maven_project=/home/admin/ruifang.crf/code/X/arrow/java --repositories=http://mvn.dev.alipay.net:8080/artifactory/repo +# TODO: now this file is not automatically generated def generated_maven_jars(): From 3a2acd96e3216980a19af9da021ba4f3c8cdb3dc Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Thu, 7 Mar 2019 11:34:13 +0800 Subject: [PATCH 018/115] modify the java/test.sh --- java/test.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/java/test.sh b/java/test.sh index dfca1989a1b1..25fa0e6aab33 100755 --- a/java/test.sh +++ b/java/test.sh @@ -7,18 +7,21 @@ set -x ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +pushd $ROOT_DIR/../java echo "Checking code format." mvn checkstyle:check +popd echo "Compiling Java code." pushd $ROOT_DIR/.. bazel build -c opt //java:all +popd pushd $ROOT_DIR/../java/test -# test raylet +echo "Running tests under cluster mode." ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml -# test raylet under SINGLE_PROCESS mode +echo "Running tests under single-process mode." java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml set +x From ec1799131298733b6f1238d2336b3f54324f97b2 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Thu, 7 Mar 2019 11:58:01 +0800 Subject: [PATCH 019/115] delete the 'test bazel build' and add install-bazel.sh to installation steps --- .travis.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3130c6437e6b..be65c587ed31 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,17 +27,13 @@ matrix: - PYTHON=3.5 PYTHONWARNINGS=ignore - RAY_INSTALL_JAVA=1 install: + - ./ci/travis/install-bazel.sh - ./ci/travis/install-dependencies.sh - export PATH="$HOME/miniconda/bin:$PATH" - ./ci/travis/install-ray.sh script: - ./java/test.sh - # Test Bazel build - - rm -rf build - - ./ci/travis/install-bazel.sh - - bazel build ... - - os: linux dist: trusty env: LINT=1 PYTHONWARNINGS=ignore From fb938042e0ea96b939b9fba1bcebe357eced489b Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Mon, 11 Mar 2019 19:23:19 +0800 Subject: [PATCH 020/115] Add java's checkstyle with bazel and modify with some other suggestions --- BUILD.bazel | 2 +- WORKSPACE | 7 +- bazel/BUILD.plasma | 11 +- bazel/ray_java.bzl | 1578 ----------------- java/BUILD.bazel | 103 +- java/build.sh | 3 - java/dependencies.yaml | 116 ++ java/generate_deps.sh | 11 + java/test.sh | 10 +- java/third_party/BUILD | 0 java/third_party/workspace.bzl | 73 + java/tools/checkstyle/BUILD | 7 + java/tools/checkstyle/README.md | 70 + .../checkstyle/check-for-missing-targets.sh | 19 + .../checkstyle}/checkstyle-suppressions.xml | 0 java/tools/checkstyle/checkstyle.bzl | 127 ++ java/tools/checkstyle/checkstyle.sh | 17 + java/{ => tools/checkstyle}/checkstyle.xml | 0 java/tools/checkstyle/license-header.txt | 15 + java/tools/checkstyle/repo.bzl | 21 + 20 files changed, 563 insertions(+), 1627 deletions(-) delete mode 100644 bazel/ray_java.bzl delete mode 100755 java/build.sh create mode 100644 java/dependencies.yaml create mode 100644 java/generate_deps.sh create mode 100644 java/third_party/BUILD create mode 100644 java/third_party/workspace.bzl create mode 100644 java/tools/checkstyle/BUILD create mode 100644 java/tools/checkstyle/README.md create mode 100755 java/tools/checkstyle/check-for-missing-targets.sh rename java/{ => tools/checkstyle}/checkstyle-suppressions.xml (100%) create mode 100644 java/tools/checkstyle/checkstyle.bzl create mode 100755 java/tools/checkstyle/checkstyle.sh rename java/{ => tools/checkstyle}/checkstyle.xml (100%) create mode 100644 java/tools/checkstyle/license-header.txt create mode 100644 java/tools/checkstyle/repo.bzl diff --git a/BUILD.bazel b/BUILD.bazel index 04656c645528..9aa4f30c95d9 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -396,8 +396,8 @@ cc_binary( linkshared = 1, linkstatic = 0, deps = [ + "//:raylet_lib", "@plasma//:plasma_client", - "//:raylet_lib" ], ) diff --git a/WORKSPACE b/WORKSPACE index 58bf0dfb0bc8..3d8fc696e130 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,8 +1,9 @@ -workspace(name = "com_github_riselab_ray") +workspace(name = "com_github_ray_project_ray") load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository") -load("//bazel:ray_java.bzl", "generated_maven_jars") -generated_maven_jars() +load("//java:tools/checkstyle/repo.bzl", "checkstyle_repositories") + +checkstyle_repositories() git_repository( name = "com_github_nelhage_rules_boost", diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index 16960674726b..c57f3dd1990d 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -1,15 +1,15 @@ load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library") -load("@com_github_riselab_ray//bazel:ray_java.bzl", "generated_java_libraries") +load("@com_github_ray_project_ray//bazel:ray_java.bzl", "generated_java_libraries") generated_java_libraries() java_library( name = "org_apache_arrow_arrow_plasma", srcs = glob(["java/plasma/src/main/java/**/*.java"]), - visibility = ["//visibility:public"], data = [":plasma_client_java"], + visibility = ["//visibility:public"], deps = [ - "@org_slf4j_slf4j_api//jar", + "@mvnorg_slf4j_slf4j_api//jar", ], ) @@ -18,7 +18,10 @@ java_binary( srcs = ["java/plasma/src/test/java/org/apache/arrow/plasma/PlasmaClientTest.java"], main_class = "org.apache.arrow.plasma.PlasmaClientTest", visibility = ["//visibility:public"], - deps = [":org_apache_arrow_arrow_plasma", "@junit_junit//jar",], + deps = [ + ":org_apache_arrow_arrow_plasma", + "@mvnjunit_junit//jar", + ], ) cc_library( diff --git a/bazel/ray_java.bzl b/bazel/ray_java.bzl deleted file mode 100644 index aefbc1df7466..000000000000 --- a/bazel/ray_java.bzl +++ /dev/null @@ -1,1578 +0,0 @@ -# The following dependencies were calculated from: -# -# generate_workspace --maven_project=/home/admin/ruifang.crf/code/X/arrow/java --repositories=http://mvn.dev.alipay.net:8080/artifactory/repo -# TODO: now this file is not automatically generated - - -def generated_maven_jars(): - # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT - # pom.xml got requested version - # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 wanted version 1.1.2 - # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 wanted version 1.1.2 - # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "ch_qos_logback_logback_classic", - artifact = "ch.qos.logback:logback-classic:1.2.3", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "7c4f3c474fb2c041d8028740440937705ebb473a", - ) - - - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "com_google_code_findbugs_jsr305", - artifact = "com.google.code.findbugs:jsr305:3.0.2", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "25ea2e8b0c338a877313bd4672d3fe056ea78f0d", - ) - - - # io.grpc:grpc-protobuf:jar:1.14.0 - native.maven_jar( - name = "io_grpc_grpc_protobuf_lite", - artifact = "io.grpc:grpc-protobuf-lite:1.14.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "2ac8c28ca927f954eaa228a931d9c163cf3d860f", - ) - - - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT wanted version 2.10 - native.maven_jar( - name = "joda_time_joda_time", - artifact = "joda-time:joda-time:2.9.9", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "f7b520c458572890807d143670c9b24f4de90897", - ) - - - # org.mockito:mockito-core:jar:2.7.22 - native.maven_jar( - name = "net_bytebuddy_byte_buddy_agent", - artifact = "net.bytebuddy:byte-buddy-agent:1.6.11", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "0200d9c012befccd211ff91082a151257b1dc084", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 - native.maven_jar( - name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_converter_classic", - artifact = "de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:0.9.44", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "6f08f9bfd65869a51edc922432f46024d94f3ccd", - ) - - - # io.netty:netty-handler-proxy:jar:4.1.22.Final got requested version - # io.netty:netty-handler:jar:4.1.22.Final got requested version - # io.netty:netty-codec:jar:4.1.22.Final - native.maven_jar( - name = "io_netty_netty_transport", - artifact = "io.netty:netty-transport:4.1.22.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "3bd455cd9e5e5fb2e08fd9cd0acfa54c079ca989", - ) - - - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "com_carrotsearch_hppc", - artifact = "com.carrotsearch:hppc:0.7.2", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "710398361f2ae8fd594a133e3619045c16b24137", - ) - - - # io.netty:netty-codec-http2:jar:4.1.22.Final - native.maven_jar( - name = "io_netty_netty_handler", - artifact = "io.netty:netty-handler:4.1.22.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "a3a16b17d5a5ed6f784b0daba95e28d940356109", - ) - - - # io.grpc:grpc-netty:jar:1.14.0 - native.maven_jar( - name = "io_netty_netty_handler_proxy", - artifact = "io.netty:netty-handler-proxy:4.1.27.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "1a822ce7760bc6eb4937b7e448c9e081fedcc807", - ) - - - # org.apache.arrow:arrow-java-root:pom:0.12.0HOT - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "org_mockito_mockito_core", - artifact = "org.mockito:mockito-core:2.7.22", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "fcf63bc8010ca77991e3cadd8d33ad1a40603404", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-core:jar:0.9.44 got requested version - # de.huxhorn.lilith:de.huxhorn.lilith.data.logging.protobuf:jar:0.9.44 got requested version - # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 - # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 got requested version - native.maven_jar( - name = "de_huxhorn_sulky_de_huxhorn_sulky_codec", - artifact = "de.huxhorn.sulky:de.huxhorn.sulky.codec:0.9.17", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "4a2a917f1cc4b75d71aadeef4f4c178f229febc2", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.data.logging:jar:0.9.44 - native.maven_jar( - name = "de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", - artifact = "de.huxhorn.lilith:de.huxhorn.lilith.data.eventsource:0.9.44", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "618f853d75d59855c955b3a1bf0f447476f8027f", - ) - - - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "com_google_flatbuffers_flatbuffers_java", - artifact = "com.google.flatbuffers:flatbuffers-java:1.9.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "460bc8f6411768659c1ffb529592e251a808b9f2", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version - # de.huxhorn.lilith:de.huxhorn.lilith.data.logging.protobuf:jar:0.9.44 wanted version 2.5.0 - # io.grpc:grpc-protobuf:jar:1.14.0 - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT wanted version 2.5.0 - native.maven_jar( - name = "com_google_protobuf_protobuf_java", - artifact = "com.google.protobuf:protobuf-java:3.5.1", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "8c3492f7662fa1cbf8ca76a0f5eb1146f7725acd", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "io_netty_netty_tcnative_boringssl_static", - artifact = "io.netty:netty-tcnative-boringssl-static:2.0.12.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "b884be1450a7fd0854b98743836b8ccb0dfd75a4", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "com_h2database_h2", - artifact = "com.h2database:h2:1.4.196", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "dd0034398d593aa3588c6773faac429bbd9aea0e", - ) - - - # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "org_slf4j_jul_to_slf4j", - artifact = "org.slf4j:jul-to-slf4j:1.7.25", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "0af5364cd6679bfffb114f0dec8a157aaa283b76", - ) - - - native.maven_jar( - name = "net_java_dev_jna_jna", - artifact = "net.java.dev.jna:jna:4.1.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "1c12d070e602efd8021891cdd7fd18bc129372d4", - ) - - - native.maven_jar( - name = "org_mockito_mockito_all", - artifact = "org.mockito:mockito-all:1.10.19", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "539df70269cc254a58cccc5d8e43286b4a73bf30", - ) - - - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "commons_codec_commons_codec", - artifact = "commons-codec:commons-codec:1.10", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "4b95f4897fa13f2cd904aee711aeafc0c5295cd8", - ) - - - native.maven_jar( - name = "com_beust_jcommander", - artifact = "com.beust:jcommander:1.72", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "6375e521c1e11d6563d4f25a07ce124ccf8cd171", - ) - - - # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "org_slf4j_log4j_over_slf4j", - artifact = "org.slf4j:log4j-over-slf4j:1.7.25", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "a87bb47468f47ee7aabbd54f93e133d4215769c3", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "commons_cli_commons_cli", - artifact = "commons-cli:commons-cli:1.4", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "c51c00206bb913cd8612b24abd9fa98ae89719b1", - ) - - - # io.grpc:grpc-core:jar:1.14.0 - native.maven_jar( - name = "com_google_errorprone_error_prone_annotations", - artifact = "com.google.errorprone:error_prone_annotations:2.1.2", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "6dcc08f90f678ac33e5ef78c3c752b6f59e63e0c", - ) - - - # io.netty:netty-transport:jar:4.1.22.Final - native.maven_jar( - name = "io_netty_netty_resolver", - artifact = "io.netty:netty-resolver:4.1.22.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "b5484d17a97cb57b07d2a1ac092c249e47234c17", - ) - - - # io.grpc:grpc-core:jar:1.14.0 - native.maven_jar( - name = "io_opencensus_opencensus_contrib_grpc_metrics", - artifact = "io.opencensus:opencensus-contrib-grpc-metrics:0.12.3", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "a4c7ff238a91b901c8b459889b6d0d7a9d889b4d", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "io_grpc_grpc_stub", - artifact = "io.grpc:grpc-stub:1.14.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "74bfe83c0dc69bf903fff8df3568cbeb8b387d35", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "io_grpc_grpc_protobuf", - artifact = "io.grpc:grpc-protobuf:1.14.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "d429fdc2e0d288b34ea7588bb762eb458f385bd5", - ) - - - # io.netty:netty-handler-proxy:jar:4.1.22.Final - native.maven_jar( - name = "io_netty_netty_codec_socks", - artifact = "io.netty:netty-codec-socks:4.1.22.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "d077b39da2dedc5dc5db50a44e5f4c30353e86f3", - ) - - - # io.netty:netty-codec-http:jar:4.1.22.Final - # io.netty:netty-handler:jar:4.1.22.Final got requested version - # io.netty:netty-codec-socks:jar:4.1.22.Final got requested version - native.maven_jar( - name = "io_netty_netty_codec", - artifact = "io.netty:netty-codec:4.1.22.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "239c0af275952e70bb4adf7cf8c03d88ddc394c9", - ) - - - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version - # io.netty:netty-transport:jar:4.1.22.Final got requested version - # io.netty:netty-handler:jar:4.1.22.Final got requested version - native.maven_jar( - name = "io_netty_netty_buffer", - artifact = "io.netty:netty-buffer:4.1.22.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "15e964a2095031364f534a6e21977f5ee9ca32a9", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 - native.maven_jar( - name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_core", - artifact = "de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-core:0.9.44", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "636cc11e0e09d363af87766550c1deef8ab0c5bf", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "io_grpc_grpc_netty", - artifact = "io.grpc:grpc-netty:1.14.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "21c6edadd45b6869384f8aa0df1663d62c503617", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-core:jar:0.9.44 wanted version 1.1.2 - # ch.qos.logback:logback-classic:jar:1.2.3 - native.maven_jar( - name = "ch_qos_logback_logback_core", - artifact = "ch.qos.logback:logback-core:1.2.3", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "864344400c3d4d92dfeb0a305dc87d953677c03c", - ) - - - # org.mockito:mockito-core:jar:2.7.22 - native.maven_jar( - name = "net_bytebuddy_byte_buddy", - artifact = "net.bytebuddy:byte-buddy:1.6.11", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "8a8f9409e27f1d62c909c7eef2aa7b3a580b4901", - ) - - - # io.opencensus:opencensus-contrib-grpc-metrics:jar:0.12.3 got requested version - # io.grpc:grpc-core:jar:1.14.0 - native.maven_jar( - name = "io_opencensus_opencensus_api", - artifact = "io.opencensus:opencensus-api:0.12.3", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "743f074095f29aa985517299545e72cc99c87de0", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-core:jar:0.9.44 - native.maven_jar( - name = "de_huxhorn_lilith_de_huxhorn_lilith_sender", - artifact = "de.huxhorn.lilith:de.huxhorn.lilith.sender:0.9.44", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "fdeb2f1aaba42cad76c460ac4fd7072b8642520f", - ) - - - native.maven_jar( - name = "org_javassist_javassist", - artifact = "org.javassist:javassist:3.19.0-GA", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "50120f69224dd8684b445a6f3a5b08fe9b5c60f6", - ) - - - # org.mockito:mockito-core:jar:2.7.22 - native.maven_jar( - name = "org_objenesis_objenesis", - artifact = "org.objenesis:objenesis:2.5", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "612ecb799912ccf77cba9b3ed8c813da086076e9", - ) - - - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT got requested version - # io.netty:netty-resolver:jar:4.1.22.Final got requested version - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT got requested version - # io.netty:netty-buffer:jar:4.1.22.Final - native.maven_jar( - name = "io_netty_netty_common", - artifact = "io.netty:netty-common:4.1.22.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "56ff4deca53fc791ed59ac2b72eb6718714a4de9", - ) - - - # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT wanted version 4.12 - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "junit_junit", - artifact = "junit:junit:4.11", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "4e031bb61df09069aeb2bffb4019e7a5034a4ee0", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 - native.maven_jar( - name = "de_huxhorn_lilith_de_huxhorn_lilith_data_logging_protobuf", - artifact = "de.huxhorn.lilith:de.huxhorn.lilith.data.logging.protobuf:0.9.44", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "56a037b365c7ff9b1c348e6d663fd3d778e2b0be", - ) - - - # de.huxhorn.sulky:de.huxhorn.sulky.codec:jar:0.9.17 - # de.huxhorn.lilith:de.huxhorn.lilith.sender:jar:0.9.44 got requested version - native.maven_jar( - name = "de_huxhorn_sulky_de_huxhorn_sulky_io", - artifact = "de.huxhorn.sulky:de.huxhorn.sulky.io:0.9.17", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "6baf1e17f80ba40f9a4b4b97235e72488047db02", - ) - - - native.maven_jar( - name = "log4j_log4j", - artifact = "log4j:log4j:1.2.17", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "5af35056b4d257e4b64b9e8069c0746e8b08629f", - ) - - - native.maven_jar( - name = "org_slf4j_slf4j_log4j12", - artifact = "org.slf4j:slf4j-log4j12:1.7.25", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "110cefe2df103412849d72ef7a67e4e91e4266b4", - ) - - - # org.slf4j:jcl-over-slf4j:jar:1.7.25 got requested version - # org.slf4j:jul-to-slf4j:jar:1.7.25 - # de.huxhorn.lilith:de.huxhorn.lilith.sender:jar:0.9.44 wanted version 1.7.7 - # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 wanted version 1.7.7 - # org.slf4j:log4j-over-slf4j:jar:1.7.25 got requested version - # ch.qos.logback:logback-classic:jar:1.2.3 got requested version - # de.huxhorn.sulky:de.huxhorn.sulky.codec:jar:0.9.17 wanted version 1.7.7 - # de.huxhorn.sulky:de.huxhorn.sulky.formatting:jar:0.9.17 wanted version 1.7.7 - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "org_slf4j_slf4j_api", - artifact = "org.slf4j:slf4j-api:1.7.25", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "da76ca59f6a57ee3102f8f9bd9cee742973efa8a", - ) - - - native.maven_jar( - name = "com_typesafe_config", - artifact = "com.typesafe:config:1.3.2", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "d6ac0ce079f114adce620f2360c92a70b2cb36dc", - ) - - - native.maven_jar( - name = "org_apache_commons_commons_lang3", - artifact = "org.apache.commons:commons-lang3:3.4", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "5fe28b9518e58819180a43a850fbc0dd24b7c050", - ) - - - # io.netty:netty-handler-proxy:jar:4.1.22.Final got requested version - # io.netty:netty-codec-http2:jar:4.1.22.Final - native.maven_jar( - name = "io_netty_netty_codec_http", - artifact = "io.netty:netty-codec-http:4.1.22.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "3805f3ca0d57630200defc7f9bb6ed3382dcb10b", - ) - - - # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOTi - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_classic", - artifact = "de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:0.9.44", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "1ac6ffe5fc63f7900f8af54b0dfebf2508b41db7", - ) - - - native.maven_jar( - name = "commons_collections_commons_collections", - artifact = "commons-collections:commons-collections:3.2.2", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "8ad72fe39fa8c91eaaf12aadb21e0c3661fe26d5", - ) - - - # junit:junit:jar:4.11 - native.maven_jar( - name = "org_hamcrest_hamcrest_core", - artifact = "org.hamcrest:hamcrest-core:1.3", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "42a25dc3219429f0e5d060061f71acb49bf010a0", - ) - - - # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "org_jmockit_jmockit", - artifact = "org.jmockit:jmockit:1.33", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "8a2746af90706d8d68f3919e1a87d5e47b8c94d7", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.data.logging:jar:0.9.44 - native.maven_jar( - name = "de_huxhorn_sulky_de_huxhorn_sulky_formatting", - artifact = "de.huxhorn.sulky:de.huxhorn.sulky.formatting:0.9.17", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "0501b143ee18ca9e7807438cc1b0acfa144f657f", - ) - - - # io.grpc:grpc-netty:jar:1.14.0 - native.maven_jar( - name = "io_netty_netty_codec_http2", - artifact = "io.netty:netty-codec-http2:4.1.27.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "3769790a2033667d663f9a526d5b63cfecdbdf4e", - ) - - - native.maven_jar( - name = "net_lingala_zip4j_zip4j", - artifact = "net.lingala.zip4j:zip4j:1.3.2", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "4ba84e98ee017b74cb52f45962f929a221f3074c", - ) - - - native.maven_jar( - name = "org_testng_testng", - artifact = "org.testng:testng:6.9.9", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "1bf509349476d6a48978cc2b04af9caa907781ab", - ) - - native.maven_jar( - name = "org_ini4j_ini4j", - artifact = "org.ini4j:ini4j:0.5.4", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "4a3ee4146a90c619b20977d65951825f5675b560", - ) - - - native.maven_jar( - name = "org_ow2_asm_asm", - artifact = "org.ow2.asm:asm:6.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "bc6fa6b19424bb9592fe43bbc20178f92d403105", - ) - - - # io.grpc:grpc-core:jar:1.14.0 - native.maven_jar( - name = "com_google_code_gson_gson", - artifact = "com.google.code.gson:gson:2.7",#2.8.5 - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "751f548c85fa49f330cecbb1875893f971b33c4e", - ) - - - # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "org_slf4j_jcl_over_slf4j", - artifact = "org.slf4j:jcl-over-slf4j:1.7.25", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "f8c32b13ff142a513eeb5b6330b1588dcb2c0461", - ) - - - # pom.xml got requested version - # com.fasterxml.jackson.core:jackson-databind:bundle:2.7.9 wanted version 2.7.0 - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version - native.maven_jar( - name = "com_fasterxml_jackson_core_jackson_annotations", - artifact = "com.fasterxml.jackson.core:jackson-annotations:2.7.9", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "eb356e825cb73da42f7c902a3fe0276fe32b26c8", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.data.logging.protobuf:jar:0.9.44 got requested version - # de.huxhorn.lilith:de.huxhorn.lilith.logback.classic:jar:0.9.44 - # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 got requested version - native.maven_jar( - name = "de_huxhorn_lilith_de_huxhorn_lilith_data_logging", - artifact = "de.huxhorn.lilith:de.huxhorn.lilith.data.logging:0.9.44", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "3574a72628f59d78ff17210f10de057188043a4c", - ) - - - # io.grpc:grpc-core:jar:1.14.0 - native.maven_jar( - name = "io_grpc_grpc_context", - artifact = "io.grpc:grpc-context:1.14.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "77252b5f926875891aaae5629e6ab2ef968cd6c6", - ) - - - # io.grpc:grpc-protobuf:jar:1.14.0 - native.maven_jar( - name = "com_google_api_grpc_proto_google_common_protos", - artifact = "com.google.api.grpc:proto-google-common-protos:1.0.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "86f070507e28b930e50d218ee5b6788ef0dd05e6", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 - native.maven_jar( - name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_classic", - artifact = "de.huxhorn.lilith:de.huxhorn.lilith.logback.classic:0.9.44", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "f84da242a1988240222f6a06b511a18ff9c64344", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "com_fasterxml_jackson_core_jackson_databind", - artifact = "com.fasterxml.jackson.core:jackson-databind:2.7.9", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "a4c0b14c7dd85bdf4d25da074e90a10fa4b9b88b", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "com_fasterxml_jackson_dataformat_jackson_dataformat_yaml", - artifact = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.7.9", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "163a1bd63ace592d2a449023a1c07056b885ff12", - ) - - - native.maven_jar( - name = "com_github_davidmoten_flatbuffers_java", - artifact = "com.github.davidmoten:flatbuffers-java:1.9.0.1", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "0eaa1b27095bb5127879458bc2ee1b78e00d4a20", - ) - - - native.maven_jar( - name = "redis_clients_jedis", - artifact = "redis.clients:jedis:2.8.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "a9486e200b2195f31ce9b8779a70a5e2450a73ba", - ) - - - native.maven_jar( - name = "org_apache_commons_commons_pool2", - artifact = "org.apache.commons:commons-pool2:2.3", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "62a559a025fd890c30364296ece14643ba9c8c5b", - ) - - - native.maven_jar( - name = "commons_io_commons_io", - artifact = "commons-io:commons-io:2.5", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "2852e6e05fbb95076fc091f6d1780f1f8fe35e0f", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - # io.grpc:grpc-protobuf:jar:1.14.0 got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version - # io.grpc:grpc-protobuf-lite:jar:1.14.0 got requested version - # io.grpc:grpc-core:jar:1.14.0 - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT wanted version 23.0 - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "com_google_guava_guava", - artifact = "com.google.guava:guava:20.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "89507701249388e1ed5ddcf8c41f4ce1be7831ef", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 - native.maven_jar( - name = "de_huxhorn_lilith_de_huxhorn_lilith_data_converter", - artifact = "de.huxhorn.lilith:de.huxhorn.lilith.data.converter:0.9.44", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "31e4994f9bc768c43491c3c0d12237ae478d21e3", - ) - - - native.maven_jar( - name = "de_ruedigermoeller_fst", - artifact = "de.ruedigermoeller:fst:2.47", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "281d390ebed24a3621d3053825affc13d0eead8b", - ) - - - # pom.xml got requested version - # com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:bundle:2.7.9 got requested version - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # com.fasterxml.jackson.core:jackson-databind:bundle:2.7.9 got requested version - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "com_fasterxml_jackson_core_jackson_core", - artifact = "com.fasterxml.jackson.core:jackson-core:2.7.9", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "09b530cec4fd2eb841ab8e79f19fc7cf0ec487b2", - ) - - - # pom.xml got requested version - # io.grpc:grpc-protobuf:jar:1.14.0 got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version - # io.grpc:grpc-protobuf-lite:jar:1.14.0 got requested version - # io.grpc:grpc-stub:jar:1.14.0 got requested version - # io.grpc:grpc-netty:jar:1.14.0 - native.maven_jar( - name = "io_grpc_grpc_core", - artifact = "io.grpc:grpc-core:1.14.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "526e5be291c96e248789d769c108a084febda07f", - ) - - - # com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:bundle:2.7.9 - native.maven_jar( - name = "org_yaml_snakeyaml", - artifact = "org.yaml:snakeyaml:1.15", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "3b132bea69e8ee099f416044970997bde80f4ea6", - ) - - - - -def generated_java_libraries(): - native.java_library( - name = "log4j_log4j", - visibility = ["//visibility:public"], - exports = ["@log4j_log4j//jar"], - ) - - - native.java_library( - name = "org_slf4j_slf4j_log4j12", - visibility = ["//visibility:public"], - exports = ["@org_slf4j_slf4j_log4j12//jar"], - runtime_deps = [ - ":log4j_log4j", - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "com_typesafe_config", - visibility = ["//visibility:public"], - exports = ["@com_typesafe_config//jar"], - ) - - - native.java_library( - name = "org_apache_commons_commons_lang3", - visibility = ["//visibility:public"], - exports = ["@org_apache_commons_commons_lang3//jar"], - ) - - - native.java_library( - name = "com_google_code_findbugs_jsr305", - visibility = ["//visibility:public"], - exports = ["@com_google_code_findbugs_jsr305//jar"], - ) - - - native.java_library( - name = "io_grpc_grpc_core", - visibility = ["//visibility:public"], - exports = ["@io_grpc_grpc_core//jar"], - runtime_deps = [ - ":com_google_code_findbugs_jsr305", - ":com_google_code_gson_gson", - ":com_google_errorprone_error_prone_annotations", - ":com_google_guava_guava", - ":io_grpc_grpc_context", - ":io_opencensus_opencensus_api", - ":io_opencensus_opencensus_contrib_grpc_metrics", - ], - ) - - - native.java_library( - name = "ch_qos_logback_logback_classic", - visibility = ["//visibility:public"], - exports = ["@ch_qos_logback_logback_classic//jar"], - runtime_deps = [ - ":ch_qos_logback_logback_core", - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "joda_time_joda_time", - visibility = ["//visibility:public"], - exports = ["@joda_time_joda_time//jar"], - ) - - - native.java_library( - name = "net_bytebuddy_byte_buddy_agent", - visibility = ["//visibility:public"], - exports = ["@net_bytebuddy_byte_buddy_agent//jar"], - ) - - - native.java_library( - name = "io_grpc_grpc_protobuf_lite", - visibility = ["//visibility:public"], - exports = ["@io_grpc_grpc_protobuf_lite//jar"], - runtime_deps = [ - ":com_google_guava_guava", - ":io_grpc_grpc_core", - ], - ) - - - native.java_library( - name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_converter_classic", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_logback_converter_classic//jar"], - runtime_deps = [ - ":ch_qos_logback_logback_classic", - ":de_huxhorn_lilith_de_huxhorn_lilith_data_converter", - ":de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", - ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging", - ":de_huxhorn_lilith_de_huxhorn_lilith_logback_classic", - ":de_huxhorn_sulky_de_huxhorn_sulky_codec", - ":de_huxhorn_sulky_de_huxhorn_sulky_formatting", - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "io_netty_netty_transport", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_transport//jar"], - runtime_deps = [ - ":io_netty_netty_buffer", - ":io_netty_netty_common", - ":io_netty_netty_resolver", - ], - ) - - - native.java_library( - name = "com_carrotsearch_hppc", - visibility = ["//visibility:public"], - exports = ["@com_carrotsearch_hppc//jar"], - ) - - - native.java_library( - name = "org_mockito_mockito_core", - visibility = ["//visibility:public"], - exports = ["@org_mockito_mockito_core//jar"], - runtime_deps = [ - ":net_bytebuddy_byte_buddy", - ":net_bytebuddy_byte_buddy_agent", - ":org_objenesis_objenesis", - ], - ) - - - native.java_library( - name = "io_netty_netty_handler_proxy", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_handler_proxy//jar"], - runtime_deps = [ - ":io_netty_netty_codec", - ":io_netty_netty_codec_http", - ":io_netty_netty_codec_socks", - ":io_netty_netty_transport", - ], - ) - - - native.java_library( - name = "de_huxhorn_sulky_de_huxhorn_sulky_codec", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_sulky_de_huxhorn_sulky_codec//jar"], - runtime_deps = [ - ":de_huxhorn_sulky_de_huxhorn_sulky_io", - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource//jar"], - ) - - - native.java_library( - name = "com_google_flatbuffers_flatbuffers_java", - visibility = ["//visibility:public"], - exports = ["@com_google_flatbuffers_flatbuffers_java//jar"], - ) - - - native.java_library( - name = "com_google_protobuf_protobuf_java", - visibility = ["//visibility:public"], - exports = ["@com_google_protobuf_protobuf_java//jar"], - ) - - - native.java_library( - name = "com_h2database_h2", - visibility = ["//visibility:public"], - exports = ["@com_h2database_h2//jar"], - ) - - - native.java_library( - name = "org_slf4j_jul_to_slf4j", - visibility = ["//visibility:public"], - exports = ["@org_slf4j_jul_to_slf4j//jar"], - runtime_deps = [ - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "commons_codec_commons_codec", - visibility = ["//visibility:public"], - exports = ["@commons_codec_commons_codec//jar"], - ) - - - native.java_library( - name = "org_slf4j_log4j_over_slf4j", - visibility = ["//visibility:public"], - exports = ["@org_slf4j_log4j_over_slf4j//jar"], - runtime_deps = [ - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "commons_cli_commons_cli", - visibility = ["//visibility:public"], - exports = ["@commons_cli_commons_cli//jar"], - ) - - - native.java_library( - name = "io_netty_netty_tcnative_boringssl_static", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_tcnative_boringssl_static//jar"], - ) - - - native.java_library( - name = "io_grpc_grpc_stub", - visibility = ["//visibility:public"], - exports = ["@io_grpc_grpc_stub//jar"], - runtime_deps = [ - ":io_grpc_grpc_core", - ], - ) - - - native.java_library( - name = "io_grpc_grpc_protobuf", - visibility = ["//visibility:public"], - exports = ["@io_grpc_grpc_protobuf//jar"], - runtime_deps = [ - ":com_google_api_grpc_proto_google_common_protos", - ":com_google_guava_guava", - ":com_google_protobuf_protobuf_java", - ":io_grpc_grpc_core", - ":io_grpc_grpc_protobuf_lite", - ], - ) - - - native.java_library( - name = "io_netty_netty_codec_socks", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_codec_socks//jar"], - runtime_deps = [ - ":io_netty_netty_codec", - ], - ) - - - native.java_library( - name = "io_netty_netty_codec", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_codec//jar"], - runtime_deps = [ - ":io_netty_netty_buffer", - ":io_netty_netty_common", - ":io_netty_netty_resolver", - ":io_netty_netty_transport", - ], - ) - - - native.java_library( - name = "io_netty_netty_buffer", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_buffer//jar"], - runtime_deps = [ - ":io_netty_netty_common", - ], - ) - - - native.java_library( - name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_core", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_core//jar"], - runtime_deps = [ - ":ch_qos_logback_logback_core", - ":de_huxhorn_lilith_de_huxhorn_lilith_sender", - ":de_huxhorn_sulky_de_huxhorn_sulky_codec", - ":de_huxhorn_sulky_de_huxhorn_sulky_io", - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "io_grpc_grpc_netty", - visibility = ["//visibility:public"], - exports = ["@io_grpc_grpc_netty//jar"], - runtime_deps = [ - ":com_google_code_findbugs_jsr305", - ":com_google_code_gson_gson", - ":com_google_errorprone_error_prone_annotations", - ":com_google_guava_guava", - ":io_grpc_grpc_context", - ":io_grpc_grpc_core", - ":io_netty_netty_buffer", - ":io_netty_netty_codec", - ":io_netty_netty_codec_http", - ":io_netty_netty_codec_http2", - ":io_netty_netty_codec_socks", - ":io_netty_netty_common", - ":io_netty_netty_handler", - ":io_netty_netty_handler_proxy", - ":io_netty_netty_resolver", - ":io_netty_netty_transport", - ":io_opencensus_opencensus_api", - ":io_opencensus_opencensus_contrib_grpc_metrics", - ], - ) - - - native.java_library( - name = "ch_qos_logback_logback_core", - visibility = ["//visibility:public"], - exports = ["@ch_qos_logback_logback_core//jar"], - ) - - - native.java_library( - name = "net_bytebuddy_byte_buddy", - visibility = ["//visibility:public"], - exports = ["@net_bytebuddy_byte_buddy//jar"], - ) - - - native.java_library( - name = "io_opencensus_opencensus_api", - visibility = ["//visibility:public"], - exports = ["@io_opencensus_opencensus_api//jar"], - ) - - - native.java_library( - name = "de_huxhorn_lilith_de_huxhorn_lilith_sender", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_sender//jar"], - runtime_deps = [ - ":de_huxhorn_sulky_de_huxhorn_sulky_io", - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "com_google_errorprone_error_prone_annotations", - visibility = ["//visibility:public"], - exports = ["@com_google_errorprone_error_prone_annotations//jar"], - ) - - - native.java_library( - name = "io_netty_netty_resolver", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_resolver//jar"], - runtime_deps = [ - ":io_netty_netty_common", - ], - ) - - - native.java_library( - name = "io_opencensus_opencensus_contrib_grpc_metrics", - visibility = ["//visibility:public"], - exports = ["@io_opencensus_opencensus_contrib_grpc_metrics//jar"], - runtime_deps = [ - ":io_opencensus_opencensus_api", - ], - ) - - - native.java_library( - name = "org_javassist_javassist", - visibility = ["//visibility:public"], - exports = ["@org_javassist_javassist//jar"], - ) - - - native.java_library( - name = "org_objenesis_objenesis", - visibility = ["//visibility:public"], - exports = ["@org_objenesis_objenesis//jar"], - ) - - - native.java_library( - name = "io_netty_netty_common", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_common//jar"], - ) - - - native.java_library( - name = "junit_junit", - visibility = ["//visibility:public"], - exports = ["@junit_junit//jar"], - runtime_deps = [ - ":org_hamcrest_hamcrest_core", - ], - ) - - - native.java_library( - name = "de_huxhorn_lilith_de_huxhorn_lilith_data_logging_protobuf", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_data_logging_protobuf//jar"], - runtime_deps = [ - ":com_google_protobuf_protobuf_java", - ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging", - ":de_huxhorn_sulky_de_huxhorn_sulky_codec", - ], - ) - - - native.java_library( - name = "de_huxhorn_sulky_de_huxhorn_sulky_io", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_sulky_de_huxhorn_sulky_io//jar"], - ) - - - native.java_library( - name = "org_slf4j_slf4j_api", - visibility = ["//visibility:public"], - exports = ["@org_slf4j_slf4j_api//jar"], - ) - - - native.java_library( - name = "io_netty_netty_codec_http", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_codec_http//jar"], - runtime_deps = [ - ":io_netty_netty_buffer", - ":io_netty_netty_codec", - ":io_netty_netty_common", - ":io_netty_netty_resolver", - ":io_netty_netty_transport", - ], - ) - - - native.java_library( - name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_classic", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_classic//jar"], - runtime_deps = [ - ":ch_qos_logback_logback_classic", - ":ch_qos_logback_logback_core", - ":com_google_protobuf_protobuf_java", - ":de_huxhorn_lilith_de_huxhorn_lilith_data_converter", - ":de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", - ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging", - ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging_protobuf", - ":de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_core", - ":de_huxhorn_lilith_de_huxhorn_lilith_logback_classic", - ":de_huxhorn_lilith_de_huxhorn_lilith_logback_converter_classic", - ":de_huxhorn_lilith_de_huxhorn_lilith_sender", - ":de_huxhorn_sulky_de_huxhorn_sulky_codec", - ":de_huxhorn_sulky_de_huxhorn_sulky_formatting", - ":de_huxhorn_sulky_de_huxhorn_sulky_io", - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "org_hamcrest_hamcrest_core", - visibility = ["//visibility:public"], - exports = ["@org_hamcrest_hamcrest_core//jar"], - ) - - - native.java_library( - name = "io_netty_netty_handler", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_handler//jar"], - runtime_deps = [ - ":io_netty_netty_buffer", - ":io_netty_netty_codec", - ":io_netty_netty_transport", - ], - ) - - - native.java_library( - name = "org_jmockit_jmockit", - visibility = ["//visibility:public"], - exports = ["@org_jmockit_jmockit//jar"], - ) - - - native.java_library( - name = "de_huxhorn_sulky_de_huxhorn_sulky_formatting", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_sulky_de_huxhorn_sulky_formatting//jar"], - runtime_deps = [ - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "io_netty_netty_codec_http2", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_codec_http2//jar"], - runtime_deps = [ - ":io_netty_netty_buffer", - ":io_netty_netty_codec", - ":io_netty_netty_codec_http", - ":io_netty_netty_common", - ":io_netty_netty_handler", - ":io_netty_netty_resolver", - ":io_netty_netty_transport", - ], - ) - - - native.java_library( - name = "net_lingala_zip4j_zip4j", - visibility = ["//visibility:public"], - exports = ["@net_lingala_zip4j_zip4j//jar"], - ) - - - native.java_library( - name = "org_ini4j_ini4j", - visibility = ["//visibility:public"], - exports = ["@org_ini4j_ini4j//jar"], - ) - - - native.java_library( - name = "org_ow2_asm_asm", - visibility = ["//visibility:public"], - exports = ["@org_ow2_asm_asm//jar"], - ) - - - native.java_library( - name = "com_google_code_gson_gson", - visibility = ["//visibility:public"], - exports = ["@com_google_code_gson_gson//jar"], - ) - - - native.java_library( - name = "org_slf4j_jcl_over_slf4j", - visibility = ["//visibility:public"], - exports = ["@org_slf4j_jcl_over_slf4j//jar"], - runtime_deps = [ - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "com_fasterxml_jackson_core_jackson_annotations", - visibility = ["//visibility:public"], - exports = ["@com_fasterxml_jackson_core_jackson_annotations//jar"], - ) - - - native.java_library( - name = "de_huxhorn_lilith_de_huxhorn_lilith_data_logging", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_data_logging//jar"], - runtime_deps = [ - ":de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", - ":de_huxhorn_sulky_de_huxhorn_sulky_formatting", - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_classic", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_logback_classic//jar"], - runtime_deps = [ - ":de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", - ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging", - ":de_huxhorn_sulky_de_huxhorn_sulky_formatting", - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "com_fasterxml_jackson_core_jackson_databind", - visibility = ["//visibility:public"], - exports = ["@com_fasterxml_jackson_core_jackson_databind//jar"], - runtime_deps = [ - ":com_fasterxml_jackson_core_jackson_annotations", - ":com_fasterxml_jackson_core_jackson_core", - ], - ) - - - native.java_library( - name = "com_fasterxml_jackson_dataformat_jackson_dataformat_yaml", - visibility = ["//visibility:public"], - exports = ["@com_fasterxml_jackson_dataformat_jackson_dataformat_yaml//jar"], - runtime_deps = [ - ":com_fasterxml_jackson_core_jackson_core", - ":org_yaml_snakeyaml", - ], - ) - - - native.java_library( - name = "io_grpc_grpc_context", - visibility = ["//visibility:public"], - exports = ["@io_grpc_grpc_context//jar"], - ) - - - native.java_library( - name = "com_google_api_grpc_proto_google_common_protos", - visibility = ["//visibility:public"], - exports = ["@com_google_api_grpc_proto_google_common_protos//jar"], - ) - - - native.java_library( - name = "com_github_davidmoten_flatbuffers_java", - visibility = ["//visibility:public"], - exports = ["@com_github_davidmoten_flatbuffers_java//jar"], - ) - - - native.java_library( - name = "redis_clients_jedis", - visibility = ["//visibility:public"], - exports = ["@redis_clients_jedis//jar"], - runtime_deps = [ - ":org_apache_commons_commons_pool2", - ], - ) - - - native.java_library( - name = "org_apache_commons_commons_pool2", - visibility = ["//visibility:public"], - exports = ["@org_apache_commons_commons_pool2//jar"], - ) - - - native.java_library( - name = "com_google_guava_guava", - visibility = ["//visibility:public"], - exports = ["@com_google_guava_guava//jar"], - ) - - - native.java_library( - name = "de_huxhorn_lilith_de_huxhorn_lilith_data_converter", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_data_converter//jar"], - ) - - - native.java_library( - name = "de_ruedigermoeller_fst", - visibility = ["//visibility:public"], - exports = ["@de_ruedigermoeller_fst//jar"], - runtime_deps = [ - ":com_fasterxml_jackson_core_jackson_core", - ":org_javassist_javassist", - ":org_objenesis_objenesis", - ], - ) - - - native.java_library( - name = "com_fasterxml_jackson_core_jackson_core", - visibility = ["//visibility:public"], - exports = ["@com_fasterxml_jackson_core_jackson_core//jar"], - ) - - - native.java_library( - name = "org_yaml_snakeyaml", - visibility = ["//visibility:public"], - exports = ["@org_yaml_snakeyaml//jar"], - ) - - - native.java_library( - name = "net_java_dev_jna_jna", - visibility = ["//visibility:public"], - exports = ["@net_java_dev_jna_jna//jar"], - ) - - - native.java_library( - name = "org_mockito_mockito_all", - visibility = ["//visibility:public"], - exports = ["@org_mockito_mockito_all//jar"], - ) - - - native.java_library( - name = "com_beust_jcommander", - visibility = ["//visibility:public"], - exports = ["@com_beust_jcommander//jar"], - ) - - native.java_library( - name = "org_testng_testng", - visibility = ["//visibility:public"], - exports = ["@org_testng_testng//jar"], - ) diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 90ab4f8a53f6..43eec664b94f 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -1,8 +1,11 @@ - +load("//java/tools/checkstyle:checkstyle.bzl", "checkstyle_test") load("//bazel:ray.bzl", "flatbuffer_java_library") -load("//bazel:ray_java.bzl", "generated_java_libraries") -generated_java_libraries() +java_binary( + name = "bazel_deps", + main_class = "com.github.johnynek.bazel_deps.ParseProject", + runtime_deps = ["@bazel_deps//jar"], +) java_import( name = "org_ray_ray_java_root", @@ -33,11 +36,20 @@ java_library( name = "org_ray_ray_api", srcs = glob(["api/src/main/java/**/*.java"]), deps = [ - "@org_slf4j_slf4j_log4j12//jar", - "@log4j_log4j//jar", + "@mvnorg_slf4j_slf4j_log4j12//jar", + "@mvnlog4j_log4j//jar", + "@mvnjavax_xml_bind_jaxb_api//jar", + "@mvncom_sun_xml_bind_jaxb_core//jar", + "@mvncom_sun_xml_bind_jaxb_impl//jar", ], ) +checkstyle_test( + name = "org_ray_ray_api-checkstyle", + target = ":org_ray_ray_api", + size = "small", +) + java_library( name = "org_ray_ray_runtime", srcs = glob(["runtime/src/main/java/**/*.java"]), @@ -46,27 +58,33 @@ java_library( data = ["//:raylet_library_java", ":generate_java_gcs_fbs"], deps = [ ":org_ray_ray_api", - "@com_typesafe_config//jar", - "@org_apache_commons_commons_lang3//jar", - "@de_ruedigermoeller_fst//jar", - "@com_github_davidmoten_flatbuffers_java//jar", - "@redis_clients_jedis//jar", "@plasma//:org_apache_arrow_arrow_plasma", - "@commons_io_commons_io//jar", - "@com_google_guava_guava//jar", - "@net_lingala_zip4j_zip4j//jar", - "@org_slf4j_slf4j_log4j12//jar", - "@org_slf4j_slf4j_api//jar", - "@org_ini4j_ini4j//jar", - "@org_ow2_asm_asm//jar", - "@com_google_code_gson_gson//jar", - "@com_fasterxml_jackson_core_jackson_core//jar", - "@org_apache_commons_commons_pool2//jar", - "@org_javassist_javassist//jar", - "@org_objenesis_objenesis//jar", + "@mvnorg_slf4j_slf4j_api//jar", + #"@mvncom_google_code_gson_gson//jar", + "@mvncom_fasterxml_jackson_core_jackson_core//jar", + "@mvnorg_apache_commons_commons_pool2//jar", + "@mvnorg_javassist_javassist//jar", + "@mvnorg_objenesis_objenesis//jar", + "@mvncom_typesafe_config//jar", + "@mvnorg_apache_commons_commons_lang3//jar", + "@mvnde_ruedigermoeller_fst//jar", + "@mvncom_github_davidmoten_flatbuffers_java//jar", + "@mvnredis_clients_jedis//jar", + "@mvncommons_io_commons_io//jar", + "@mvncom_google_guava_guava//jar", + "@mvnnet_lingala_zip4j_zip4j//jar", + "@mvnorg_slf4j_slf4j_log4j12//jar", + "@mvnorg_ini4j_ini4j//jar", + "@mvnorg_ow2_asm_asm//jar", ], ) +checkstyle_test( + name = "org_ray_ray_runtime-checkstyle", + target = ":org_ray_ray_runtime", + size = "small", +) + java_binary( name = "org_ray_ray_tutorial", srcs = glob(["tutorial/src/main/java/**/*.java"]), @@ -75,40 +93,59 @@ java_binary( deps = [ ":org_ray_ray_api", ":org_ray_ray_runtime", - "@com_google_guava_guava//jar", + "@mvncom_google_guava_guava//jar", ], ) +checkstyle_test( + name = "org_ray_ray_tutorial-checkstyle", + target = ":org_ray_ray_tutorial", + size = "small", +) + java_library( name = "org_ray_ray_test", srcs = glob(["test/src/main/java/**/*.java"]), deps = [ - "@com_google_guava_guava//jar", - "@commons_collections_commons_collections//jar", ":org_ray_ray_api", ":org_ray_ray_runtime", - "@org_apache_commons_commons_lang3//jar", - "@org_slf4j_slf4j_api//jar", + "@mvnorg_apache_commons_commons_lang3//jar", + "@mvnorg_slf4j_slf4j_api//jar", "@plasma//:org_apache_arrow_arrow_plasma", - "@org_testng_testng//jar", + "@mvnorg_testng_testng//jar", + "@mvncommons_collections_commons_collections//jar", + "@mvncom_google_guava_guava//jar", ], ) +checkstyle_test( + name = "org_ray_ray_test-checkstyle", + target = ":org_ray_ray_test", + size = "small", +) + java_library( name = "org_ray_ray_runtime_test", srcs = glob(["runtime/src/test/java/**/*.java"]), deps = [ ":org_ray_ray_api", ":org_ray_ray_runtime", - "@org_testng_testng//jar", - "@org_apache_commons_commons_lang3//jar", - "@com_beust_jcommander//jar", + "@mvnorg_apache_commons_commons_lang3//jar", + "@mvncom_beust_jcommander//jar", + "@mvnorg_testng_testng//jar", ], ) -java_test( - name = "AllTests", +checkstyle_test( + name = "org_ray_ray_runtime_test-checkstyle", + target = ":org_ray_ray_runtime_test", size = "small", +) + +exports_files(["testng.xml"]) + +java_binary( + name = "AllTests", main_class = "org.testng.TestNG", data = ["testng.xml"], runtime_deps = [ diff --git a/java/build.sh b/java/build.sh deleted file mode 100755 index 9fc0aa7ab742..000000000000 --- a/java/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -bazel build -c opt //java:all diff --git a/java/dependencies.yaml b/java/dependencies.yaml new file mode 100644 index 000000000000..f37adc748047 --- /dev/null +++ b/java/dependencies.yaml @@ -0,0 +1,116 @@ +options: + languages: [ "java" ] + licenses: [ "permissive" ] + namePrefix: "mvn" + resolverType: "coursier" + resolvers: + - id: "mavencentral" + type: "default" + url: https://repo.maven.apache.org/maven2/ + thirdPartyDirectory: "java/third_party/" + transitivity: runtime_deps + versionConflictPolicy: highest + +dependencies: + com.puppycrawl.tools: + checkstyle: + lang: java + version: "8.15" + + de.ruedigermoeller: + fst: + lang: java + version: "2.47" + + org.ini4j: + ini4j: + lang: java + version: "0.5.2" + + org.ow2.asm: + asm: + lang: java + version: "6.0" + + com.github.davidmoten: + flatbuffers-java: + lang: java + version: "1.9.0.1" + + com.beust: + jcommander: + lang: java + version: "1.72" + + redis.clients: + jedis: + lang: java + version: "2.8.0" + + commons-io: + commons-io: + lang: java + version: "2.5" + + org.apache.commons: + commons-lang3: + lang: java + version: "3.4" + + commons-codec: + commons-codec: + lang: java + version: "1.4" + + net.lingala.zip4j: + zip4j: + lang: java + version: "1.3.2" + + com.google.guava: + guava: + lang: java + version: "19.0" + + commons-collections: + commons-collections: + lang: java + version: "3.2.2" + + org.slf4j: + slf4j-log4j12: + lang: java + version: "1.7.25" + + com.typesafe: + config: + lang: java + version: "1.3.2" + + net.java.dev.jna: + jna: + lang: java + version: "4.1.0" + + org.mockito: + mockito-all: + lang: java + version: "1.10.19" + + org.testng: + testng: + lang: java + version: "6.9.9" + + javax.xml.bind: + jaxb-api: + lang: java + version: "2.3.0" + + com.sun.xml.bind: + jaxb-core: + lang: java + version: "2.3.0" + jaxb-impl: + lang: java + version: "2.3.0" diff --git a/java/generate_deps.sh b/java/generate_deps.sh new file mode 100644 index 000000000000..3c512c97db25 --- /dev/null +++ b/java/generate_deps.sh @@ -0,0 +1,11 @@ +#/bin/bash + +set -e +# Show explicitly which commands are currently running. +set -x + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +pushd $ROOT_DIR/.. +bazel run //java:bazel_deps -- generate -r `pwd` -s java/third_party/workspace.bzl -d java/dependencies.yaml +popd diff --git a/java/test.sh b/java/test.sh index 25fa0e6aab33..df3d2a4a9073 100755 --- a/java/test.sh +++ b/java/test.sh @@ -7,14 +7,14 @@ set -x ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -pushd $ROOT_DIR/../java -echo "Checking code format." -mvn checkstyle:check -popd +# run this file before compile the targets +sh $ROOT_DIR/generate_deps.sh echo "Compiling Java code." pushd $ROOT_DIR/.. -bazel build -c opt //java:all +bazel test -c opt //java:all + +bazel build -c opt //java:org_ray_ray_java_root popd pushd $ROOT_DIR/../java/test diff --git a/java/third_party/BUILD b/java/third_party/BUILD new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/java/third_party/workspace.bzl b/java/third_party/workspace.bzl new file mode 100644 index 000000000000..4aa7c40b298f --- /dev/null +++ b/java/third_party/workspace.bzl @@ -0,0 +1,73 @@ +# Do not edit. bazel-deps autogenerates this file from java/dependencies.yaml. +def _jar_artifact_impl(ctx): + jar_name = "%s.jar" % ctx.name + ctx.download( + output=ctx.path("jar/%s" % jar_name), + url=ctx.attr.urls, + sha256=ctx.attr.sha256, + executable=False + ) + src_name="%s-sources.jar" % ctx.name + srcjar_attr="" + has_sources = len(ctx.attr.src_urls) != 0 + if has_sources: + ctx.download( + output=ctx.path("jar/%s" % src_name), + url=ctx.attr.src_urls, + sha256=ctx.attr.src_sha256, + executable=False + ) + srcjar_attr ='\n srcjar = ":%s",' % src_name + + build_file_contents = """ +package(default_visibility = ['//visibility:public']) +java_import( + name = 'jar', + jars = ['{jar_name}'],{srcjar_attr} +) +filegroup( + name = 'file', + srcs = [ + '{jar_name}', + '{src_name}' + ], + visibility = ['//visibility:public'] +)\n""".format(jar_name = jar_name, src_name = src_name, srcjar_attr = srcjar_attr) + ctx.file(ctx.path("jar/BUILD"), build_file_contents, False) + return None + +jar_artifact = repository_rule( + attrs = { + "artifact": attr.string(mandatory = True), + "sha256": attr.string(mandatory = True), + "urls": attr.string_list(mandatory = True), + "src_sha256": attr.string(mandatory = False, default=""), + "src_urls": attr.string_list(mandatory = False, default=[]), + }, + implementation = _jar_artifact_impl +) + +def jar_artifact_callback(hash): + src_urls = [] + src_sha256 = "" + source=hash.get("source", None) + if source != None: + src_urls = [source["url"]] + src_sha256 = source["sha256"] + jar_artifact( + artifact = hash["artifact"], + name = hash["name"], + urls = [hash["url"]], + sha256 = hash["sha256"], + src_urls = src_urls, + src_sha256 = src_sha256 + ) + native.bind(name = hash["bind"], actual = hash["actual"]) + +def list_dependencies(): + return [ + ] + +def maven_dependencies(callback = jar_artifact_callback): + for hash in list_dependencies(): + callback(hash) diff --git a/java/tools/checkstyle/BUILD b/java/tools/checkstyle/BUILD new file mode 100644 index 000000000000..f580ac937ca2 --- /dev/null +++ b/java/tools/checkstyle/BUILD @@ -0,0 +1,7 @@ +exports_files([ + "checkstyle.sh", + "license-header.txt", + "checkstyle.xml", + "checkstyle-suppressions.xml", + "repo.bzl", +]) diff --git a/java/tools/checkstyle/README.md b/java/tools/checkstyle/README.md new file mode 100644 index 000000000000..0a83260d8081 --- /dev/null +++ b/java/tools/checkstyle/README.md @@ -0,0 +1,70 @@ +# Checkstyle integration + +## What is Checkstyle + +>[checkstyle](http://checkstyle.sourceforge.net/) is a development tool to +help programmers write Java code that adheres to a coding standard. +It automates the process of checking Java code to spare humans of this +boring (but important) task. This makes it ideal for projects that want to +enforce a coding standard. + +## Integrating with `bazel` + +To apply `checkstyle` check to Java code built with `bazel`, +we use the `checkstyle_test` rule. + +Example: + +`WORKSPACE`: +``` +load("//tools/checkstyle:checkstyle.bzl", "checkstyle_dependencies") +checkstyle_dependencies() +``` + +`BUILD`: + +``` +load("//tools/checkstyle:checkstyle.bzl", "checkstyle_test") + +java_library( + name = "lib", + srcs = ["Lib.java"], +) + +java_library( + name = "ignored_lib", + srcs = ["Lib.java"], + tags = ["checkstyle_ignore"] +) + +checkstyle_test( + name = "lib-checkstyle", + allow_failure = 1, + target = ":lib", +) + +``` + +Sample code illustrates several points: +* To apply Checkstyle to Java target named `lib`, create a `checkstyle_test` +named `lib-checkstyle` (with a dash). Such naming convention is enforced at rule level. +* `checkstyle_test` **is** a test in Bazel context, which means you can do +`bazel test //:lib-checkstyle` and get result of that inspection. +* To perform initial integration, `allow_failure` attribute can be temporarily set, +which will _still_ perform Checkstyle test, but will succeed (as Bazel test) +independently of check's result. +* Targets that do not need to be checked (e.g. generated code), such as `ignored_lib` +can be ignored by assigning `tags = ["checkstyle_ignore"]` +* Checkstyle test **completeness** (that is, all Java code that _needs_ checking +**is** being checked) is enforced via the shell script named `check-for-missing-targets.sh` +ran by CI. If you omit **both** `checkstyle_test` target and `checkstyle_ignore` tag, the script +will exit with non-zero code and print out non-covered targets. + +## Configuring Checkstyle + +Checkstyle is configured by an XML file. By default, current rule implementation uses +`//tools/checkstyle:config.xml` which is based on Google Java style +with added checks of license header +(via [`RegexpHeader`](http://checkstyle.sourceforge.net/config_header.html#RegexpHeader) +and `//tools/checkstyle:license-header.txt`) and package header +(via [`PackageName`](http://checkstyle.sourceforge.net/config_naming.html#PackageName)) diff --git a/java/tools/checkstyle/check-for-missing-targets.sh b/java/tools/checkstyle/check-for-missing-targets.sh new file mode 100755 index 000000000000..a28b6d2a1db4 --- /dev/null +++ b/java/tools/checkstyle/check-for-missing-targets.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# This script verifies that all Java targets except those having assigned `checkstyle_ignore` tag +# do have a accompanying Checkstyle target + +# [1] Get all Java targets except those having assigned `checkstyle_ignore` tag +bazel query 'kind(java_*, //...) - kind(java_proto_library, //...) - //third_party/... - attr("tags", "checkstyle_ignore", //...)' | sort >/tmp/all_java_targets +# [2] Get all checkstyle targets' names, replace '-checkstyle' with nothing so it matches Java target name +bazel query 'kind(checkstyle_test, //...)' | sed -e 's|-checkstyle||g' | sort >/tmp/checkstyle_covered_java_targets + +# [3] Targets in [1] which are not in [2] +NON_COVERED_JAVA_TARGETS=$(comm -23 /tmp/all_java_targets /tmp/checkstyle_covered_java_targets) + +# Having non-empty [3] is an error +if [[ ! -z "$NON_COVERED_JAVA_TARGETS" ]]; then + echo "$(tput setaf 1)[!] These java targets do not have accompanying checkstyle targets:"$(tput sgr0) + echo ${NON_COVERED_JAVA_TARGETS} + exit 1 +fi diff --git a/java/checkstyle-suppressions.xml b/java/tools/checkstyle/checkstyle-suppressions.xml similarity index 100% rename from java/checkstyle-suppressions.xml rename to java/tools/checkstyle/checkstyle-suppressions.xml diff --git a/java/tools/checkstyle/checkstyle.bzl b/java/tools/checkstyle/checkstyle.bzl new file mode 100644 index 000000000000..ec44d48d5a27 --- /dev/null +++ b/java/tools/checkstyle/checkstyle.bzl @@ -0,0 +1,127 @@ +#Structure representing info about Java source files +JavaSourceFiles = provider( + fields = { + 'files' : 'java source files' + } +) + +# This aspect is responsible for collecting Java sources for target +# When applied to target, `JavaSourceFiles` struct will be attached to +# target info +def collect_sources_impl(target, ctx): + files = [] + if hasattr(ctx.rule.attr, 'srcs'): + for src in ctx.rule.attr.srcs: + for file in src.files: + if file.extension == 'java': + files.append(file) + return [JavaSourceFiles(files = files)] + + +collect_sources = aspect( + implementation = collect_sources_impl, +) + + +# `ctx` is rule context: https://docs.bazel.build/versions/master/skylark/lib/ctx.html +def _checkstyle_test_impl(ctx): + # verify target name matches naming conventions + if "{}-checkstyle".format(ctx.attr.target.label.name) != ctx.attr.name: + fail("target should follow `{java_library target name}-checkstyle` pattern") + + suppressions = ctx.file.suppressions + opts = ctx.attr.opts + sopts = ctx.attr.string_opts + + # Checkstyle and its dependencies + checkstyle_dependencies = ctx.attr._checkstyle.java.transitive_runtime_deps + classpath = ":".join([file.path for file in checkstyle_dependencies]) + + args = "" + inputs = [] + if ctx.file.config: + args += " -c %s" % ctx.file.config.path + inputs.append(ctx.file.config) + if suppressions: + inputs.append(suppressions) + + # Build command to run Checkstyle test + cmd = " ".join( + ["java -cp %s com.puppycrawl.tools.checkstyle.Main" % classpath] + + [args] + + ["--%s" % x for x in opts] + + ["--%s %s" % (k, sopts[k]) for k in sopts] + + [file.path for file in ctx.attr.target[JavaSourceFiles].files] + ) + + # Wrap checkstyle command in a shell script so allow_failure is supported + ctx.actions.expand_template( + template = ctx.file._checkstyle_sh_template, + output = ctx.outputs.checkstyle_script, + substitutions = { + "{command}" : cmd, + "{allow_failure}": str(int(ctx.attr.allow_failure)), + }, + is_executable = True, + ) + + files = [ctx.outputs.checkstyle_script, ctx.file.license] + ctx.attr.target[JavaSourceFiles].files + checkstyle_dependencies.to_list() + inputs + runfiles = ctx.runfiles( + files = files, + collect_data = True + ) + return DefaultInfo( + executable = ctx.outputs.checkstyle_script, + files = depset(files), + runfiles = runfiles, + ) + +checkstyle_test = rule( + implementation = _checkstyle_test_impl, + test = True, + attrs = { + "config": attr.label( + allow_single_file=True, + doc = "A checkstyle configuration file", + default = "//java/tools/checkstyle:checkstyle.xml", + ), + "suppressions": attr.label( + allow_single_file=True, + doc = ("A file for specifying files and lines " + + "that should be suppressed from checks." + + "Example: https://github.com/checkstyle/checkstyle/blob/master/config/suppressions.xml"), + default = "//java/tools/checkstyle:checkstyle-suppressions.xml", + ), + "license": attr.label( + allow_single_file=True, + doc = "A license file that can be used with the checkstyle license target", + default = "//java/tools/checkstyle:license-header.txt", + ), + "opts": attr.string_list( + doc = "Options to be passed on the command line that have no argument" + ), + "string_opts": attr.string_dict( + doc = "Options to be passed on the command line that have an argument" + ), + "target": attr.label( + doc = "The java_library target to check sources on", + aspects = [collect_sources], + mandatory = True + ), + "allow_failure": attr.bool( + default = False, + doc = "Successfully finish the test even if checkstyle failed" + ), + "_checkstyle_sh_template": attr.label( + allow_single_file = True, + default = "//java/tools/checkstyle:checkstyle.sh" + ), + "_checkstyle": attr.label( + default = "//java/third_party/com/puppycrawl/tools:checkstyle" + ), + }, + outputs = { + "checkstyle_script": "%{name}.sh", + }, +) + diff --git a/java/tools/checkstyle/checkstyle.sh b/java/tools/checkstyle/checkstyle.sh new file mode 100755 index 000000000000..622a9cda9db9 --- /dev/null +++ b/java/tools/checkstyle/checkstyle.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +COMMAND="{command}" +ALLOW_FAILURE="{allow_failure}" + +echo "ALLOW_FAILURE IS $ALLOW_FAILURE" + +source ~/.bash_profile + +${COMMAND} +result=$? + +if [[ ${ALLOW_FAILURE} -eq 1 ]]; then + exit 0 +fi + +exit ${result} diff --git a/java/checkstyle.xml b/java/tools/checkstyle/checkstyle.xml similarity index 100% rename from java/checkstyle.xml rename to java/tools/checkstyle/checkstyle.xml diff --git a/java/tools/checkstyle/license-header.txt b/java/tools/checkstyle/license-header.txt new file mode 100644 index 000000000000..2d4ccdb2ee66 --- /dev/null +++ b/java/tools/checkstyle/license-header.txt @@ -0,0 +1,15 @@ +^\W* +^\W*Copyright 20[0-9]{2} The StartupOS Authors. +^\W* +^\W*Licensed under the Apache License, Version 2.0 \(the "License"\); +^\W*you may not use this file except in compliance with the License. +^\W*You may obtain a copy of the License at +^\W* +^\W*https://www.apache.org/licenses/LICENSE-2.0 +^\W* +^\W*Unless required by applicable law or agreed to in writing, software +^\W*distributed under the License is distributed on an "AS IS" BASIS, +^\W*WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +^\W*See the License for the specific language governing permissions and +^\W*limitations under the License. +^\W* \ No newline at end of file diff --git a/java/tools/checkstyle/repo.bzl b/java/tools/checkstyle/repo.bzl new file mode 100644 index 000000000000..8119b8d4fe4f --- /dev/null +++ b/java/tools/checkstyle/repo.bzl @@ -0,0 +1,21 @@ +load("//java/third_party:workspace.bzl", "maven_dependencies") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_jar") + +def __maybe(repo_rule, name, **kwargs): + if name not in native.existing_rules(): + repo_rule(name = name, **kwargs) + +def __maven_repositories(): + maven_dependencies() + +def __bazel_deps(): + __maybe( + http_jar, + name = "bazel_deps", + sha256 = "98b05c2826f2248f70e7356dc6c78bc52395904bb932fbb409a5abf5416e4292", + urls = ["https://github.com/oferb/startupos-binaries/releases/download/0.1.01/bazel_deps.jar"], + ) + +def checkstyle_repositories(): + __maven_repositories() + __bazel_deps() From 248cdef8d1158e77f6728caf5bf33c64f6d84cb7 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Mon, 11 Mar 2019 19:47:34 +0800 Subject: [PATCH 021/115] modify bazel format --- bazel/BUILD.plasma | 2 +- bazel/ray.bzl | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index c57f3dd1990d..d89868628d7e 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -19,7 +19,7 @@ java_binary( main_class = "org.apache.arrow.plasma.PlasmaClientTest", visibility = ["//visibility:public"], deps = [ - ":org_apache_arrow_arrow_plasma", + ":org_apache_arrow_arrow_plasma", "@mvnjunit_junit//jar", ], ) diff --git a/bazel/ray.bzl b/bazel/ray.bzl index 2f68b25127b6..85ae845ef1c0 100644 --- a/bazel/ray.bzl +++ b/bazel/ray.bzl @@ -21,4 +21,3 @@ def flatbuffer_java_library(name, srcs, outs, out_prefix, includes = [], include include_paths = include_paths, includes = includes, ) - From 7fbe6ac5bf78c22ddc2c087addb1cb5b090834d8 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Mon, 11 Mar 2019 20:03:42 +0800 Subject: [PATCH 022/115] Some code errors --- WORKSPACE | 4 ++-- java/{tools/checkstyle => }/repo.bzl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename java/{tools/checkstyle => }/repo.bzl (95%) diff --git a/WORKSPACE b/WORKSPACE index 3d8fc696e130..7c18beac161c 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,9 +1,9 @@ workspace(name = "com_github_ray_project_ray") load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository") -load("//java:tools/checkstyle/repo.bzl", "checkstyle_repositories") +load("//java:repo.bzl", "java_repositories") -checkstyle_repositories() +java_repositories() git_repository( name = "com_github_nelhage_rules_boost", diff --git a/java/tools/checkstyle/repo.bzl b/java/repo.bzl similarity index 95% rename from java/tools/checkstyle/repo.bzl rename to java/repo.bzl index 8119b8d4fe4f..e0989b0edba0 100644 --- a/java/tools/checkstyle/repo.bzl +++ b/java/repo.bzl @@ -16,6 +16,6 @@ def __bazel_deps(): urls = ["https://github.com/oferb/startupos-binaries/releases/download/0.1.01/bazel_deps.jar"], ) -def checkstyle_repositories(): +def java_repositories(): __maven_repositories() __bazel_deps() From 13ece31e4e5b89f32c3b532bd137fa131bc11800 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Mon, 11 Mar 2019 20:17:09 +0800 Subject: [PATCH 023/115] code errors --- bazel/BUILD.plasma | 3 --- 1 file changed, 3 deletions(-) diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index d89868628d7e..e73798741044 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -1,7 +1,4 @@ load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library") -load("@com_github_ray_project_ray//bazel:ray_java.bzl", "generated_java_libraries") - -generated_java_libraries() java_library( name = "org_apache_arrow_arrow_plasma", From 9b6f46d04b41b7501828922fd167c00a18767d80 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Mon, 11 Mar 2019 20:55:23 +0800 Subject: [PATCH 024/115] modify the error 'pushd not find' --- java/generate_deps.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/java/generate_deps.sh b/java/generate_deps.sh index 3c512c97db25..39ecc9f488ae 100644 --- a/java/generate_deps.sh +++ b/java/generate_deps.sh @@ -1,11 +1,12 @@ -#/bin/bash +#!/usr/bin/env bash set -e -# Show explicitly which commands are currently running. set -x ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -pushd $ROOT_DIR/.. +cd $ROOT_DIR/.. bazel run //java:bazel_deps -- generate -r `pwd` -s java/third_party/workspace.bzl -d java/dependencies.yaml -popd + +set +x +set +e From a6cfd79a3622569e4546c86cd901ad9c34123e53 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Mon, 11 Mar 2019 21:34:24 +0800 Subject: [PATCH 025/115] modify compile error --- java/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/test.sh b/java/test.sh index df3d2a4a9073..f7b11be45d8d 100755 --- a/java/test.sh +++ b/java/test.sh @@ -12,9 +12,9 @@ sh $ROOT_DIR/generate_deps.sh echo "Compiling Java code." pushd $ROOT_DIR/.. -bazel test -c opt //java:all +bazel build -c opt //java:all -bazel build -c opt //java:org_ray_ray_java_root +bazel test -c opt //java:all popd pushd $ROOT_DIR/../java/test From 5540b93a8363f70f9be26faa2d5de992eaed0d80 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Tue, 12 Mar 2019 11:48:35 +0800 Subject: [PATCH 026/115] modify generate_java_gcs_fbs to ensure runtime's compile is successful --- java/BUILD.bazel | 73 +++++++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 43eec664b94f..403c8a1dbf64 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -52,10 +52,14 @@ checkstyle_test( java_library( name = "org_ray_ray_runtime", - srcs = glob(["runtime/src/main/java/**/*.java"]), + srcs = glob([ + "runtime/src/main/java/**/*.java" + ]) + [ + ":generate_java_gcs_fbs", + ], resources = glob(["runtime/src/main/resources/**"]), visibility = ["//visibility:public"], - data = ["//:raylet_library_java", ":generate_java_gcs_fbs"], + data = ["//:raylet_library_java"], deps = [ ":org_ray_ray_api", "@plasma//:org_apache_arrow_arrow_plasma", @@ -156,11 +160,45 @@ java_binary( genrule( name = "generate_java_gcs_fbs", - srcs = [":copy_java_gcs_fbs"], - outs = ["generate_java_gcs_fbs.out"], + srcs = [":java_gcs_fbs"], + outs = [ + "runtime/src/main/java/org/ray/runtime/generated/ActorCheckpointData.java", + "runtime/src/main/java/org/ray/runtime/generated/ActorCheckpointIdData.java", + "runtime/src/main/java/org/ray/runtime/generated/ActorState.java", + "runtime/src/main/java/org/ray/runtime/generated/ActorTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/Arg.java", + "runtime/src/main/java/org/ray/runtime/generated/ClassTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/ClientTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/ConfigTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/CustomSerializerData.java", + "runtime/src/main/java/org/ray/runtime/generated/DriverTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/ErrorTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/ErrorType.java", + "runtime/src/main/java/org/ray/runtime/generated/FunctionTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/GcsTableEntry.java", + "runtime/src/main/java/org/ray/runtime/generated/HeartbeatBatchTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/HeartbeatTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/Language.java", + "runtime/src/main/java/org/ray/runtime/generated/ObjectTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/ProfileEvent.java", + "runtime/src/main/java/org/ray/runtime/generated/ProfileTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/RayResource.java", + "runtime/src/main/java/org/ray/runtime/generated/ResourcePair.java", + "runtime/src/main/java/org/ray/runtime/generated/SchedulingState.java", + "runtime/src/main/java/org/ray/runtime/generated/TablePrefix.java", + "runtime/src/main/java/org/ray/runtime/generated/TablePubsub.java", + "runtime/src/main/java/org/ray/runtime/generated/TaskInfo.java", + "runtime/src/main/java/org/ray/runtime/generated/TaskLeaseData.java", + "runtime/src/main/java/org/ray/runtime/generated/TaskReconstructionData.java", + "runtime/src/main/java/org/ray/runtime/generated/TaskTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/TaskTableTestAndUpdate.java", + ], cmd = """ - python $$(pwd)/java/modify_generated_java_flatbuffers_files.py $$(pwd) - echo $$(pwd) > $@ + for f in $(locations //java:java_gcs_fbs); do + chmod +w $$f + cp -f $$f $(@D)/runtime/src/main/java/org/ray/runtime/generated + done + python $$(pwd)/java/modify_generated_java_flatbuffers_files.py $(@D)/.. """, local = 1, ) @@ -200,26 +238,5 @@ flatbuffer_java_library( "TaskTableData.java", "TaskTableTestAndUpdate.java", ], - out_prefix = "java/runtime/src/main/java/org/ray/runtime/generated", -) - -genrule( - name = "copy_java_gcs_fbs", - srcs = [ - "//java:java_gcs_fbs", - ], - outs = [ - "java_gcs_fbs.out", - ], - cmd = """ - WORK_DIR=$$(pwd) - GENERATED_DIR=$$WORK_DIR/java/runtime/src/main/java/org/ray/runtime/generated - mkdir -p $$GENERATED_DIR - for f in $(locations //java:java_gcs_fbs); do - chmod +w $$f - cp -f $$f $$GENERATED_DIR/ - done - echo $$(pwd) >> $@ - """, - local = 1, + out_prefix = "runtime/src/main/java/org/ray/runtime/generated", ) From e745a05c0d214ec84a7ee2145ec76f1a9a640e91 Mon Sep 17 00:00:00 2001 From: Yuhong Guo Date: Tue, 12 Mar 2019 15:34:31 +0800 Subject: [PATCH 027/115] Update Arrow to plasma-client-separation-on-c0a2e73 --- WORKSPACE | 2 +- bazel/BUILD.plasma | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 7c18beac161c..4a3526a9036a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -42,7 +42,7 @@ git_repository( new_git_repository( name = "plasma", build_file = "@//bazel:BUILD.plasma", - commit = "6a27c660ea700febf6fd73b2e851ab96e9315134", + commit = "fa8967cb9ec228f6544b2eecc0f1bd78f29daa4d", remote = "https://github.com/ray-project/arrow", ) diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index e73798741044..efe69098d710 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -29,6 +29,7 @@ cc_library( "cpp/src/arrow/status.cc", "cpp/src/arrow/util/io-util.cc", "cpp/src/arrow/util/logging.cc", + "cpp/src/arrow/util/memory.cc", "cpp/src/arrow/util/thread-pool.cc", ], hdrs = [ @@ -40,6 +41,7 @@ cc_library( "cpp/src/arrow/util/io-util.h", "cpp/src/arrow/util/logging.h", "cpp/src/arrow/util/macros.h", + "cpp/src/arrow/util/memory.h", "cpp/src/arrow/util/string_builder.h", "cpp/src/arrow/util/string_view.h", "cpp/src/arrow/util/thread-pool.h", @@ -62,6 +64,7 @@ cc_library( "cpp/src/plasma/io.cc", "cpp/src/plasma/malloc.cc", "cpp/src/plasma/plasma.cc", + "cpp/src/plasma/plasma_allocator.cc", "cpp/src/plasma/protocol.cc", ], hdrs = [ @@ -73,6 +76,7 @@ cc_library( "cpp/src/plasma/io.h", "cpp/src/plasma/malloc.h", "cpp/src/plasma/plasma.h", + "cpp/src/plasma/plasma_allocator.h", "cpp/src/plasma/plasma_generated.h", "cpp/src/plasma/protocol.h", ], @@ -143,10 +147,12 @@ cc_library( "cpp/src/plasma/dlmalloc.cc", "cpp/src/plasma/events.cc", "cpp/src/plasma/eviction_policy.cc", + "cpp/src/plasma/external_store.cc", ], hdrs = [ "cpp/src/plasma/events.h", "cpp/src/plasma/eviction_policy.h", + "cpp/src/plasma/external_store.h", "cpp/src/plasma/store.h", "cpp/src/plasma/thirdparty/ae/ae.h", "cpp/src/plasma/thirdparty/ae/ae_epoll.c", From a2938168a2ac91f0d9a81c587991daa9b32dec5f Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Tue, 12 Mar 2019 16:02:33 +0800 Subject: [PATCH 028/115] modify code with raulchen's and pcmoritz's suggestions --- .gitignore | 1 + bazel/BUILD.plasma | 4 +- build.sh | 7 +- java/BUILD.bazel | 178 ++++++++---------- .../checkstyle-suppressions.xml | 0 java/{tools/checkstyle => }/checkstyle.xml | 0 java/dependencies.yaml | 2 +- java/repo.bzl | 15 +- java/test.sh | 11 +- java/tools/checkstyle/BUILD | 1 - java/tools/checkstyle/checkstyle.bzl | 4 +- 11 files changed, 96 insertions(+), 127 deletions(-) rename java/{tools/checkstyle => }/checkstyle-suppressions.xml (100%) rename java/{tools/checkstyle => }/checkstyle.xml (100%) diff --git a/.gitignore b/.gitignore index da464a65266a..7abe95b31a25 100644 --- a/.gitignore +++ b/.gitignore @@ -148,6 +148,7 @@ java/**/lib java/**/.settings java/**/.classpath java/**/.project +java/third_party/workspace.bzl # python virtual env venv diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index efe69098d710..f221c8af17fd 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -6,7 +6,7 @@ java_library( data = [":plasma_client_java"], visibility = ["//visibility:public"], deps = [ - "@mvnorg_slf4j_slf4j_api//jar", + "@org_slf4j_slf4j_api//jar", ], ) @@ -17,7 +17,7 @@ java_binary( visibility = ["//visibility:public"], deps = [ ":org_apache_arrow_arrow_plasma", - "@mvnjunit_junit//jar", + "@junit_junit//jar", ], ) diff --git a/build.sh b/build.sh index 7d94a4992a3e..aa0f7b0e2b1b 100755 --- a/build.sh +++ b/build.sh @@ -123,8 +123,9 @@ else --find-links https://s3-us-west-2.amazonaws.com/arrow-wheels/9357dc130789ee42f8181d8724bee1d5d1509060/index.html if [ "$RAY_BUILD_JAVA" == "YES" ]; then - bazel build //:ray_java_pkg -c opt - #soft link + bazel build //:ray_java_pkg -c opt --verbose_failures + # The following are soft links. + # TODO: remove this once cmake is removed mkdir -p $ROOT_DIR/build/src/ray/raylet/ mkdir -p $ROOT_DIR/build/src/ray/gcs/redis_module/ mkdir -p $ROOT_DIR/build/src/ray/thirdparty/redis/src/ @@ -138,7 +139,7 @@ else fi if [ "$RAY_BUILD_PYTHON" == "YES" ]; then - bazel build //:ray_pkg -c opt + bazel build //:ray_pkg -c opt --verbose_failures # Copy files and keep them writeable. This is a workaround, as Bazel # marks all generated files non-writeable. If we would just copy them # over without adding write permission, the copy would fail the next time. diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 403c8a1dbf64..26307706cd42 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -20,8 +20,8 @@ java_import( "liborg_ray_ray_test-src.jar", "org_ray_ray_tutorial_deploy.jar", "org_ray_ray_tutorial_deploy-src.jar", - "AllTests_deploy.jar", - "AllTests_deploy-src.jar", + "all_tests_deploy.jar", + "all_tests_deploy-src.jar", ], deps = [ ":org_ray_ray_api", @@ -36,11 +36,11 @@ java_library( name = "org_ray_ray_api", srcs = glob(["api/src/main/java/**/*.java"]), deps = [ - "@mvnorg_slf4j_slf4j_log4j12//jar", - "@mvnlog4j_log4j//jar", - "@mvnjavax_xml_bind_jaxb_api//jar", - "@mvncom_sun_xml_bind_jaxb_core//jar", - "@mvncom_sun_xml_bind_jaxb_impl//jar", + "@org_slf4j_slf4j_log4j12//jar", + "@log4j_log4j//jar", + "@javax_xml_bind_jaxb_api//jar", + "@com_sun_xml_bind_jaxb_core//jar", + "@com_sun_xml_bind_jaxb_impl//jar", ], ) @@ -58,28 +58,26 @@ java_library( ":generate_java_gcs_fbs", ], resources = glob(["runtime/src/main/resources/**"]), - visibility = ["//visibility:public"], data = ["//:raylet_library_java"], deps = [ ":org_ray_ray_api", "@plasma//:org_apache_arrow_arrow_plasma", - "@mvnorg_slf4j_slf4j_api//jar", - #"@mvncom_google_code_gson_gson//jar", - "@mvncom_fasterxml_jackson_core_jackson_core//jar", - "@mvnorg_apache_commons_commons_pool2//jar", - "@mvnorg_javassist_javassist//jar", - "@mvnorg_objenesis_objenesis//jar", - "@mvncom_typesafe_config//jar", - "@mvnorg_apache_commons_commons_lang3//jar", - "@mvnde_ruedigermoeller_fst//jar", - "@mvncom_github_davidmoten_flatbuffers_java//jar", - "@mvnredis_clients_jedis//jar", - "@mvncommons_io_commons_io//jar", - "@mvncom_google_guava_guava//jar", - "@mvnnet_lingala_zip4j_zip4j//jar", - "@mvnorg_slf4j_slf4j_log4j12//jar", - "@mvnorg_ini4j_ini4j//jar", - "@mvnorg_ow2_asm_asm//jar", + "@org_slf4j_slf4j_api//jar", + "@com_fasterxml_jackson_core_jackson_core//jar", + "@org_apache_commons_commons_pool2//jar", + "@org_javassist_javassist//jar", + "@org_objenesis_objenesis//jar", + "@com_typesafe_config//jar", + "@org_apache_commons_commons_lang3//jar", + "@de_ruedigermoeller_fst//jar", + "@com_github_davidmoten_flatbuffers_java//jar", + "@redis_clients_jedis//jar", + "@commons_io_commons_io//jar", + "@com_google_guava_guava//jar", + "@net_lingala_zip4j_zip4j//jar", + "@org_slf4j_slf4j_log4j12//jar", + "@org_ini4j_ini4j//jar", + "@org_ow2_asm_asm//jar", ], ) @@ -93,11 +91,10 @@ java_binary( name = "org_ray_ray_tutorial", srcs = glob(["tutorial/src/main/java/**/*.java"]), resources = glob(["tutorial/src/main/resources/**"]), - visibility = ["//visibility:public"], deps = [ ":org_ray_ray_api", ":org_ray_ray_runtime", - "@mvncom_google_guava_guava//jar", + "@com_google_guava_guava//jar", ], ) @@ -113,12 +110,12 @@ java_library( deps = [ ":org_ray_ray_api", ":org_ray_ray_runtime", - "@mvnorg_apache_commons_commons_lang3//jar", - "@mvnorg_slf4j_slf4j_api//jar", + "@org_apache_commons_commons_lang3//jar", + "@org_slf4j_slf4j_api//jar", "@plasma//:org_apache_arrow_arrow_plasma", - "@mvnorg_testng_testng//jar", - "@mvncommons_collections_commons_collections//jar", - "@mvncom_google_guava_guava//jar", + "@org_testng_testng//jar", + "@commons_collections_commons_collections//jar", + "@com_google_guava_guava//jar", ], ) @@ -134,9 +131,9 @@ java_library( deps = [ ":org_ray_ray_api", ":org_ray_ray_runtime", - "@mvnorg_apache_commons_commons_lang3//jar", - "@mvncom_beust_jcommander//jar", - "@mvnorg_testng_testng//jar", + "@org_apache_commons_commons_lang3//jar", + "@com_beust_jcommander//jar", + "@org_testng_testng//jar", ], ) @@ -149,7 +146,7 @@ checkstyle_test( exports_files(["testng.xml"]) java_binary( - name = "AllTests", + name = "all_tests", main_class = "org.testng.TestNG", data = ["testng.xml"], runtime_deps = [ @@ -158,40 +155,51 @@ java_binary( ], ) +flatbuffers_generate_files = [ + "ActorCheckpointData.java", + "ActorCheckpointIdData.java", + "ActorState.java", + "ActorTableData.java", + "Arg.java", + "ClassTableData.java", + "ClientTableData.java", + "ConfigTableData.java", + "CustomSerializerData.java", + "DriverTableData.java", + "ErrorTableData.java", + "ErrorType.java", + "FunctionTableData.java", + "GcsTableEntry.java", + "HeartbeatBatchTableData.java", + "HeartbeatTableData.java", + "Language.java", + "ObjectTableData.java", + "ProfileEvent.java", + "ProfileTableData.java", + "RayResource.java", + "ResourcePair.java", + "SchedulingState.java", + "TablePrefix.java", + "TablePubsub.java", + "TaskInfo.java", + "TaskLeaseData.java", + "TaskReconstructionData.java", + "TaskTableData.java", + "TaskTableTestAndUpdate.java", +] + +flatbuffer_java_library( + name = "java_gcs_fbs", + srcs = ["//:gcs_fbs_file"], + outs = flatbuffers_generate_files, + out_prefix = "", +) + genrule( name = "generate_java_gcs_fbs", srcs = [":java_gcs_fbs"], outs = [ - "runtime/src/main/java/org/ray/runtime/generated/ActorCheckpointData.java", - "runtime/src/main/java/org/ray/runtime/generated/ActorCheckpointIdData.java", - "runtime/src/main/java/org/ray/runtime/generated/ActorState.java", - "runtime/src/main/java/org/ray/runtime/generated/ActorTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/Arg.java", - "runtime/src/main/java/org/ray/runtime/generated/ClassTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/ClientTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/ConfigTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/CustomSerializerData.java", - "runtime/src/main/java/org/ray/runtime/generated/DriverTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/ErrorTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/ErrorType.java", - "runtime/src/main/java/org/ray/runtime/generated/FunctionTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/GcsTableEntry.java", - "runtime/src/main/java/org/ray/runtime/generated/HeartbeatBatchTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/HeartbeatTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/Language.java", - "runtime/src/main/java/org/ray/runtime/generated/ObjectTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/ProfileEvent.java", - "runtime/src/main/java/org/ray/runtime/generated/ProfileTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/RayResource.java", - "runtime/src/main/java/org/ray/runtime/generated/ResourcePair.java", - "runtime/src/main/java/org/ray/runtime/generated/SchedulingState.java", - "runtime/src/main/java/org/ray/runtime/generated/TablePrefix.java", - "runtime/src/main/java/org/ray/runtime/generated/TablePubsub.java", - "runtime/src/main/java/org/ray/runtime/generated/TaskInfo.java", - "runtime/src/main/java/org/ray/runtime/generated/TaskLeaseData.java", - "runtime/src/main/java/org/ray/runtime/generated/TaskReconstructionData.java", - "runtime/src/main/java/org/ray/runtime/generated/TaskTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/TaskTableTestAndUpdate.java", + "runtime/src/main/java/org/ray/runtime/generated/" + file for file in flatbuffers_generate_files ], cmd = """ for f in $(locations //java:java_gcs_fbs); do @@ -202,41 +210,3 @@ genrule( """, local = 1, ) - -flatbuffer_java_library( - name = "java_gcs_fbs", - srcs = ["//:gcs_fbs_file"], - outs = [ - "ActorCheckpointData.java", - "ActorCheckpointIdData.java", - "ActorState.java", - "ActorTableData.java", - "Arg.java", - "ClassTableData.java", - "ClientTableData.java", - "ConfigTableData.java", - "CustomSerializerData.java", - "DriverTableData.java", - "ErrorTableData.java", - "ErrorType.java", - "FunctionTableData.java", - "GcsTableEntry.java", - "HeartbeatBatchTableData.java", - "HeartbeatTableData.java", - "Language.java", - "ObjectTableData.java", - "ProfileEvent.java", - "ProfileTableData.java", - "RayResource.java", - "ResourcePair.java", - "SchedulingState.java", - "TablePrefix.java", - "TablePubsub.java", - "TaskInfo.java", - "TaskLeaseData.java", - "TaskReconstructionData.java", - "TaskTableData.java", - "TaskTableTestAndUpdate.java", - ], - out_prefix = "runtime/src/main/java/org/ray/runtime/generated", -) diff --git a/java/tools/checkstyle/checkstyle-suppressions.xml b/java/checkstyle-suppressions.xml similarity index 100% rename from java/tools/checkstyle/checkstyle-suppressions.xml rename to java/checkstyle-suppressions.xml diff --git a/java/tools/checkstyle/checkstyle.xml b/java/checkstyle.xml similarity index 100% rename from java/tools/checkstyle/checkstyle.xml rename to java/checkstyle.xml diff --git a/java/dependencies.yaml b/java/dependencies.yaml index f37adc748047..62ca6af2eb36 100644 --- a/java/dependencies.yaml +++ b/java/dependencies.yaml @@ -1,7 +1,7 @@ options: languages: [ "java" ] licenses: [ "permissive" ] - namePrefix: "mvn" + namePrefix: "" resolverType: "coursier" resolvers: - id: "mavencentral" diff --git a/java/repo.bzl b/java/repo.bzl index e0989b0edba0..a06c285f234b 100644 --- a/java/repo.bzl +++ b/java/repo.bzl @@ -1,15 +1,12 @@ load("//java/third_party:workspace.bzl", "maven_dependencies") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_jar") -def __maybe(repo_rule, name, **kwargs): +def create_if_needed(rule, name, **kwargs): if name not in native.existing_rules(): - repo_rule(name = name, **kwargs) + rule(name = name, **kwargs) -def __maven_repositories(): - maven_dependencies() - -def __bazel_deps(): - __maybe( +def bazel_deps(): + create_if_needed( http_jar, name = "bazel_deps", sha256 = "98b05c2826f2248f70e7356dc6c78bc52395904bb932fbb409a5abf5416e4292", @@ -17,5 +14,5 @@ def __bazel_deps(): ) def java_repositories(): - __maven_repositories() - __bazel_deps() + maven_dependencies() + bazel_deps() diff --git a/java/test.sh b/java/test.sh index f7b11be45d8d..689f39aa08f8 100755 --- a/java/test.sh +++ b/java/test.sh @@ -12,17 +12,18 @@ sh $ROOT_DIR/generate_deps.sh echo "Compiling Java code." pushd $ROOT_DIR/.. -bazel build -c opt //java:all - -bazel test -c opt //java:all +# bazel checkstyle for java +bazel test //java:all -c opt +# compile all the targets +bazel build //java:all -c opt --verbose_failures popd pushd $ROOT_DIR/../java/test echo "Running tests under cluster mode." -ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml +ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/../java/testng.xml echo "Running tests under single-process mode." -java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml +java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/../java/testng.xml set +x set +e diff --git a/java/tools/checkstyle/BUILD b/java/tools/checkstyle/BUILD index f580ac937ca2..b323bb6066f8 100644 --- a/java/tools/checkstyle/BUILD +++ b/java/tools/checkstyle/BUILD @@ -3,5 +3,4 @@ exports_files([ "license-header.txt", "checkstyle.xml", "checkstyle-suppressions.xml", - "repo.bzl", ]) diff --git a/java/tools/checkstyle/checkstyle.bzl b/java/tools/checkstyle/checkstyle.bzl index ec44d48d5a27..51129aeb830a 100644 --- a/java/tools/checkstyle/checkstyle.bzl +++ b/java/tools/checkstyle/checkstyle.bzl @@ -83,14 +83,14 @@ checkstyle_test = rule( "config": attr.label( allow_single_file=True, doc = "A checkstyle configuration file", - default = "//java/tools/checkstyle:checkstyle.xml", + default = "//java:checkstyle.xml", ), "suppressions": attr.label( allow_single_file=True, doc = ("A file for specifying files and lines " + "that should be suppressed from checks." + "Example: https://github.com/checkstyle/checkstyle/blob/master/config/suppressions.xml"), - default = "//java/tools/checkstyle:checkstyle-suppressions.xml", + default = "//java:checkstyle-suppressions.xml", ), "license": attr.label( allow_single_file=True, From e296c0b0d6fefee75a594f1b568b7630f90be661 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Tue, 12 Mar 2019 21:08:30 +0800 Subject: [PATCH 029/115] add soft link to some path that cmake used --- java/test.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/java/test.sh b/java/test.sh index 689f39aa08f8..10cb943b5b41 100755 --- a/java/test.sh +++ b/java/test.sh @@ -18,7 +18,14 @@ bazel test //java:all -c opt bazel build //java:all -c opt --verbose_failures popd -pushd $ROOT_DIR/../java/test +# The following are soft links +# TODO: remove this once cmake is removed +mkdir -p $ROOT_DIR/../build/java/ +ln -sf $ROOT_DIR/../bazel-bin/java/* $ROOT_DIR/../build/java/ +mkdir -p $ROOT_DIR/tutorial/target/ +ln -sf $ROOT_DIR/../bazel-bin/java/org_ray_ray_tutorial_deploy.jar $ROOT_DIR/tutorial/target/ray-tutorial-0.1-SNAPSHOT.jar + +pushd $ROOT_DIR/test echo "Running tests under cluster mode." ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/../java/testng.xml From 27b9e97734c410c763447a6edf0011c6eda0b42a Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Tue, 12 Mar 2019 21:56:11 +0800 Subject: [PATCH 030/115] modify java/test.sh --- java/test.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/java/test.sh b/java/test.sh index 10cb943b5b41..874f8c58a8e4 100755 --- a/java/test.sh +++ b/java/test.sh @@ -32,7 +32,4 @@ ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../ba echo "Running tests under single-process mode." java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/../java/testng.xml -set +x -set +e - popd From 29ed8bbf16107d1e17b5521dc5388742fda4c267 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Wed, 13 Mar 2019 18:17:29 +0800 Subject: [PATCH 031/115] modify java/test.sh with the exitcode 2 --- java/test.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/java/test.sh b/java/test.sh index 874f8c58a8e4..deff9f35a56c 100755 --- a/java/test.sh +++ b/java/test.sh @@ -27,9 +27,13 @@ ln -sf $ROOT_DIR/../bazel-bin/java/org_ray_ray_tutorial_deploy.jar $ROOT_DIR/tut pushd $ROOT_DIR/test echo "Running tests under cluster mode." -ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/../java/testng.xml +ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml || exit_code=$? echo "Running tests under single-process mode." -java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/../java/testng.xml +java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml || exit_code=$? +# exit_code == 2 means there are some tests skiped. +if [ $exit_code -eq 2 ] && [ $exit_code -eq 0 ] ; then + exit $exit_code +fi popd From 50bb868a992f3cb41b166a3306bf27d075525e53 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Wed, 13 Mar 2019 18:32:39 +0800 Subject: [PATCH 032/115] modify java/test.sh --- java/test.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/java/test.sh b/java/test.sh index deff9f35a56c..6200d632264d 100755 --- a/java/test.sh +++ b/java/test.sh @@ -27,12 +27,17 @@ ln -sf $ROOT_DIR/../bazel-bin/java/org_ray_ray_tutorial_deploy.jar $ROOT_DIR/tut pushd $ROOT_DIR/test echo "Running tests under cluster mode." -ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml || exit_code=$? +ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml || cluster_exit_code=$? + +# exit_code == 2 means there are some tests skiped. +if [ $cluster_exit_code -eq 2 ] && [ $cluster_exit_code -eq 0 ] ; then + exit $exit_code +fi echo "Running tests under single-process mode." -java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml || exit_code=$? +java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml || single_exit_code=$? # exit_code == 2 means there are some tests skiped. -if [ $exit_code -eq 2 ] && [ $exit_code -eq 0 ] ; then +if [ $single_exit_code -eq 2 ] && [ $single_exit_code -eq 0 ] ; then exit $exit_code fi From ed909b2084cef3e852f1d9512cc38dea04ce03cb Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Thu, 14 Mar 2019 21:40:08 +0800 Subject: [PATCH 033/115] [Java] Package binary dependencies into jar --- .gitignore | 1 + CMakeLists.txt | 21 +++++- java/README.rst | 2 +- java/example.conf | 5 -- java/runtime/pom.xml | 8 +++ .../org/ray/runtime/RayNativeRuntime.java | 28 +++++++- .../org/ray/runtime/config/RayConfig.java | 71 +++++-------------- .../org/ray/runtime/runner/RunManager.java | 45 ++++++++++-- .../src/main/resources/ray.default.conf | 5 -- .../main/java/org/ray/api/test/BaseTest.java | 21 +++--- .../api/test/MultiLanguageClusterTest.java | 5 +- .../java/org/ray/api/test/RayConfigTest.java | 14 +--- java/tutorial/README.rst | 2 +- java/tutorial/src/main/resources/ray.conf | 3 +- 14 files changed, 129 insertions(+), 102 deletions(-) diff --git a/.gitignore b/.gitignore index da464a65266a..af7f82d066cd 100644 --- a/.gitignore +++ b/.gitignore @@ -148,6 +148,7 @@ java/**/lib java/**/.settings java/**/.classpath java/**/.project +java/runtime/bin/ # python virtual env venv diff --git a/CMakeLists.txt b/CMakeLists.txt index 47f7b2ebe677..ca985be8b08a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -152,8 +152,23 @@ if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES") get_raylet_library("java" RAYLET_LIBRARY_JAVA) add_dependencies(copy_ray ${RAYLET_LIBRARY_JAVA}) - # copy libplasma_java files + # Copy java binary dependencies. add_custom_command(TARGET copy_ray POST_BUILD - COMMAND bash -c "mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/src/plasma" - COMMAND bash -c "cp ${ARROW_LIBRARY_DIR}/libplasma_java.* ${CMAKE_CURRENT_BINARY_DIR}/src/plasma/") + COMMAND mkdir -p ${CMAKE_SOURCE_DIR}/java/runtime/bin) + set(java_binary_dependencies + "src/ray/thirdparty/redis/src/redis-server" + "src/ray/gcs/redis_module/libray_redis_module.so" + "src/ray/raylet/raylet" + "src/ray/raylet/libraylet_library_java.*") + foreach(file ${java_binary_dependencies}) + add_custom_command(TARGET copy_ray POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/${file} + ${CMAKE_SOURCE_DIR}/java/runtime/bin) + endforeach() +add_custom_command(TARGET copy_ray POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy ${ARROW_HOME}/bin/plasma_store_server + ${CMAKE_SOURCE_DIR}/java/runtime/bin) + add_custom_command(TARGET copy_ray POST_BUILD + COMMAND $(CMAKE_COMMAND) -E copy ${ARROW_LIBRARY_DIR}/libplasma_java.* + ${CMAKE_SOURCE_DIR}/java/runtime/bin) endif() diff --git a/java/README.rst b/java/README.rst index e01616935787..f774ce57efaa 100644 --- a/java/README.rst +++ b/java/README.rst @@ -5,7 +5,7 @@ Configuration ------------- Ray will read your configurations in the following order: -* Java system properties: e.g., ``-Dray.home=/path/to/ray``. +* Java system properties: e.g., ``-Dray.run-mode=SINGLE_PROCESS``. * A ``ray.conf`` file in the classpath: `example `_. * Customise your own ``ray.conf`` path using system property ``-Dray.config=/path/to/ray.conf`` diff --git a/java/example.conf b/java/example.conf index ca42b336167f..e6aaa37ea8b7 100644 --- a/java/example.conf +++ b/java/example.conf @@ -6,11 +6,6 @@ # For config file format, see 'https://github.com/lightbend/config/blob/master/HOCON.md'. ray { - // This is the path to the directory where Ray is installed, e.g., - // something like /home/ubmutu/ray. This can be an absolute path or - // a relative path from the current working directory. - home = "/path/to/your/ray/home" - // Run mode, available options are: // // `SINGLE_PROCESS`: Ray is running in one single Java process, without Raylet backend, diff --git a/java/runtime/pom.xml b/java/runtime/pom.xml index 8e4fd8059594..7c5052c86486 100644 --- a/java/runtime/pom.xml +++ b/java/runtime/pom.xml @@ -79,6 +79,14 @@ + + + src/main/resources + + + bin + + org.apache.maven.plugins diff --git a/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java b/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java index 4c070ed88d7b..65afa70534e5 100644 --- a/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java +++ b/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java @@ -2,14 +2,19 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import java.io.File; import java.lang.reflect.Field; import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.SystemUtils; import org.ray.api.Checkpointable.Checkpoint; import org.ray.api.id.UniqueId; import org.ray.runtime.config.RayConfig; @@ -72,11 +77,28 @@ private void resetLibraryPath() { @Override public void start() throws Exception { - // Load native libraries. try { + // Reset library path at runtime. resetLibraryPath(); - System.loadLibrary("raylet_library_java"); - System.loadLibrary("plasma_java"); + + // Load native libraries. + String[] libraries = new String[]{"raylet_library_java", "plasma_java"}; + for (String library : libraries) { + String fileName = "lib" + library; + if (SystemUtils.IS_OS_LINUX) { + fileName += ".so"; + } else if (SystemUtils.IS_OS_MAC) { + fileName += ".dylib"; + } else { + throw new RuntimeException("Unsupported OS: " + System.getProperty("os.name")); + } + // Copy the file from resources to a temp dir, and load the native library. + File file = File.createTempFile(fileName, ""); + file.deleteOnExit(); + Files.copy(RayNativeRuntime.class.getResourceAsStream("/" + fileName), + Paths.get(file.getAbsolutePath()), StandardCopyOption.REPLACE_EXISTING); + System.load(file.getAbsolutePath()); + } } catch (Exception e) { LOGGER.error("Failed to load native libraries.", e); throw e; diff --git a/java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java b/java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java index 6aab89764930..079b9538f7cd 100644 --- a/java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java +++ b/java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java @@ -30,7 +30,6 @@ public class RayConfig { public static final String DEFAULT_CONFIG_FILE = "ray.default.conf"; public static final String CUSTOM_CONFIG_FILE = "ray.conf"; - public final String rayHome; public final String nodeIp; public final WorkerMode workerMode; public final RunMode runMode; @@ -56,10 +55,6 @@ public class RayConfig { public final String rayletSocketName; public final List rayletConfigParameters; - public final String redisServerExecutablePath; - public final String redisModulePath; - public final String plasmaStoreExecutablePath; - public final String rayletExecutablePath; public final String driverResourcePath; public final String pythonWorkerCommand; @@ -72,9 +67,6 @@ private void validate() { if (workerMode == WorkerMode.WORKER) { Preconditions.checkArgument(redisAddress != null, "Redis address must be set in worker mode."); - } else { - Preconditions.checkArgument(!rayHome.isEmpty(), - "'ray.home' must be set in driver mode"); } } @@ -87,32 +79,24 @@ private String removeTrailingSlash(String path) { } public RayConfig(Config config) { - // worker mode + // Worker mode. WorkerMode localWorkerMode; try { localWorkerMode = config.getEnum(WorkerMode.class, "ray.worker.mode"); } catch (ConfigException.Missing e) { localWorkerMode = WorkerMode.DRIVER; } - workerMode = localWorkerMode; boolean isDriver = workerMode == WorkerMode.DRIVER; - // run mode + // Run mode. runMode = config.getEnum(RunMode.class, "ray.run-mode"); - // ray home - String localRayHome = config.getString("ray.home"); - if (!localRayHome.startsWith("/")) { - // If ray.home isn't an absolute path, prepend it with current work dir. - localRayHome = System.getProperty("user.dir") + "/" + localRayHome; - } - rayHome = removeTrailingSlash(localRayHome); - // node ip + // Node ip. String nodeIp = config.getString("ray.node-ip"); if (nodeIp.isEmpty()) { nodeIp = NetworkUtil.getIpAddress(null); } this.nodeIp = nodeIp; - // resources + // Resources. resources = ResourceUtil.getResourcesMapFromString( config.getString("ray.resources")); if (isDriver) { @@ -127,22 +111,22 @@ public RayConfig(Config config) { resources.put("GPU", 0.0); } } - // driver id + // Driver id. String driverId = config.getString("ray.driver.id"); if (!driverId.isEmpty()) { this.driverId = UniqueId.fromHexString(driverId); } else { this.driverId = UniqueId.randomId(); } - // log dir + // Log dir. logDir = removeTrailingSlash(config.getString("ray.log-dir")); - // redirect output + // Redirect output. redirectOutput = config.getBoolean("ray.redirect-output"); - // custom library path - List customLibraryPath = config.getStringList("ray.library.path"); - // custom classpath + // Library path. + libraryPath = config.getStringList("ray.library.path"); + // Custom classpath. classpath = config.getStringList("ray.classpath"); - // custom worker jvm parameters + // Custom worker jvm parameters. if (config.hasPath("ray.worker.jvm-parameters")) { jvmParameters = config.getStringList("ray.worker.jvm-parameters"); } else { @@ -155,7 +139,7 @@ public RayConfig(Config config) { pythonWorkerCommand = null; } - // redis configurations + // Redis configurations. String redisAddress = config.getString("ray.redis.address"); if (!redisAddress.isEmpty()) { setRedisAddress(redisAddress); @@ -167,34 +151,22 @@ public RayConfig(Config config) { headRedisPassword = config.getString("ray.redis.head-password"); redisPassword = config.getString("ray.redis.password"); - // object store configurations + // Object store configurations. objectStoreSocketName = config.getString("ray.object-store.socket-name"); objectStoreSize = config.getBytes("ray.object-store.size"); - // raylet socket name + // Raylet socket name. rayletSocketName = config.getString("ray.raylet.socket-name"); - // raylet parameters - rayletConfigParameters = new ArrayList(); + // Raylet parameters. + rayletConfigParameters = new ArrayList<>(); Config rayletConfig = config.getConfig("ray.raylet.config"); for (Map.Entry entry : rayletConfig.entrySet()) { - String parameter = entry.getKey() + "," + String.valueOf(entry.getValue().unwrapped()); + String parameter = entry.getKey() + "," + entry.getValue().unwrapped(); rayletConfigParameters.add(parameter); } - // library path - this.libraryPath = new ImmutableList.Builder().add( - rayHome + "/build/src/plasma", - rayHome + "/build/src/ray/raylet" - ).addAll(customLibraryPath).build(); - - redisServerExecutablePath = rayHome + - "/build/src/ray/thirdparty/redis/src/redis-server"; - redisModulePath = rayHome + "/build/src/ray/gcs/redis_module/libray_redis_module.so"; - plasmaStoreExecutablePath = rayHome + "/build/src/plasma/plasma_store_server"; - rayletExecutablePath = rayHome + "/build/src/ray/raylet/raylet"; - - // driver resource path + // Driver resource path. if (config.hasPath("ray.driver.resource-path")) { driverResourcePath = config.getString("ray.driver.resource-path"); } else { @@ -204,7 +176,7 @@ public RayConfig(Config config) { // Number of threads that execute tasks. numberExecThreadsForDevRuntime = config.getInt("ray.dev-runtime.execution-parallelism"); - // validate config + // Validate config. validate(); LOGGER.debug("Created config: {}", this); } @@ -235,7 +207,6 @@ public Integer getRedisPort() { @Override public String toString() { return "RayConfig{" - + "rayHome='" + rayHome + '\'' + ", nodeIp='" + nodeIp + '\'' + ", workerMode=" + workerMode + ", runMode=" + runMode @@ -255,10 +226,6 @@ public String toString() { + ", objectStoreSize=" + objectStoreSize + ", rayletSocketName='" + rayletSocketName + '\'' + ", rayletConfigParameters=" + rayletConfigParameters - + ", redisServerExecutablePath='" + redisServerExecutablePath + '\'' - + ", redisModulePath='" + redisModulePath + '\'' - + ", plasmaStoreExecutablePath='" + plasmaStoreExecutablePath + '\'' - + ", rayletExecutablePath='" + rayletExecutablePath + '\'' + ", driverResourcePath='" + driverResourcePath + '\'' + ", pythonWorkerCommand='" + pythonWorkerCommand + '\'' + '}'; diff --git a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java index f0f1df8befa9..bec4eea591e8 100644 --- a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java +++ b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java @@ -5,9 +5,14 @@ import com.google.common.collect.Lists; import java.io.File; import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; @@ -42,10 +47,13 @@ public class RunManager { private static final int KILL_PROCESS_WAIT_TIMEOUT_SECONDS = 1; + private final Map tempFiles; + public RunManager(RayConfig rayConfig) { this.rayConfig = rayConfig; processes = new ArrayList<>(); random = new Random(); + tempFiles = new HashMap<>(); } public void cleanup() { @@ -61,7 +69,7 @@ public void cleanup() { p.waitFor(KILL_PROCESS_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); } catch (InterruptedException e) { LOGGER.warn("Got InterruptedException while waiting for process {}" + - " to be terminated.", processes.get(i)); + " to be terminated.", processes.get(i)); } if (p.isAlive()) { @@ -76,8 +84,30 @@ private void createTempDirs() { FileUtil.mkDir(new File(rayConfig.objectStoreSocketName).getParentFile()); } + /** + * Copy a file from resources to a temp dir, and return the file object. + */ + private File getTempFile(String fileName) { + File file = tempFiles.get(fileName); + if (file == null) { + try { + file = File.createTempFile(fileName, ""); + file.deleteOnExit(); + try (InputStream in = RunManager.class.getResourceAsStream(fileName)) { + Files.copy(in, Paths.get(file.getCanonicalPath()), StandardCopyOption.REPLACE_EXISTING); + } + file.setExecutable(true); + } catch (IOException e) { + throw new RuntimeException("Couldn't get temp file " + fileName, e); + } + tempFiles.put(fileName, file); + } + return file; + } + /** * Start a process. + * * @param command The command to start the process with. * @param env Environment variables. * @param name Process name. @@ -126,6 +156,7 @@ private void startProcess(List command, Map env, String /** * Start all Ray processes on this node. + * * @param isHead Whether this node is the head node. If true, redis server will be started. */ public void startRayProcesses(boolean isHead) { @@ -171,7 +202,8 @@ private void startRedisServer() { private String startRedisInstance(String ip, int port, String password, Integer shard) { List command = Lists.newArrayList( - rayConfig.redisServerExecutablePath, + // The redis-server executable file. + getTempFile("/redis-server").getAbsolutePath(), "--protected-mode", "no", "--port", @@ -179,7 +211,8 @@ private String startRedisInstance(String ip, int port, String password, Integer "--loglevel", "warning", "--loadmodule", - rayConfig.redisModulePath + // The redis module file. + getTempFile("/libray_redis_module.so").getAbsolutePath() ); if (!StringUtil.isNullOrEmpty(password)) { @@ -216,7 +249,8 @@ private void startRaylet() { // See `src/ray/raylet/main.cc` for the meaning of each parameter. List command = ImmutableList.of( - rayConfig.rayletExecutablePath, + // The raylet executable file. + getTempFile("/raylet").getAbsolutePath(), rayConfig.rayletSocketName, rayConfig.objectStoreSocketName, "0", // The object manager port. @@ -291,7 +325,8 @@ private String buildWorkerCommandRaylet() { private void startObjectStore() { List command = ImmutableList.of( - rayConfig.plasmaStoreExecutablePath, + // The plasma store executable file. + getTempFile("/plasma_store_server").getAbsolutePath(), "-s", rayConfig.objectStoreSocketName, "-m", diff --git a/java/runtime/src/main/resources/ray.default.conf b/java/runtime/src/main/resources/ray.default.conf index 5faeda7cfedf..a2762e76dd60 100644 --- a/java/runtime/src/main/resources/ray.default.conf +++ b/java/runtime/src/main/resources/ray.default.conf @@ -7,11 +7,6 @@ ray { // Basic configurations // ---------------------- - // This is the path to the directory where Ray is installed, e.g., - // something like /home/ubmutu/ray. This can be an absolute path or - // a relative path from the current working directory. - home: "" - // IP of this node. if not provided, IP will be automatically detected. node-ip: "" diff --git a/java/test/src/main/java/org/ray/api/test/BaseTest.java b/java/test/src/main/java/org/ray/api/test/BaseTest.java index b67a8f64c7ce..04bc36ba9794 100644 --- a/java/test/src/main/java/org/ray/api/test/BaseTest.java +++ b/java/test/src/main/java/org/ray/api/test/BaseTest.java @@ -1,6 +1,9 @@ package org.ray.api.test; +import com.google.common.collect.ImmutableList; +import java.io.File; import java.lang.reflect.Method; +import java.util.List; import org.ray.api.Ray; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,27 +14,29 @@ public class BaseTest { private static final Logger LOGGER = LoggerFactory.getLogger(BaseTest.class); + private List filesToDelete; + @BeforeMethod public void setUpBase(Method method) { LOGGER.info("===== Running test: " + method.getDeclaringClass().getName() + "." + method.getName()); - System.setProperty("ray.home", "../.."); System.setProperty("ray.resources", "CPU:4,RES-A:4"); Ray.init(); + // Delete socket files in tear-down. + filesToDelete = ImmutableList.of(Ray.getRuntimeContext().getRayletSocketName(), + Ray.getRuntimeContext().getObjectStoreSocketName()); } @AfterMethod public void tearDownBase() { - // TODO(qwang): This is double check to check that the socket file is removed actually. - // We could not enable this until `systemInfo` enabled. - //File rayletSocketFIle = new File(Ray.systemInfo().rayletSocketName()); + Ray.shutdown(); - //remove raylet socket file - //rayletSocketFIle.delete(); + for (String file : filesToDelete) { + new File(file).delete(); + } - // unset system properties - System.clearProperty("ray.home"); + // Unset system properties. System.clearProperty("ray.resources"); } diff --git a/java/test/src/main/java/org/ray/api/test/MultiLanguageClusterTest.java b/java/test/src/main/java/org/ray/api/test/MultiLanguageClusterTest.java index c81a148980f9..ad3033681a68 100644 --- a/java/test/src/main/java/org/ray/api/test/MultiLanguageClusterTest.java +++ b/java/test/src/main/java/org/ray/api/test/MultiLanguageClusterTest.java @@ -66,8 +66,7 @@ public void setUp(Method method) { // Start ray cluster. String testDir = System.getProperty("user.dir"); - String workerOptions = String.format("-Dray.home=%s/../../", testDir); - workerOptions += + String workerOptions = " -classpath " + String.format("%s/../../build/java/*:%s/target/*", testDir, testDir); final List startCommand = ImmutableList.of( "ray", @@ -85,7 +84,6 @@ public void setUp(Method method) { } // Connect to the cluster. - System.setProperty("ray.home", "../.."); System.setProperty("ray.redis.address", "127.0.0.1:6379"); System.setProperty("ray.object-store.socket-name", PLASMA_STORE_SOCKET_NAME); System.setProperty("ray.raylet.socket-name", RAYLET_SOCKET_NAME); @@ -96,7 +94,6 @@ public void setUp(Method method) { public void tearDown() { // Disconnect to the cluster. Ray.shutdown(); - System.clearProperty("ray.home"); System.clearProperty("ray.redis.address"); System.clearProperty("ray.object-store.socket-name"); System.clearProperty("ray.raylet.socket-name"); diff --git a/java/test/src/main/java/org/ray/api/test/RayConfigTest.java b/java/test/src/main/java/org/ray/api/test/RayConfigTest.java index c728eda49eee..26702ebe4c28 100644 --- a/java/test/src/main/java/org/ray/api/test/RayConfigTest.java +++ b/java/test/src/main/java/org/ray/api/test/RayConfigTest.java @@ -10,24 +10,12 @@ public class RayConfigTest { @Test public void testCreateRayConfig() { try { - System.setProperty("ray.home", "/path/to/ray"); System.setProperty("ray.driver.resource-path", "path/to/ray/driver/resource/path"); RayConfig rayConfig = RayConfig.create(); - - Assert.assertEquals("/path/to/ray", rayConfig.rayHome); Assert.assertEquals(WorkerMode.DRIVER, rayConfig.workerMode); - - System.setProperty("ray.home", ""); - rayConfig = RayConfig.create(); - - Assert.assertEquals(System.getProperty("user.dir"), rayConfig.rayHome); - Assert.assertEquals(System.getProperty("user.dir") + - "/build/src/ray/thirdparty/redis/src/redis-server", rayConfig.redisServerExecutablePath); - Assert.assertEquals("path/to/ray/driver/resource/path", rayConfig.driverResourcePath); } finally { - //unset the system property - System.clearProperty("ray.home"); + // Unset system properties. System.clearProperty("ray.driver.resource-path"); } diff --git a/java/tutorial/README.rst b/java/tutorial/README.rst index e099ad25badc..01430605834e 100644 --- a/java/tutorial/README.rst +++ b/java/tutorial/README.rst @@ -12,7 +12,7 @@ To run them, execute the following command under ``ray/java`` folder. .. code-block:: shell - java -Dray.home=.. -classpath "tutorial/target/ray-tutorial-1.0.jar:tutorial/lib/*" org.ray.exercise.Exercise01 + java -classpath "tutorial/target/ray-tutorial-1.0.jar:tutorial/lib/*" org.ray.exercise.Exercise01 `Exercise 1 `_: Define a remote function, and execute multiple remote functions in parallel. diff --git a/java/tutorial/src/main/resources/ray.conf b/java/tutorial/src/main/resources/ray.conf index a6cdf42f8f15..e79b1d232982 100644 --- a/java/tutorial/src/main/resources/ray.conf +++ b/java/tutorial/src/main/resources/ray.conf @@ -1,5 +1,4 @@ -ray{ - home: ".." +ray { run-mode: CLUSTER redirect-output: false } From d4707f23820b6bca9d9c14d803837ab55bf65a9b Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Thu, 14 Mar 2019 21:57:09 +0800 Subject: [PATCH 034/115] format --- java/runtime/pom.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/java/runtime/pom.xml b/java/runtime/pom.xml index 7c5052c86486..6b6239884cc1 100644 --- a/java/runtime/pom.xml +++ b/java/runtime/pom.xml @@ -79,14 +79,14 @@ - - - src/main/resources - - - bin - - + + + src/main/resources + + + bin + + org.apache.maven.plugins From f1f6e21b5c9ab9ddb1ab880c79dc3e385ec855f3 Mon Sep 17 00:00:00 2001 From: Yuhong Guo Date: Fri, 15 Mar 2019 12:11:45 +0800 Subject: [PATCH 035/115] add @boost//:asio to logging_test --- BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/BUILD.bazel b/BUILD.bazel index 2fcc49f32141..b0331298cb27 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -99,6 +99,7 @@ cc_test( args = ["--gtest_filter=PrintLogTest*"], copts = COPTS, deps = [ + "@boost//:asio", ":ray_util", "@com_google_googletest//:gtest_main", ], From ae02b6ad66b66d460b17099c4d7f125bfda35878 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Fri, 15 Feb 2019 18:05:19 +0800 Subject: [PATCH 036/115] Refine JNI bazel script to make it suitable for more systems --- BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILD.bazel b/BUILD.bazel index b597939896b9..d093db5eebe0 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -396,8 +396,8 @@ cc_binary( linkshared = 1, linkstatic = 0, deps = [ - "//:raylet_lib", "@plasma//:plasma_client", + "//:raylet_lib" ], ) From 6236d594f579ee62fe86f1b5709d0afec6c9d0b2 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Wed, 6 Mar 2019 02:47:14 +0800 Subject: [PATCH 037/115] java bazel config --- BUILD.bazel | 45 +- WORKSPACE | 4 + bazel/BUILD.plasma | 31 +- bazel/ray.bzl | 12 + bazel/ray_java.bzl | 1577 ++++++++++++++++++++++++++++++++++++++++++++ build.sh | 23 +- java/BUILD.bazel | 211 ++++++ java/build.sh | 6 +- java/test.sh | 48 +- java/testng.xml | 11 + 10 files changed, 1933 insertions(+), 35 deletions(-) create mode 100644 bazel/ray_java.bzl create mode 100644 java/BUILD.bazel create mode 100644 java/testng.xml diff --git a/BUILD.bazel b/BUILD.bazel index d093db5eebe0..f06e13147d83 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -372,7 +372,7 @@ pyx_library( ) cc_binary( - name = "raylet_library_java.so", + name = "libraylet_library_java.so", srcs = [ "src/ray/raylet/lib/java/org_ray_runtime_raylet_RayletClientImpl.h", "src/ray/raylet/lib/java/org_ray_runtime_raylet_RayletClientImpl.cc", @@ -403,8 +403,8 @@ cc_binary( genrule( name = "raylet-jni-darwin-compat", - srcs = [":raylet_library_java.so"], - outs = ["raylet_library_java.dylib"], + srcs = [":libraylet_library_java.so"], + outs = ["libraylet_library_java.dylib"], cmd = "cp $< $@", output_to_bindir = 1, ) @@ -412,16 +412,22 @@ genrule( filegroup( name = "raylet_library_java", srcs = select({ - "@bazel_tools//src/conditions:darwin": [":raylet_library_java.dylib"], - "//conditions:default": [":raylet_library_java.so"], + "@bazel_tools//src/conditions:darwin": [":libraylet_library_java.dylib"], + "//conditions:default": [":libraylet_library_java.so"], }), visibility = ["//visibility:public"], ) +filegroup( + name = "gcs_fbs_file", + srcs = ["src/ray/gcs/format/gcs.fbs"], + visibility = ["//visibility:public"], +) + flatbuffer_py_library( name = "python_gcs_fbs", srcs = [ - "src/ray/gcs/format/gcs.fbs", + ":gcs_fbs_file", ], outs = [ "ActorCheckpointIdData.py", @@ -578,3 +584,30 @@ genrule( mv python $(location ray_pkg) """, ) + +genrule( + name = "ray_java_pkg", + srcs = [ + "//:redis-server", + "//:ray_redis_module.so", + "//:raylet", + "//:raylet_library_java", + "@plasma//:plasma_store_server", + "@plasma//:plasma_client_java", + ], + outs = ["ray_java_pkg"], + cmd = """ + set -x && + mkdir -p java/ray/src/plasma/ && + cp $(location @plasma//:plasma_client_java) java/ray/src/plasma/ && + cp $(location @plasma//:plasma_store_server) java/ray/src/plasma/ && + mkdir -p java/ray/src/ray/raylet/ && + cp $(location //:raylet) java/ray/src/ray/raylet/ && + cp $(location //:raylet_library_java) java/ray/src/ray/raylet/ && + mkdir -p java/ray/src/ray/gcs/redis_module/ && + cp $(location //:ray_redis_module.so) java/ray/src/ray/gcs/redis_module/libray_redis_module.so && + mkdir -p java/ray/src/ray/thirdparty/redis/src/ && + cp $(location //:redis-server) java/ray/src/ray/thirdparty/redis/src/ && + mv java $(location ray_java_pkg) + """, +) diff --git a/WORKSPACE b/WORKSPACE index 450bc6a65567..58bf0dfb0bc8 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,4 +1,8 @@ +workspace(name = "com_github_riselab_ray") + load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository") +load("//bazel:ray_java.bzl", "generated_maven_jars") +generated_maven_jars() git_repository( name = "com_github_nelhage_rules_boost", diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index 33353caaf6f7..16960674726b 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -1,4 +1,25 @@ load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library") +load("@com_github_riselab_ray//bazel:ray_java.bzl", "generated_java_libraries") + +generated_java_libraries() + +java_library( + name = "org_apache_arrow_arrow_plasma", + srcs = glob(["java/plasma/src/main/java/**/*.java"]), + visibility = ["//visibility:public"], + data = [":plasma_client_java"], + deps = [ + "@org_slf4j_slf4j_api//jar", + ], +) + +java_binary( + name = "org_apache_arrow_arrow_plasma_test", + srcs = ["java/plasma/src/test/java/org/apache/arrow/plasma/PlasmaClientTest.java"], + main_class = "org.apache.arrow.plasma.PlasmaClientTest", + visibility = ["//visibility:public"], + deps = [":org_apache_arrow_arrow_plasma", "@junit_junit//jar",], +) cc_library( name = "arrow", @@ -65,7 +86,7 @@ cc_library( ) cc_binary( - name = "plasma_client_java.so", + name = "libplasma_java.so", srcs = [ "cpp/src/plasma/lib/java/org_apache_arrow_plasma_PlasmaClientJNI.cc", "cpp/src/plasma/lib/java/org_apache_arrow_plasma_PlasmaClientJNI.h", @@ -101,8 +122,8 @@ genrule( genrule( name = "plasma-jni-darwin-compat", - srcs = [":plasma_client_java.so"], - outs = ["plasma_client_java.dylib"], + srcs = [":libplasma_java.so"], + outs = ["libplasma_java.dylib"], cmd = "cp $< $@", output_to_bindir = 1, ) @@ -110,8 +131,8 @@ genrule( filegroup( name = "plasma_client_java", srcs = select({ - "@bazel_tools//src/conditions:darwin": [":plasma_client_java.dylib"], - "//conditions:default": [":plasma_client_java.so"], + "@bazel_tools//src/conditions:darwin": [":libplasma_java.dylib"], + "//conditions:default": [":libplasma_java.so"], }), visibility = ["//visibility:public"], ) diff --git a/bazel/ray.bzl b/bazel/ray.bzl index 4a1e4f1063e4..2f68b25127b6 100644 --- a/bazel/ray.bzl +++ b/bazel/ray.bzl @@ -10,3 +10,15 @@ def flatbuffer_py_library(name, srcs, outs, out_prefix, includes = [], include_p include_paths = include_paths, includes = includes, ) + +def flatbuffer_java_library(name, srcs, outs, out_prefix, includes = [], include_paths = []): + flatbuffer_library_public( + name = name, + srcs = srcs, + outs = outs, + language_flag = "-j", + out_prefix = out_prefix, + include_paths = include_paths, + includes = includes, + ) + diff --git a/bazel/ray_java.bzl b/bazel/ray_java.bzl new file mode 100644 index 000000000000..c66feda3f227 --- /dev/null +++ b/bazel/ray_java.bzl @@ -0,0 +1,1577 @@ +# The following dependencies were calculated from: +# +# generate_workspace --maven_project=/home/admin/ruifang.crf/code/X/arrow/java --repositories=http://mvn.dev.alipay.net:8080/artifactory/repo + + +def generated_maven_jars(): + # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT + # pom.xml got requested version + # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 wanted version 1.1.2 + # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 wanted version 1.1.2 + # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "ch_qos_logback_logback_classic", + artifact = "ch.qos.logback:logback-classic:1.2.3", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "7c4f3c474fb2c041d8028740440937705ebb473a", + ) + + + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "com_google_code_findbugs_jsr305", + artifact = "com.google.code.findbugs:jsr305:3.0.2", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "25ea2e8b0c338a877313bd4672d3fe056ea78f0d", + ) + + + # io.grpc:grpc-protobuf:jar:1.14.0 + native.maven_jar( + name = "io_grpc_grpc_protobuf_lite", + artifact = "io.grpc:grpc-protobuf-lite:1.14.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "2ac8c28ca927f954eaa228a931d9c163cf3d860f", + ) + + + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT wanted version 2.10 + native.maven_jar( + name = "joda_time_joda_time", + artifact = "joda-time:joda-time:2.9.9", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "f7b520c458572890807d143670c9b24f4de90897", + ) + + + # org.mockito:mockito-core:jar:2.7.22 + native.maven_jar( + name = "net_bytebuddy_byte_buddy_agent", + artifact = "net.bytebuddy:byte-buddy-agent:1.6.11", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "0200d9c012befccd211ff91082a151257b1dc084", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 + native.maven_jar( + name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_converter_classic", + artifact = "de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:0.9.44", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "6f08f9bfd65869a51edc922432f46024d94f3ccd", + ) + + + # io.netty:netty-handler-proxy:jar:4.1.22.Final got requested version + # io.netty:netty-handler:jar:4.1.22.Final got requested version + # io.netty:netty-codec:jar:4.1.22.Final + native.maven_jar( + name = "io_netty_netty_transport", + artifact = "io.netty:netty-transport:4.1.22.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "3bd455cd9e5e5fb2e08fd9cd0acfa54c079ca989", + ) + + + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "com_carrotsearch_hppc", + artifact = "com.carrotsearch:hppc:0.7.2", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "710398361f2ae8fd594a133e3619045c16b24137", + ) + + + # io.netty:netty-codec-http2:jar:4.1.22.Final + native.maven_jar( + name = "io_netty_netty_handler", + artifact = "io.netty:netty-handler:4.1.22.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "a3a16b17d5a5ed6f784b0daba95e28d940356109", + ) + + + # io.grpc:grpc-netty:jar:1.14.0 + native.maven_jar( + name = "io_netty_netty_handler_proxy", + artifact = "io.netty:netty-handler-proxy:4.1.27.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "1a822ce7760bc6eb4937b7e448c9e081fedcc807", + ) + + + # org.apache.arrow:arrow-java-root:pom:0.12.0HOT + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "org_mockito_mockito_core", + artifact = "org.mockito:mockito-core:2.7.22", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "fcf63bc8010ca77991e3cadd8d33ad1a40603404", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-core:jar:0.9.44 got requested version + # de.huxhorn.lilith:de.huxhorn.lilith.data.logging.protobuf:jar:0.9.44 got requested version + # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 + # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 got requested version + native.maven_jar( + name = "de_huxhorn_sulky_de_huxhorn_sulky_codec", + artifact = "de.huxhorn.sulky:de.huxhorn.sulky.codec:0.9.17", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "4a2a917f1cc4b75d71aadeef4f4c178f229febc2", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.data.logging:jar:0.9.44 + native.maven_jar( + name = "de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", + artifact = "de.huxhorn.lilith:de.huxhorn.lilith.data.eventsource:0.9.44", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "618f853d75d59855c955b3a1bf0f447476f8027f", + ) + + + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "com_google_flatbuffers_flatbuffers_java", + artifact = "com.google.flatbuffers:flatbuffers-java:1.9.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "460bc8f6411768659c1ffb529592e251a808b9f2", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version + # de.huxhorn.lilith:de.huxhorn.lilith.data.logging.protobuf:jar:0.9.44 wanted version 2.5.0 + # io.grpc:grpc-protobuf:jar:1.14.0 + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT wanted version 2.5.0 + native.maven_jar( + name = "com_google_protobuf_protobuf_java", + artifact = "com.google.protobuf:protobuf-java:3.5.1", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "8c3492f7662fa1cbf8ca76a0f5eb1146f7725acd", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "io_netty_netty_tcnative_boringssl_static", + artifact = "io.netty:netty-tcnative-boringssl-static:2.0.12.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "b884be1450a7fd0854b98743836b8ccb0dfd75a4", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "com_h2database_h2", + artifact = "com.h2database:h2:1.4.196", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "dd0034398d593aa3588c6773faac429bbd9aea0e", + ) + + + # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "org_slf4j_jul_to_slf4j", + artifact = "org.slf4j:jul-to-slf4j:1.7.25", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "0af5364cd6679bfffb114f0dec8a157aaa283b76", + ) + + + native.maven_jar( + name = "net_java_dev_jna_jna", + artifact = "net.java.dev.jna:jna:4.1.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "1c12d070e602efd8021891cdd7fd18bc129372d4", + ) + + + native.maven_jar( + name = "org_mockito_mockito_all", + artifact = "org.mockito:mockito-all:1.10.19", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "539df70269cc254a58cccc5d8e43286b4a73bf30", + ) + + + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "commons_codec_commons_codec", + artifact = "commons-codec:commons-codec:1.10", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "4b95f4897fa13f2cd904aee711aeafc0c5295cd8", + ) + + + native.maven_jar( + name = "com_beust_jcommander", + artifact = "com.beust:jcommander:1.72", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "6375e521c1e11d6563d4f25a07ce124ccf8cd171", + ) + + + # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "org_slf4j_log4j_over_slf4j", + artifact = "org.slf4j:log4j-over-slf4j:1.7.25", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "a87bb47468f47ee7aabbd54f93e133d4215769c3", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "commons_cli_commons_cli", + artifact = "commons-cli:commons-cli:1.4", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "c51c00206bb913cd8612b24abd9fa98ae89719b1", + ) + + + # io.grpc:grpc-core:jar:1.14.0 + native.maven_jar( + name = "com_google_errorprone_error_prone_annotations", + artifact = "com.google.errorprone:error_prone_annotations:2.1.2", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "6dcc08f90f678ac33e5ef78c3c752b6f59e63e0c", + ) + + + # io.netty:netty-transport:jar:4.1.22.Final + native.maven_jar( + name = "io_netty_netty_resolver", + artifact = "io.netty:netty-resolver:4.1.22.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "b5484d17a97cb57b07d2a1ac092c249e47234c17", + ) + + + # io.grpc:grpc-core:jar:1.14.0 + native.maven_jar( + name = "io_opencensus_opencensus_contrib_grpc_metrics", + artifact = "io.opencensus:opencensus-contrib-grpc-metrics:0.12.3", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "a4c7ff238a91b901c8b459889b6d0d7a9d889b4d", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "io_grpc_grpc_stub", + artifact = "io.grpc:grpc-stub:1.14.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "74bfe83c0dc69bf903fff8df3568cbeb8b387d35", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "io_grpc_grpc_protobuf", + artifact = "io.grpc:grpc-protobuf:1.14.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "d429fdc2e0d288b34ea7588bb762eb458f385bd5", + ) + + + # io.netty:netty-handler-proxy:jar:4.1.22.Final + native.maven_jar( + name = "io_netty_netty_codec_socks", + artifact = "io.netty:netty-codec-socks:4.1.22.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "d077b39da2dedc5dc5db50a44e5f4c30353e86f3", + ) + + + # io.netty:netty-codec-http:jar:4.1.22.Final + # io.netty:netty-handler:jar:4.1.22.Final got requested version + # io.netty:netty-codec-socks:jar:4.1.22.Final got requested version + native.maven_jar( + name = "io_netty_netty_codec", + artifact = "io.netty:netty-codec:4.1.22.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "239c0af275952e70bb4adf7cf8c03d88ddc394c9", + ) + + + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version + # io.netty:netty-transport:jar:4.1.22.Final got requested version + # io.netty:netty-handler:jar:4.1.22.Final got requested version + native.maven_jar( + name = "io_netty_netty_buffer", + artifact = "io.netty:netty-buffer:4.1.22.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "15e964a2095031364f534a6e21977f5ee9ca32a9", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 + native.maven_jar( + name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_core", + artifact = "de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-core:0.9.44", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "636cc11e0e09d363af87766550c1deef8ab0c5bf", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "io_grpc_grpc_netty", + artifact = "io.grpc:grpc-netty:1.14.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "21c6edadd45b6869384f8aa0df1663d62c503617", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-core:jar:0.9.44 wanted version 1.1.2 + # ch.qos.logback:logback-classic:jar:1.2.3 + native.maven_jar( + name = "ch_qos_logback_logback_core", + artifact = "ch.qos.logback:logback-core:1.2.3", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "864344400c3d4d92dfeb0a305dc87d953677c03c", + ) + + + # org.mockito:mockito-core:jar:2.7.22 + native.maven_jar( + name = "net_bytebuddy_byte_buddy", + artifact = "net.bytebuddy:byte-buddy:1.6.11", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "8a8f9409e27f1d62c909c7eef2aa7b3a580b4901", + ) + + + # io.opencensus:opencensus-contrib-grpc-metrics:jar:0.12.3 got requested version + # io.grpc:grpc-core:jar:1.14.0 + native.maven_jar( + name = "io_opencensus_opencensus_api", + artifact = "io.opencensus:opencensus-api:0.12.3", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "743f074095f29aa985517299545e72cc99c87de0", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-core:jar:0.9.44 + native.maven_jar( + name = "de_huxhorn_lilith_de_huxhorn_lilith_sender", + artifact = "de.huxhorn.lilith:de.huxhorn.lilith.sender:0.9.44", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "fdeb2f1aaba42cad76c460ac4fd7072b8642520f", + ) + + + native.maven_jar( + name = "org_javassist_javassist", + artifact = "org.javassist:javassist:3.19.0-GA", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "50120f69224dd8684b445a6f3a5b08fe9b5c60f6", + ) + + + # org.mockito:mockito-core:jar:2.7.22 + native.maven_jar( + name = "org_objenesis_objenesis", + artifact = "org.objenesis:objenesis:2.5", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "612ecb799912ccf77cba9b3ed8c813da086076e9", + ) + + + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT got requested version + # io.netty:netty-resolver:jar:4.1.22.Final got requested version + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT got requested version + # io.netty:netty-buffer:jar:4.1.22.Final + native.maven_jar( + name = "io_netty_netty_common", + artifact = "io.netty:netty-common:4.1.22.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "56ff4deca53fc791ed59ac2b72eb6718714a4de9", + ) + + + # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT wanted version 4.12 + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "junit_junit", + artifact = "junit:junit:4.11", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "4e031bb61df09069aeb2bffb4019e7a5034a4ee0", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 + native.maven_jar( + name = "de_huxhorn_lilith_de_huxhorn_lilith_data_logging_protobuf", + artifact = "de.huxhorn.lilith:de.huxhorn.lilith.data.logging.protobuf:0.9.44", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "56a037b365c7ff9b1c348e6d663fd3d778e2b0be", + ) + + + # de.huxhorn.sulky:de.huxhorn.sulky.codec:jar:0.9.17 + # de.huxhorn.lilith:de.huxhorn.lilith.sender:jar:0.9.44 got requested version + native.maven_jar( + name = "de_huxhorn_sulky_de_huxhorn_sulky_io", + artifact = "de.huxhorn.sulky:de.huxhorn.sulky.io:0.9.17", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "6baf1e17f80ba40f9a4b4b97235e72488047db02", + ) + + + native.maven_jar( + name = "log4j_log4j", + artifact = "log4j:log4j:1.2.17", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "5af35056b4d257e4b64b9e8069c0746e8b08629f", + ) + + + native.maven_jar( + name = "org_slf4j_slf4j_log4j12", + artifact = "org.slf4j:slf4j-log4j12:1.7.25", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "110cefe2df103412849d72ef7a67e4e91e4266b4", + ) + + + # org.slf4j:jcl-over-slf4j:jar:1.7.25 got requested version + # org.slf4j:jul-to-slf4j:jar:1.7.25 + # de.huxhorn.lilith:de.huxhorn.lilith.sender:jar:0.9.44 wanted version 1.7.7 + # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 wanted version 1.7.7 + # org.slf4j:log4j-over-slf4j:jar:1.7.25 got requested version + # ch.qos.logback:logback-classic:jar:1.2.3 got requested version + # de.huxhorn.sulky:de.huxhorn.sulky.codec:jar:0.9.17 wanted version 1.7.7 + # de.huxhorn.sulky:de.huxhorn.sulky.formatting:jar:0.9.17 wanted version 1.7.7 + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "org_slf4j_slf4j_api", + artifact = "org.slf4j:slf4j-api:1.7.25", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "da76ca59f6a57ee3102f8f9bd9cee742973efa8a", + ) + + + native.maven_jar( + name = "com_typesafe_config", + artifact = "com.typesafe:config:1.3.2", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "d6ac0ce079f114adce620f2360c92a70b2cb36dc", + ) + + + native.maven_jar( + name = "org_apache_commons_commons_lang3", + artifact = "org.apache.commons:commons-lang3:3.4", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "5fe28b9518e58819180a43a850fbc0dd24b7c050", + ) + + + # io.netty:netty-handler-proxy:jar:4.1.22.Final got requested version + # io.netty:netty-codec-http2:jar:4.1.22.Final + native.maven_jar( + name = "io_netty_netty_codec_http", + artifact = "io.netty:netty-codec-http:4.1.22.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "3805f3ca0d57630200defc7f9bb6ed3382dcb10b", + ) + + + # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOTi + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_classic", + artifact = "de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:0.9.44", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "1ac6ffe5fc63f7900f8af54b0dfebf2508b41db7", + ) + + + native.maven_jar( + name = "commons_collections_commons_collections", + artifact = "commons-collections:commons-collections:3.2.2", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "8ad72fe39fa8c91eaaf12aadb21e0c3661fe26d5", + ) + + + # junit:junit:jar:4.11 + native.maven_jar( + name = "org_hamcrest_hamcrest_core", + artifact = "org.hamcrest:hamcrest-core:1.3", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "42a25dc3219429f0e5d060061f71acb49bf010a0", + ) + + + # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "org_jmockit_jmockit", + artifact = "org.jmockit:jmockit:1.33", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "8a2746af90706d8d68f3919e1a87d5e47b8c94d7", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.data.logging:jar:0.9.44 + native.maven_jar( + name = "de_huxhorn_sulky_de_huxhorn_sulky_formatting", + artifact = "de.huxhorn.sulky:de.huxhorn.sulky.formatting:0.9.17", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "0501b143ee18ca9e7807438cc1b0acfa144f657f", + ) + + + # io.grpc:grpc-netty:jar:1.14.0 + native.maven_jar( + name = "io_netty_netty_codec_http2", + artifact = "io.netty:netty-codec-http2:4.1.27.Final", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "3769790a2033667d663f9a526d5b63cfecdbdf4e", + ) + + + native.maven_jar( + name = "net_lingala_zip4j_zip4j", + artifact = "net.lingala.zip4j:zip4j:1.3.2", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "4ba84e98ee017b74cb52f45962f929a221f3074c", + ) + + + native.maven_jar( + name = "org_testng_testng", + artifact = "org.testng:testng:6.9.9", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "1bf509349476d6a48978cc2b04af9caa907781ab", + ) + + native.maven_jar( + name = "org_ini4j_ini4j", + artifact = "org.ini4j:ini4j:0.5.4", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "4a3ee4146a90c619b20977d65951825f5675b560", + ) + + + native.maven_jar( + name = "org_ow2_asm_asm", + artifact = "org.ow2.asm:asm:6.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "bc6fa6b19424bb9592fe43bbc20178f92d403105", + ) + + + # io.grpc:grpc-core:jar:1.14.0 + native.maven_jar( + name = "com_google_code_gson_gson", + artifact = "com.google.code.gson:gson:2.7",#2.8.5 + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "751f548c85fa49f330cecbb1875893f971b33c4e", + ) + + + # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT + # pom.xml got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "org_slf4j_jcl_over_slf4j", + artifact = "org.slf4j:jcl-over-slf4j:1.7.25", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "f8c32b13ff142a513eeb5b6330b1588dcb2c0461", + ) + + + # pom.xml got requested version + # com.fasterxml.jackson.core:jackson-databind:bundle:2.7.9 wanted version 2.7.0 + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version + native.maven_jar( + name = "com_fasterxml_jackson_core_jackson_annotations", + artifact = "com.fasterxml.jackson.core:jackson-annotations:2.7.9", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "eb356e825cb73da42f7c902a3fe0276fe32b26c8", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.data.logging.protobuf:jar:0.9.44 got requested version + # de.huxhorn.lilith:de.huxhorn.lilith.logback.classic:jar:0.9.44 + # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 got requested version + native.maven_jar( + name = "de_huxhorn_lilith_de_huxhorn_lilith_data_logging", + artifact = "de.huxhorn.lilith:de.huxhorn.lilith.data.logging:0.9.44", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "3574a72628f59d78ff17210f10de057188043a4c", + ) + + + # io.grpc:grpc-core:jar:1.14.0 + native.maven_jar( + name = "io_grpc_grpc_context", + artifact = "io.grpc:grpc-context:1.14.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "77252b5f926875891aaae5629e6ab2ef968cd6c6", + ) + + + # io.grpc:grpc-protobuf:jar:1.14.0 + native.maven_jar( + name = "com_google_api_grpc_proto_google_common_protos", + artifact = "com.google.api.grpc:proto-google-common-protos:1.0.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "86f070507e28b930e50d218ee5b6788ef0dd05e6", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 + native.maven_jar( + name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_classic", + artifact = "de.huxhorn.lilith:de.huxhorn.lilith.logback.classic:0.9.44", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "f84da242a1988240222f6a06b511a18ff9c64344", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "com_fasterxml_jackson_core_jackson_databind", + artifact = "com.fasterxml.jackson.core:jackson-databind:2.7.9", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "a4c0b14c7dd85bdf4d25da074e90a10fa4b9b88b", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "com_fasterxml_jackson_dataformat_jackson_dataformat_yaml", + artifact = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.7.9", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "163a1bd63ace592d2a449023a1c07056b885ff12", + ) + + + native.maven_jar( + name = "com_github_davidmoten_flatbuffers_java", + artifact = "com.github.davidmoten:flatbuffers-java:1.9.0.1", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "0eaa1b27095bb5127879458bc2ee1b78e00d4a20", + ) + + + native.maven_jar( + name = "redis_clients_jedis", + artifact = "redis.clients:jedis:2.8.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "a9486e200b2195f31ce9b8779a70a5e2450a73ba", + ) + + + native.maven_jar( + name = "org_apache_commons_commons_pool2", + artifact = "org.apache.commons:commons-pool2:2.3", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "62a559a025fd890c30364296ece14643ba9c8c5b", + ) + + + native.maven_jar( + name = "commons_io_commons_io", + artifact = "commons-io:commons-io:2.5", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "2852e6e05fbb95076fc091f6d1780f1f8fe35e0f", + ) + + + # pom.xml got requested version + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT + # io.grpc:grpc-protobuf:jar:1.14.0 got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version + # io.grpc:grpc-protobuf-lite:jar:1.14.0 got requested version + # io.grpc:grpc-core:jar:1.14.0 + # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT wanted version 23.0 + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "com_google_guava_guava", + artifact = "com.google.guava:guava:20.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "89507701249388e1ed5ddcf8c41f4ce1be7831ef", + ) + + + # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 + native.maven_jar( + name = "de_huxhorn_lilith_de_huxhorn_lilith_data_converter", + artifact = "de.huxhorn.lilith:de.huxhorn.lilith.data.converter:0.9.44", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "31e4994f9bc768c43491c3c0d12237ae478d21e3", + ) + + + native.maven_jar( + name = "de_ruedigermoeller_fst", + artifact = "de.ruedigermoeller:fst:2.47", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "281d390ebed24a3621d3053825affc13d0eead8b", + ) + + + # pom.xml got requested version + # com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:bundle:2.7.9 got requested version + # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT + # com.fasterxml.jackson.core:jackson-databind:bundle:2.7.9 got requested version + # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version + # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT + native.maven_jar( + name = "com_fasterxml_jackson_core_jackson_core", + artifact = "com.fasterxml.jackson.core:jackson-core:2.7.9", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "09b530cec4fd2eb841ab8e79f19fc7cf0ec487b2", + ) + + + # pom.xml got requested version + # io.grpc:grpc-protobuf:jar:1.14.0 got requested version + # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version + # io.grpc:grpc-protobuf-lite:jar:1.14.0 got requested version + # io.grpc:grpc-stub:jar:1.14.0 got requested version + # io.grpc:grpc-netty:jar:1.14.0 + native.maven_jar( + name = "io_grpc_grpc_core", + artifact = "io.grpc:grpc-core:1.14.0", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "526e5be291c96e248789d769c108a084febda07f", + ) + + + # com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:bundle:2.7.9 + native.maven_jar( + name = "org_yaml_snakeyaml", + artifact = "org.yaml:snakeyaml:1.15", + repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", + sha1 = "3b132bea69e8ee099f416044970997bde80f4ea6", + ) + + + + +def generated_java_libraries(): + native.java_library( + name = "log4j_log4j", + visibility = ["//visibility:public"], + exports = ["@log4j_log4j//jar"], + ) + + + native.java_library( + name = "org_slf4j_slf4j_log4j12", + visibility = ["//visibility:public"], + exports = ["@org_slf4j_slf4j_log4j12//jar"], + runtime_deps = [ + ":log4j_log4j", + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "com_typesafe_config", + visibility = ["//visibility:public"], + exports = ["@com_typesafe_config//jar"], + ) + + + native.java_library( + name = "org_apache_commons_commons_lang3", + visibility = ["//visibility:public"], + exports = ["@org_apache_commons_commons_lang3//jar"], + ) + + + native.java_library( + name = "com_google_code_findbugs_jsr305", + visibility = ["//visibility:public"], + exports = ["@com_google_code_findbugs_jsr305//jar"], + ) + + + native.java_library( + name = "io_grpc_grpc_core", + visibility = ["//visibility:public"], + exports = ["@io_grpc_grpc_core//jar"], + runtime_deps = [ + ":com_google_code_findbugs_jsr305", + ":com_google_code_gson_gson", + ":com_google_errorprone_error_prone_annotations", + ":com_google_guava_guava", + ":io_grpc_grpc_context", + ":io_opencensus_opencensus_api", + ":io_opencensus_opencensus_contrib_grpc_metrics", + ], + ) + + + native.java_library( + name = "ch_qos_logback_logback_classic", + visibility = ["//visibility:public"], + exports = ["@ch_qos_logback_logback_classic//jar"], + runtime_deps = [ + ":ch_qos_logback_logback_core", + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "joda_time_joda_time", + visibility = ["//visibility:public"], + exports = ["@joda_time_joda_time//jar"], + ) + + + native.java_library( + name = "net_bytebuddy_byte_buddy_agent", + visibility = ["//visibility:public"], + exports = ["@net_bytebuddy_byte_buddy_agent//jar"], + ) + + + native.java_library( + name = "io_grpc_grpc_protobuf_lite", + visibility = ["//visibility:public"], + exports = ["@io_grpc_grpc_protobuf_lite//jar"], + runtime_deps = [ + ":com_google_guava_guava", + ":io_grpc_grpc_core", + ], + ) + + + native.java_library( + name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_converter_classic", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_logback_converter_classic//jar"], + runtime_deps = [ + ":ch_qos_logback_logback_classic", + ":de_huxhorn_lilith_de_huxhorn_lilith_data_converter", + ":de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", + ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging", + ":de_huxhorn_lilith_de_huxhorn_lilith_logback_classic", + ":de_huxhorn_sulky_de_huxhorn_sulky_codec", + ":de_huxhorn_sulky_de_huxhorn_sulky_formatting", + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "io_netty_netty_transport", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_transport//jar"], + runtime_deps = [ + ":io_netty_netty_buffer", + ":io_netty_netty_common", + ":io_netty_netty_resolver", + ], + ) + + + native.java_library( + name = "com_carrotsearch_hppc", + visibility = ["//visibility:public"], + exports = ["@com_carrotsearch_hppc//jar"], + ) + + + native.java_library( + name = "org_mockito_mockito_core", + visibility = ["//visibility:public"], + exports = ["@org_mockito_mockito_core//jar"], + runtime_deps = [ + ":net_bytebuddy_byte_buddy", + ":net_bytebuddy_byte_buddy_agent", + ":org_objenesis_objenesis", + ], + ) + + + native.java_library( + name = "io_netty_netty_handler_proxy", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_handler_proxy//jar"], + runtime_deps = [ + ":io_netty_netty_codec", + ":io_netty_netty_codec_http", + ":io_netty_netty_codec_socks", + ":io_netty_netty_transport", + ], + ) + + + native.java_library( + name = "de_huxhorn_sulky_de_huxhorn_sulky_codec", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_sulky_de_huxhorn_sulky_codec//jar"], + runtime_deps = [ + ":de_huxhorn_sulky_de_huxhorn_sulky_io", + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource//jar"], + ) + + + native.java_library( + name = "com_google_flatbuffers_flatbuffers_java", + visibility = ["//visibility:public"], + exports = ["@com_google_flatbuffers_flatbuffers_java//jar"], + ) + + + native.java_library( + name = "com_google_protobuf_protobuf_java", + visibility = ["//visibility:public"], + exports = ["@com_google_protobuf_protobuf_java//jar"], + ) + + + native.java_library( + name = "com_h2database_h2", + visibility = ["//visibility:public"], + exports = ["@com_h2database_h2//jar"], + ) + + + native.java_library( + name = "org_slf4j_jul_to_slf4j", + visibility = ["//visibility:public"], + exports = ["@org_slf4j_jul_to_slf4j//jar"], + runtime_deps = [ + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "commons_codec_commons_codec", + visibility = ["//visibility:public"], + exports = ["@commons_codec_commons_codec//jar"], + ) + + + native.java_library( + name = "org_slf4j_log4j_over_slf4j", + visibility = ["//visibility:public"], + exports = ["@org_slf4j_log4j_over_slf4j//jar"], + runtime_deps = [ + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "commons_cli_commons_cli", + visibility = ["//visibility:public"], + exports = ["@commons_cli_commons_cli//jar"], + ) + + + native.java_library( + name = "io_netty_netty_tcnative_boringssl_static", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_tcnative_boringssl_static//jar"], + ) + + + native.java_library( + name = "io_grpc_grpc_stub", + visibility = ["//visibility:public"], + exports = ["@io_grpc_grpc_stub//jar"], + runtime_deps = [ + ":io_grpc_grpc_core", + ], + ) + + + native.java_library( + name = "io_grpc_grpc_protobuf", + visibility = ["//visibility:public"], + exports = ["@io_grpc_grpc_protobuf//jar"], + runtime_deps = [ + ":com_google_api_grpc_proto_google_common_protos", + ":com_google_guava_guava", + ":com_google_protobuf_protobuf_java", + ":io_grpc_grpc_core", + ":io_grpc_grpc_protobuf_lite", + ], + ) + + + native.java_library( + name = "io_netty_netty_codec_socks", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_codec_socks//jar"], + runtime_deps = [ + ":io_netty_netty_codec", + ], + ) + + + native.java_library( + name = "io_netty_netty_codec", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_codec//jar"], + runtime_deps = [ + ":io_netty_netty_buffer", + ":io_netty_netty_common", + ":io_netty_netty_resolver", + ":io_netty_netty_transport", + ], + ) + + + native.java_library( + name = "io_netty_netty_buffer", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_buffer//jar"], + runtime_deps = [ + ":io_netty_netty_common", + ], + ) + + + native.java_library( + name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_core", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_core//jar"], + runtime_deps = [ + ":ch_qos_logback_logback_core", + ":de_huxhorn_lilith_de_huxhorn_lilith_sender", + ":de_huxhorn_sulky_de_huxhorn_sulky_codec", + ":de_huxhorn_sulky_de_huxhorn_sulky_io", + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "io_grpc_grpc_netty", + visibility = ["//visibility:public"], + exports = ["@io_grpc_grpc_netty//jar"], + runtime_deps = [ + ":com_google_code_findbugs_jsr305", + ":com_google_code_gson_gson", + ":com_google_errorprone_error_prone_annotations", + ":com_google_guava_guava", + ":io_grpc_grpc_context", + ":io_grpc_grpc_core", + ":io_netty_netty_buffer", + ":io_netty_netty_codec", + ":io_netty_netty_codec_http", + ":io_netty_netty_codec_http2", + ":io_netty_netty_codec_socks", + ":io_netty_netty_common", + ":io_netty_netty_handler", + ":io_netty_netty_handler_proxy", + ":io_netty_netty_resolver", + ":io_netty_netty_transport", + ":io_opencensus_opencensus_api", + ":io_opencensus_opencensus_contrib_grpc_metrics", + ], + ) + + + native.java_library( + name = "ch_qos_logback_logback_core", + visibility = ["//visibility:public"], + exports = ["@ch_qos_logback_logback_core//jar"], + ) + + + native.java_library( + name = "net_bytebuddy_byte_buddy", + visibility = ["//visibility:public"], + exports = ["@net_bytebuddy_byte_buddy//jar"], + ) + + + native.java_library( + name = "io_opencensus_opencensus_api", + visibility = ["//visibility:public"], + exports = ["@io_opencensus_opencensus_api//jar"], + ) + + + native.java_library( + name = "de_huxhorn_lilith_de_huxhorn_lilith_sender", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_sender//jar"], + runtime_deps = [ + ":de_huxhorn_sulky_de_huxhorn_sulky_io", + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "com_google_errorprone_error_prone_annotations", + visibility = ["//visibility:public"], + exports = ["@com_google_errorprone_error_prone_annotations//jar"], + ) + + + native.java_library( + name = "io_netty_netty_resolver", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_resolver//jar"], + runtime_deps = [ + ":io_netty_netty_common", + ], + ) + + + native.java_library( + name = "io_opencensus_opencensus_contrib_grpc_metrics", + visibility = ["//visibility:public"], + exports = ["@io_opencensus_opencensus_contrib_grpc_metrics//jar"], + runtime_deps = [ + ":io_opencensus_opencensus_api", + ], + ) + + + native.java_library( + name = "org_javassist_javassist", + visibility = ["//visibility:public"], + exports = ["@org_javassist_javassist//jar"], + ) + + + native.java_library( + name = "org_objenesis_objenesis", + visibility = ["//visibility:public"], + exports = ["@org_objenesis_objenesis//jar"], + ) + + + native.java_library( + name = "io_netty_netty_common", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_common//jar"], + ) + + + native.java_library( + name = "junit_junit", + visibility = ["//visibility:public"], + exports = ["@junit_junit//jar"], + runtime_deps = [ + ":org_hamcrest_hamcrest_core", + ], + ) + + + native.java_library( + name = "de_huxhorn_lilith_de_huxhorn_lilith_data_logging_protobuf", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_data_logging_protobuf//jar"], + runtime_deps = [ + ":com_google_protobuf_protobuf_java", + ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging", + ":de_huxhorn_sulky_de_huxhorn_sulky_codec", + ], + ) + + + native.java_library( + name = "de_huxhorn_sulky_de_huxhorn_sulky_io", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_sulky_de_huxhorn_sulky_io//jar"], + ) + + + native.java_library( + name = "org_slf4j_slf4j_api", + visibility = ["//visibility:public"], + exports = ["@org_slf4j_slf4j_api//jar"], + ) + + + native.java_library( + name = "io_netty_netty_codec_http", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_codec_http//jar"], + runtime_deps = [ + ":io_netty_netty_buffer", + ":io_netty_netty_codec", + ":io_netty_netty_common", + ":io_netty_netty_resolver", + ":io_netty_netty_transport", + ], + ) + + + native.java_library( + name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_classic", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_classic//jar"], + runtime_deps = [ + ":ch_qos_logback_logback_classic", + ":ch_qos_logback_logback_core", + ":com_google_protobuf_protobuf_java", + ":de_huxhorn_lilith_de_huxhorn_lilith_data_converter", + ":de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", + ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging", + ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging_protobuf", + ":de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_core", + ":de_huxhorn_lilith_de_huxhorn_lilith_logback_classic", + ":de_huxhorn_lilith_de_huxhorn_lilith_logback_converter_classic", + ":de_huxhorn_lilith_de_huxhorn_lilith_sender", + ":de_huxhorn_sulky_de_huxhorn_sulky_codec", + ":de_huxhorn_sulky_de_huxhorn_sulky_formatting", + ":de_huxhorn_sulky_de_huxhorn_sulky_io", + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "org_hamcrest_hamcrest_core", + visibility = ["//visibility:public"], + exports = ["@org_hamcrest_hamcrest_core//jar"], + ) + + + native.java_library( + name = "io_netty_netty_handler", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_handler//jar"], + runtime_deps = [ + ":io_netty_netty_buffer", + ":io_netty_netty_codec", + ":io_netty_netty_transport", + ], + ) + + + native.java_library( + name = "org_jmockit_jmockit", + visibility = ["//visibility:public"], + exports = ["@org_jmockit_jmockit//jar"], + ) + + + native.java_library( + name = "de_huxhorn_sulky_de_huxhorn_sulky_formatting", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_sulky_de_huxhorn_sulky_formatting//jar"], + runtime_deps = [ + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "io_netty_netty_codec_http2", + visibility = ["//visibility:public"], + exports = ["@io_netty_netty_codec_http2//jar"], + runtime_deps = [ + ":io_netty_netty_buffer", + ":io_netty_netty_codec", + ":io_netty_netty_codec_http", + ":io_netty_netty_common", + ":io_netty_netty_handler", + ":io_netty_netty_resolver", + ":io_netty_netty_transport", + ], + ) + + + native.java_library( + name = "net_lingala_zip4j_zip4j", + visibility = ["//visibility:public"], + exports = ["@net_lingala_zip4j_zip4j//jar"], + ) + + + native.java_library( + name = "org_ini4j_ini4j", + visibility = ["//visibility:public"], + exports = ["@org_ini4j_ini4j//jar"], + ) + + + native.java_library( + name = "org_ow2_asm_asm", + visibility = ["//visibility:public"], + exports = ["@org_ow2_asm_asm//jar"], + ) + + + native.java_library( + name = "com_google_code_gson_gson", + visibility = ["//visibility:public"], + exports = ["@com_google_code_gson_gson//jar"], + ) + + + native.java_library( + name = "org_slf4j_jcl_over_slf4j", + visibility = ["//visibility:public"], + exports = ["@org_slf4j_jcl_over_slf4j//jar"], + runtime_deps = [ + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "com_fasterxml_jackson_core_jackson_annotations", + visibility = ["//visibility:public"], + exports = ["@com_fasterxml_jackson_core_jackson_annotations//jar"], + ) + + + native.java_library( + name = "de_huxhorn_lilith_de_huxhorn_lilith_data_logging", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_data_logging//jar"], + runtime_deps = [ + ":de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", + ":de_huxhorn_sulky_de_huxhorn_sulky_formatting", + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_classic", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_logback_classic//jar"], + runtime_deps = [ + ":de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", + ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging", + ":de_huxhorn_sulky_de_huxhorn_sulky_formatting", + ":org_slf4j_slf4j_api", + ], + ) + + + native.java_library( + name = "com_fasterxml_jackson_core_jackson_databind", + visibility = ["//visibility:public"], + exports = ["@com_fasterxml_jackson_core_jackson_databind//jar"], + runtime_deps = [ + ":com_fasterxml_jackson_core_jackson_annotations", + ":com_fasterxml_jackson_core_jackson_core", + ], + ) + + + native.java_library( + name = "com_fasterxml_jackson_dataformat_jackson_dataformat_yaml", + visibility = ["//visibility:public"], + exports = ["@com_fasterxml_jackson_dataformat_jackson_dataformat_yaml//jar"], + runtime_deps = [ + ":com_fasterxml_jackson_core_jackson_core", + ":org_yaml_snakeyaml", + ], + ) + + + native.java_library( + name = "io_grpc_grpc_context", + visibility = ["//visibility:public"], + exports = ["@io_grpc_grpc_context//jar"], + ) + + + native.java_library( + name = "com_google_api_grpc_proto_google_common_protos", + visibility = ["//visibility:public"], + exports = ["@com_google_api_grpc_proto_google_common_protos//jar"], + ) + + + native.java_library( + name = "com_github_davidmoten_flatbuffers_java", + visibility = ["//visibility:public"], + exports = ["@com_github_davidmoten_flatbuffers_java//jar"], + ) + + + native.java_library( + name = "redis_clients_jedis", + visibility = ["//visibility:public"], + exports = ["@redis_clients_jedis//jar"], + runtime_deps = [ + ":org_apache_commons_commons_pool2", + ], + ) + + + native.java_library( + name = "org_apache_commons_commons_pool2", + visibility = ["//visibility:public"], + exports = ["@org_apache_commons_commons_pool2//jar"], + ) + + + native.java_library( + name = "com_google_guava_guava", + visibility = ["//visibility:public"], + exports = ["@com_google_guava_guava//jar"], + ) + + + native.java_library( + name = "de_huxhorn_lilith_de_huxhorn_lilith_data_converter", + visibility = ["//visibility:public"], + exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_data_converter//jar"], + ) + + + native.java_library( + name = "de_ruedigermoeller_fst", + visibility = ["//visibility:public"], + exports = ["@de_ruedigermoeller_fst//jar"], + runtime_deps = [ + ":com_fasterxml_jackson_core_jackson_core", + ":org_javassist_javassist", + ":org_objenesis_objenesis", + ], + ) + + + native.java_library( + name = "com_fasterxml_jackson_core_jackson_core", + visibility = ["//visibility:public"], + exports = ["@com_fasterxml_jackson_core_jackson_core//jar"], + ) + + + native.java_library( + name = "org_yaml_snakeyaml", + visibility = ["//visibility:public"], + exports = ["@org_yaml_snakeyaml//jar"], + ) + + + native.java_library( + name = "net_java_dev_jna_jna", + visibility = ["//visibility:public"], + exports = ["@net_java_dev_jna_jna//jar"], + ) + + + native.java_library( + name = "org_mockito_mockito_all", + visibility = ["//visibility:public"], + exports = ["@org_mockito_mockito_all//jar"], + ) + + + native.java_library( + name = "com_beust_jcommander", + visibility = ["//visibility:public"], + exports = ["@com_beust_jcommander//jar"], + ) + + native.java_library( + name = "org_testng_testng", + visibility = ["//visibility:public"], + exports = ["@org_testng_testng//jar"], + ) diff --git a/build.sh b/build.sh index 283f16757e49..1b143b6da838 100755 --- a/build.sh +++ b/build.sh @@ -121,14 +121,21 @@ else $PYTHON_EXECUTABLE -m pip install \ --target=$ROOT_DIR/python/ray/pyarrow_files pyarrow==0.12.0.RAY \ --find-links https://s3-us-west-2.amazonaws.com/arrow-wheels/9357dc130789ee42f8181d8724bee1d5d1509060/index.html - bazel build //:ray_pkg -c opt --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHON_EXECUTABLE - # Copy files and keep them writeable. This is a workaround, as Bazel - # marks all generated files non-writeable. If we would just copy them - # over without adding write permission, the copy would fail the next time. - # TODO(pcm): It would be great to have a solution here that does not - # require us to copy the files. - find $ROOT_DIR/bazel-genfiles/ray_pkg/ -exec chmod +w {} \; - cp -r $ROOT_DIR/bazel-genfiles/ray_pkg/ray $ROOT_DIR/python || true + if [ "$RAY_BUILD_PYTHON" == "YES" ]; then + bazel build //:ray_pkg -c opt + # Copy files and keep them writeable. This is a workaround, as Bazel + # marks all generated files non-writeable. If we would just copy them + # over without adding write permission, the copy would fail the next time. + # TODO(pcm): It would be great to have a solution here that does not + # require us to copy the files. + find $ROOT_DIR/bazel-genfiles/ray_pkg/ -exec chmod +w {} \; + cp -r $ROOT_DIR/bazel-genfiles/ray_pkg/ray $ROOT_DIR/python || true + fi + if [ "$RAY_BUILD_JAVA" == "YES" ]; then + bazel build //:ray_java_pkg -c opt + find $ROOT_DIR/bazel-genfiles/ray_java_pkg/ -exec chmod +w {} \; + cp -r $ROOT_DIR/bazel-genfiles/ray_java_pkg/ray/* $ROOT_DIR/build || true + fi fi popd diff --git a/java/BUILD.bazel b/java/BUILD.bazel new file mode 100644 index 000000000000..adb4bf3cff20 --- /dev/null +++ b/java/BUILD.bazel @@ -0,0 +1,211 @@ + +load("//bazel:ray.bzl", "flatbuffer_java_library") +load("//bazel:ray_java.bzl", "generated_java_libraries") + +generated_java_libraries() + +java_import( + name = "org_ray_ray_java_root", + jars = [ + "liborg_ray_ray_api.jar", + "liborg_ray_ray_api-src.jar", + "liborg_ray_ray_runtime.jar", + "liborg_ray_ray_runtime-src.jar", + "liborg_ray_ray_cli.jar", + "liborg_ray_ray_cli-src.jar", + "liborg_ray_ray_runtime_test.jar", + "liborg_ray_ray_runtime_test-src.jar", + "liborg_ray_ray_test.jar", + "liborg_ray_ray_test-src.jar", + "org_ray_ray_tutorial_deploy.jar", + "org_ray_ray_tutorial_deploy-src.jar", + "AllTests_deploy.jar", + "AllTests_deploy-src.jar", + ], + deps = [ + ":org_ray_ray_api", + ":org_ray_ray_cli", + ":org_ray_ray_runtime", + ":org_ray_ray_tutorial", + ":org_ray_ray_test", + ":org_ray_ray_runtime_test", + ], +) + +java_library( + name = "org_ray_ray_api", + srcs = glob(["api/src/main/java/**/*.java"]), + deps = [ + "@org_slf4j_slf4j_log4j12//jar", + "@log4j_log4j//jar", + ], +) + +java_library( + name = "org_ray_ray_cli", + srcs = glob(["cli/src/main/java/**/*.java"]), + visibility = ["//visibility:public"], + deps = [ + "@com_beust_jcommander//jar", + "@com_github_davidmoten_flatbuffers_java//jar", + "@de_ruedigermoeller_fst//jar", + ":org_ray_ray_api", + ":org_ray_ray_runtime", + "@redis_clients_jedis//jar", + "@org_slf4j_slf4j_api//jar", + ], +) + +java_library( + name = "org_ray_ray_runtime", + srcs = glob(["runtime/src/main/java/**/*.java"]), + resources = glob(["runtime/src/main/resources/**"]), + visibility = ["//visibility:public"], + data = ["//:raylet_library_java", ":copy_java_gcs_fbs"], + deps = [ + ":org_ray_ray_api", + "@com_typesafe_config//jar", + "@org_apache_commons_commons_lang3//jar", + "@de_ruedigermoeller_fst//jar", + "@com_github_davidmoten_flatbuffers_java//jar", + "@redis_clients_jedis//jar", + "@plasma//:org_apache_arrow_arrow_plasma", + "@commons_io_commons_io//jar", + "@com_google_guava_guava//jar", + "@net_lingala_zip4j_zip4j//jar", + "@org_slf4j_slf4j_log4j12//jar", + "@org_slf4j_slf4j_api//jar", + "@org_ini4j_ini4j//jar", + "@org_ow2_asm_asm//jar", + "@com_google_code_gson_gson//jar", + "@com_fasterxml_jackson_core_jackson_core//jar", + "@org_apache_commons_commons_pool2//jar", + "@org_javassist_javassist//jar", + "@org_objenesis_objenesis//jar", + ], +) + +java_binary( + name = "org_ray_ray_tutorial", + srcs = glob(["tutorial/src/main/java/**/*.java"]), + resources = glob(["tutorial/src/main/resources/**"]), + visibility = ["//visibility:public"], + deps = [ + ":org_ray_ray_api", + ":org_ray_ray_runtime", + "@com_google_guava_guava//jar", + ], +) + +java_library( + name = "org_ray_ray_test", + srcs = glob(["test/src/main/java/**/*.java"]), + deps = [ + "@com_google_guava_guava//jar", + "@commons_collections_commons_collections//jar", + ":org_ray_ray_api", + ":org_ray_ray_runtime", + "@org_apache_commons_commons_lang3//jar", + "@org_slf4j_slf4j_api//jar", + "@plasma//:org_apache_arrow_arrow_plasma", + "@org_testng_testng//jar", + ], +) + +java_library( + name = "org_ray_ray_runtime_test", + srcs = glob(["runtime/src/test/java/**/*.java"]), + deps = [ + ":org_ray_ray_api", + ":org_ray_ray_runtime", + "@org_testng_testng//jar", + "@org_apache_commons_commons_lang3//jar", + "@com_beust_jcommander//jar", + ], +) + +java_test( + name = "AllTests", + size = "small", + main_class = "org.testng.TestNG", + data = ["testng.xml"], + runtime_deps = [ + ":org_ray_ray_test", + ":org_ray_ray_runtime_test", + ], +) + +#use .py file to generate *.java, but be telled can't to find the generated files +#genrule( +# name = "generate_java_gcs_fbs", +# outs = ["generate_java_gcs_fbs.out"], +# cmd = """ +# python3 $$(pwd)/java/modify_generated_java_flatbuffers_files.py $$(pwd) +# echo $$(pwd) > $@ +# """, +# local = 1, +#) + +flatbuffer_java_library( + name = "java_gcs_fbs", + srcs = ["//:gcs_fbs_file"], + outs = [ + "ActorCheckpointData.java", + "ActorCheckpointIdData.java", + "ActorState.java", + "ActorTableData.java", + "Arg.java", + "ClassTableData.java", + "ClientTableData.java", + "ClientType.java", + "ConfigTableData.java", + "CustomSerializerData.java", + "DriverTableData.java", + "ErrorTableData.java", + "ErrorType.java", + "FunctionTableData.java", + "GcsTableEntry.java", + "GcsTableNotificationMode.java", + "HeartbeatBatchTableData.java", + "HeartbeatTableData.java", + "Language.java", + "ObjectTableData.java", + "PipeClientData.java", + "ProfileEvent.java", + "ProfileTableData.java", + "RayResource.java", + "ResourcePair.java", + "SchedulingState.java", + "TablePrefix.java", + "TablePubsub.java", + "TaskInfo.java", + "TaskLeaseData.java", + "TaskReconstructionData.java", + "TaskTableData.java", + "TaskTableTestAndUpdate.java", + ], + out_prefix = "java/runtime/src/main/java/org/ray/runtime/generated", +) + +genrule( + name = "copy_java_gcs_fbs", + srcs = [ + "//java:java_gcs_fbs", + ], + outs = [ + "java_gcs_fbs.out", + ], + cmd = """ + WORK_DIR=$$(pwd) + GENERATED_DIR=$$WORK_DIR/java/runtime/src/main/java/org/ray/runtime/generated + mkdir -p $$GENERATED_DIR + for f in $(locations //java:java_gcs_fbs); do + cp $$f $$GENERATED_DIR/; + file="$$GENERATED_DIR/$${f#*java/}"; + chmod +w $$file; + sed -i "2i\package org.ray.runtime.generated;" $$file; + done + echo $$(pwd) >> $@ + """, + local = 1, +) diff --git a/java/build.sh b/java/build.sh index 60b361ae8eb9..b9a25686e5e4 100755 --- a/java/build.sh +++ b/java/build.sh @@ -1,3 +1,7 @@ #!/usr/bin/env bash -mvn clean install -Dmaven.test.skip +if [ ! -z "$RAY_USE_CMAKE" ] ; then + mvn clean install -Dmaven.test.skip +else + bazel build //java:org_ray_ray_java_root +fi diff --git a/java/test.sh b/java/test.sh index b3e889371bcd..040662f1cae9 100755 --- a/java/test.sh +++ b/java/test.sh @@ -7,20 +7,38 @@ set -x ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -pushd $ROOT_DIR/../java -echo "Compiling Java code." -mvn clean install -Dmaven.test.skip - -echo "Checking code format." -mvn checkstyle:check - -echo "Running tests under cluster mode." -ENABLE_MULTI_LANGUAGE_TESTS=1 mvn test - -echo "Running tests under single-process mode." -mvn test -Dray.run-mode=SINGLE_PROCESS - -set +x -set +e +if [ ! -z "$RAY_USE_CMAKE" ] ; then + pushd $ROOT_DIR/../java + mvn clean install -Dmaven.test.skip + check_style=$(mvn checkstyle:check) + echo "${check_style}" + [[ ${check_style} =~ "BUILD FAILURE" ]] && exit 1 + + # test raylet + mvn test | tee mvn_test + if [ `grep -c "BUILD FAILURE" mvn_test` -eq '0' ]; then + rm mvn_test + echo "Tests passed under CLUSTER mode!" + else + rm mvn_test + exit 1 + fi + # test raylet under SINGLE_PROCESS mode + mvn test -Dray.run-mode=SINGLE_PROCESS | tee dev_mvn_test + if [ `grep -c "BUILD FAILURE" dev_mvn_test` -eq '0' ]; then + rm dev_mvn_test + echo "Tests passed under SINGLE_PROCESS mode!" + else + rm dev_mvn_test + exit 1 + fi +else + pushd $ROOT_DIR/.. + bazel build -c opt //java:all + pushd $ROOT_DIR/../java/test + java -jar -Dray.home=$ROOT_DIR/../ -Djava.library.path=$ROOT_DIR/../bazel-bin:$ROOT_DIR/../bazel-bin/externl/plasma:$ROOT_DIR/../bazel-genfiles $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml + + java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS -Djava.library.path=$ROOT_DIR/../bazel-bin:$ROOT_DIR/../bazel-bin/externl/plasma:$ROOT_DIR/../bazel-genfiles $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml +fi popd diff --git a/java/testng.xml b/java/testng.xml new file mode 100644 index 000000000000..4dc31f8f99d2 --- /dev/null +++ b/java/testng.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + From 91666b71ae745fcc6a7b045b821e1e31abbf5196 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Wed, 6 Mar 2019 03:17:33 +0800 Subject: [PATCH 038/115] java bazel config --- java/BUILD.bazel | 21 ------------------- .../org/ray/api/test/ClientExceptionTest.java | 4 ++-- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/java/BUILD.bazel b/java/BUILD.bazel index adb4bf3cff20..d0394f9aa24a 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -11,8 +11,6 @@ java_import( "liborg_ray_ray_api-src.jar", "liborg_ray_ray_runtime.jar", "liborg_ray_ray_runtime-src.jar", - "liborg_ray_ray_cli.jar", - "liborg_ray_ray_cli-src.jar", "liborg_ray_ray_runtime_test.jar", "liborg_ray_ray_runtime_test-src.jar", "liborg_ray_ray_test.jar", @@ -24,7 +22,6 @@ java_import( ], deps = [ ":org_ray_ray_api", - ":org_ray_ray_cli", ":org_ray_ray_runtime", ":org_ray_ray_tutorial", ":org_ray_ray_test", @@ -41,21 +38,6 @@ java_library( ], ) -java_library( - name = "org_ray_ray_cli", - srcs = glob(["cli/src/main/java/**/*.java"]), - visibility = ["//visibility:public"], - deps = [ - "@com_beust_jcommander//jar", - "@com_github_davidmoten_flatbuffers_java//jar", - "@de_ruedigermoeller_fst//jar", - ":org_ray_ray_api", - ":org_ray_ray_runtime", - "@redis_clients_jedis//jar", - "@org_slf4j_slf4j_api//jar", - ], -) - java_library( name = "org_ray_ray_runtime", srcs = glob(["runtime/src/main/java/**/*.java"]), @@ -157,7 +139,6 @@ flatbuffer_java_library( "Arg.java", "ClassTableData.java", "ClientTableData.java", - "ClientType.java", "ConfigTableData.java", "CustomSerializerData.java", "DriverTableData.java", @@ -165,12 +146,10 @@ flatbuffer_java_library( "ErrorType.java", "FunctionTableData.java", "GcsTableEntry.java", - "GcsTableNotificationMode.java", "HeartbeatBatchTableData.java", "HeartbeatTableData.java", "Language.java", "ObjectTableData.java", - "PipeClientData.java", "ProfileEvent.java", "ProfileTableData.java", "RayResource.java", diff --git a/java/test/src/main/java/org/ray/api/test/ClientExceptionTest.java b/java/test/src/main/java/org/ray/api/test/ClientExceptionTest.java index e9f53dddd794..b588822712c5 100644 --- a/java/test/src/main/java/org/ray/api/test/ClientExceptionTest.java +++ b/java/test/src/main/java/org/ray/api/test/ClientExceptionTest.java @@ -37,12 +37,12 @@ public void testWaitAndCrash() { Ray.wait(ImmutableList.of(notExisting), 1, 2000); Assert.fail("Should not reach here"); } catch (RayException e) { - LOGGER.debug(String.format("Expected runtime exception: {}", e)); + LOGGER.debug("Expected runtime exception: {}", e); } try { thread.join(); } catch (Exception e) { - LOGGER.error(String.format("Excpetion caught: {}", e)); + LOGGER.error("Excpetion caught: {}", e); } } } From f9d3cc404c62a9ecb2e0d8ab8da0ada2469bbfbf Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Wed, 6 Mar 2019 15:27:11 +0800 Subject: [PATCH 039/115] java bazel config --- BUILD.bazel | 15 ++------------- build.sh | 21 ++++++++++++++++----- java/BUILD.bazel | 24 +++++++++++------------- 3 files changed, 29 insertions(+), 31 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index f06e13147d83..30e3945afe00 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -595,19 +595,8 @@ genrule( "@plasma//:plasma_store_server", "@plasma//:plasma_client_java", ], - outs = ["ray_java_pkg"], + outs = ["ray_java_pkg.out"], cmd = """ - set -x && - mkdir -p java/ray/src/plasma/ && - cp $(location @plasma//:plasma_client_java) java/ray/src/plasma/ && - cp $(location @plasma//:plasma_store_server) java/ray/src/plasma/ && - mkdir -p java/ray/src/ray/raylet/ && - cp $(location //:raylet) java/ray/src/ray/raylet/ && - cp $(location //:raylet_library_java) java/ray/src/ray/raylet/ && - mkdir -p java/ray/src/ray/gcs/redis_module/ && - cp $(location //:ray_redis_module.so) java/ray/src/ray/gcs/redis_module/libray_redis_module.so && - mkdir -p java/ray/src/ray/thirdparty/redis/src/ && - cp $(location //:redis-server) java/ray/src/ray/thirdparty/redis/src/ && - mv java $(location ray_java_pkg) + echo $$(pwd) > $@ """, ) diff --git a/build.sh b/build.sh index 1b143b6da838..590c0da1713f 100755 --- a/build.sh +++ b/build.sh @@ -121,6 +121,22 @@ else $PYTHON_EXECUTABLE -m pip install \ --target=$ROOT_DIR/python/ray/pyarrow_files pyarrow==0.12.0.RAY \ --find-links https://s3-us-west-2.amazonaws.com/arrow-wheels/9357dc130789ee42f8181d8724bee1d5d1509060/index.html + +if [ "$RAY_BUILD_JAVA" == "YES" ]; then + bazel build //:ray_java_pkg -c opt + #soft link + mkdir -p $ROOT_DIR/build/src/ray/raylet/ + mkdir -p $ROOT_DIR/build/src/ray/gcs/redis_module/ + mkdir -p $ROOT_DIR/build/src/ray/thirdparty/redis/src/ + mkdir -p $ROOT_DIR/build/src/plasma/ + ln -sf $ROOT_DIR/bazel-bin/* $ROOT_DIR/build/src/ray/raylet/ + ln -sf $ROOT_DIR/bazel-bin/external/plasma/* $ROOT_DIR/build/src/plasma/ + ln -sf $ROOT_DIR/bazel-genfiles/redis-server $ROOT_DIR/build/src/ray/thirdparty/redis/src/redis-server + ln -sf $ROOT_DIR/bazel-bin/ray_redis_module.so $ROOT_DIR/build/src/ray/gcs/redis_module/libray_redis_module.so + ln -sf $ROOT_DIR/bazel-bin/external/plasma/plasma_store_server $ROOT_DIR/build/src/plasma/plasma_store_server + ln -sf $ROOT_DIR/bazel-bin/raylet $ROOT_DIR/build/src/ray/raylet/raylet + fi + if [ "$RAY_BUILD_PYTHON" == "YES" ]; then bazel build //:ray_pkg -c opt # Copy files and keep them writeable. This is a workaround, as Bazel @@ -131,11 +147,6 @@ else find $ROOT_DIR/bazel-genfiles/ray_pkg/ -exec chmod +w {} \; cp -r $ROOT_DIR/bazel-genfiles/ray_pkg/ray $ROOT_DIR/python || true fi - if [ "$RAY_BUILD_JAVA" == "YES" ]; then - bazel build //:ray_java_pkg -c opt - find $ROOT_DIR/bazel-genfiles/ray_java_pkg/ -exec chmod +w {} \; - cp -r $ROOT_DIR/bazel-genfiles/ray_java_pkg/ray/* $ROOT_DIR/build || true - fi fi popd diff --git a/java/BUILD.bazel b/java/BUILD.bazel index d0394f9aa24a..72ee53778440 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -43,7 +43,7 @@ java_library( srcs = glob(["runtime/src/main/java/**/*.java"]), resources = glob(["runtime/src/main/resources/**"]), visibility = ["//visibility:public"], - data = ["//:raylet_library_java", ":copy_java_gcs_fbs"], + data = ["//:raylet_library_java", ":generate_java_gcs_fbs"], deps = [ ":org_ray_ray_api", "@com_typesafe_config//jar", @@ -118,15 +118,16 @@ java_test( ) #use .py file to generate *.java, but be telled can't to find the generated files -#genrule( -# name = "generate_java_gcs_fbs", -# outs = ["generate_java_gcs_fbs.out"], -# cmd = """ -# python3 $$(pwd)/java/modify_generated_java_flatbuffers_files.py $$(pwd) -# echo $$(pwd) > $@ -# """, -# local = 1, -#) +genrule( + name = "generate_java_gcs_fbs", + srcs = [":copy_java_gcs_fbs"], + outs = ["generate_java_gcs_fbs.out"], + cmd = """ + python3 $$(pwd)/java/modify_generated_java_flatbuffers_files.py $$(pwd) + echo $$(pwd) > $@ + """, + local = 1, +) flatbuffer_java_library( name = "java_gcs_fbs", @@ -180,9 +181,6 @@ genrule( mkdir -p $$GENERATED_DIR for f in $(locations //java:java_gcs_fbs); do cp $$f $$GENERATED_DIR/; - file="$$GENERATED_DIR/$${f#*java/}"; - chmod +w $$file; - sed -i "2i\package org.ray.runtime.generated;" $$file; done echo $$(pwd) >> $@ """, From 0a0c52513520afc83af7c51ee84050a21c1c33d8 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Wed, 6 Mar 2019 15:33:07 +0800 Subject: [PATCH 040/115] java bazel config --- java/BUILD.bazel | 1 - 1 file changed, 1 deletion(-) diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 72ee53778440..170b04b04cbf 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -117,7 +117,6 @@ java_test( ], ) -#use .py file to generate *.java, but be telled can't to find the generated files genrule( name = "generate_java_gcs_fbs", srcs = [":copy_java_gcs_fbs"], From 9006b29263535b11be283b1b9f6e00646465edfb Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 19:12:54 +0800 Subject: [PATCH 041/115] Update test.sh delete the setting of -Djava.library.path --- java/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/test.sh b/java/test.sh index 040662f1cae9..adcd6f422d83 100755 --- a/java/test.sh +++ b/java/test.sh @@ -36,9 +36,9 @@ else pushd $ROOT_DIR/.. bazel build -c opt //java:all pushd $ROOT_DIR/../java/test - java -jar -Dray.home=$ROOT_DIR/../ -Djava.library.path=$ROOT_DIR/../bazel-bin:$ROOT_DIR/../bazel-bin/externl/plasma:$ROOT_DIR/../bazel-genfiles $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml + java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml - java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS -Djava.library.path=$ROOT_DIR/../bazel-bin:$ROOT_DIR/../bazel-bin/externl/plasma:$ROOT_DIR/../bazel-genfiles $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml + java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml fi popd From 2247b0b0863e9e36f6f17f827fb1ae7860793ebd Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Wed, 6 Mar 2019 19:29:19 +0800 Subject: [PATCH 042/115] Update java/testng.xml Modified according to the recommendations of raulchen Co-Authored-By: alegithub111 --- java/testng.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/testng.xml b/java/testng.xml index 4dc31f8f99d2..5ccbb15a19b8 100644 --- a/java/testng.xml +++ b/java/testng.xml @@ -5,7 +5,7 @@ - + From 29f45c817f479fa35ebd519a656b5c43c9f5cd5d Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 19:46:38 +0800 Subject: [PATCH 043/115] Update build.sh Modified according to the recommendations of raulchen --- java/build.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/java/build.sh b/java/build.sh index b9a25686e5e4..06a99d4f63de 100755 --- a/java/build.sh +++ b/java/build.sh @@ -1,7 +1,3 @@ #!/usr/bin/env bash -if [ ! -z "$RAY_USE_CMAKE" ] ; then - mvn clean install -Dmaven.test.skip -else - bazel build //java:org_ray_ray_java_root -fi +bazel build //java:all From a064c0a71cfb75dd4c5e3f489b8ccb1ea68dadc3 Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 19:48:18 +0800 Subject: [PATCH 044/115] Update test.sh Modified according to the recommendations of raulchen --- java/test.sh | 39 +++++++-------------------------------- 1 file changed, 7 insertions(+), 32 deletions(-) diff --git a/java/test.sh b/java/test.sh index adcd6f422d83..7c1f8461f70a 100755 --- a/java/test.sh +++ b/java/test.sh @@ -7,38 +7,13 @@ set -x ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -if [ ! -z "$RAY_USE_CMAKE" ] ; then - pushd $ROOT_DIR/../java - mvn clean install -Dmaven.test.skip - check_style=$(mvn checkstyle:check) - echo "${check_style}" - [[ ${check_style} =~ "BUILD FAILURE" ]] && exit 1 - - # test raylet - mvn test | tee mvn_test - if [ `grep -c "BUILD FAILURE" mvn_test` -eq '0' ]; then - rm mvn_test - echo "Tests passed under CLUSTER mode!" - else - rm mvn_test - exit 1 - fi - # test raylet under SINGLE_PROCESS mode - mvn test -Dray.run-mode=SINGLE_PROCESS | tee dev_mvn_test - if [ `grep -c "BUILD FAILURE" dev_mvn_test` -eq '0' ]; then - rm dev_mvn_test - echo "Tests passed under SINGLE_PROCESS mode!" - else - rm dev_mvn_test - exit 1 - fi -else - pushd $ROOT_DIR/.. - bazel build -c opt //java:all - pushd $ROOT_DIR/../java/test - java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml +pushd $ROOT_DIR/.. +bazel build -c opt //java:all +pushd $ROOT_DIR/../java/test +# test raylet +java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml - java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml -fi +# test raylet under SINGLE_PROCESS mode +java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml popd From 70f823863f0b60aeadd0fdcff5fc4d5c402e4e97 Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 19:49:02 +0800 Subject: [PATCH 045/115] Update build.sh --- java/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/build.sh b/java/build.sh index 06a99d4f63de..9fc0aa7ab742 100755 --- a/java/build.sh +++ b/java/build.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -bazel build //java:all +bazel build -c opt //java:all From f432bb38adf716adcc11f62ba54a69e9aebe3779 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Wed, 6 Mar 2019 19:54:13 +0800 Subject: [PATCH 046/115] delete the variable RAY_USR_CMAKE --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 62fb4e43d397..3a0267f6aaf0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,6 @@ matrix: env: - JDK='Oracle JDK 8' - PYTHON=3.5 PYTHONWARNINGS=ignore - - RAY_USE_CMAKE=1 - RAY_INSTALL_JAVA=1 install: - ./ci/travis/install-dependencies.sh @@ -114,7 +113,7 @@ matrix: # Build MacOS wheels. - os: osx osx_image: xcode7 - env: MAC_WHEELS=1 PYTHONWARNINGS=ignore RAY_USE_CMAKE=1 + env: MAC_WHEELS=1 PYTHONWARNINGS=ignore install: - ./ci/travis/install-dependencies.sh # This command should be kept in sync with ray/python/README-building-wheels.md. @@ -129,7 +128,6 @@ matrix: - PYTHON=3.5 - RAY_USE_NEW_GCS=on - PYTHONWARNINGS=ignore - - RAY_USE_CMAKE=1 install: From 35a0be05e02ddc2b0d16fb3a8b9d3554e2788b7e Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 19:57:45 +0800 Subject: [PATCH 047/115] Update test.sh add the mvn ckeckstyle --- java/test.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/java/test.sh b/java/test.sh index 7c1f8461f70a..0c173232e95a 100755 --- a/java/test.sh +++ b/java/test.sh @@ -7,6 +7,12 @@ set -x ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +pushd $ROOT_DIR/../java +mvn clean install -Dmaven.test.skip +check_style=$(mvn checkstyle:check) +echo "${check_style}" +[[ ${check_style} =~ "BUILD FAILURE" ]] && exit 1 + pushd $ROOT_DIR/.. bazel build -c opt //java:all pushd $ROOT_DIR/../java/test From 168a430861f6d0c0d055cd92782edc1ced597d41 Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 20:12:06 +0800 Subject: [PATCH 048/115] Update .travis.yml Revert accidentally deleted code --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3a0267f6aaf0..7ca4741189bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -102,7 +102,7 @@ matrix: # Build Linux wheels. - os: linux dist: trusty - env: LINUX_WHEELS=1 PYTHONWARNINGS=ignore + env: LINUX_WHEELS=1 PYTHONWARNINGS=ignore RAY_USE_CMAKE=1 install: - ./ci/suppress_output ./ci/travis/install-dependencies.sh # This command should be kept in sync with ray/python/README-building-wheels.md. @@ -113,7 +113,7 @@ matrix: # Build MacOS wheels. - os: osx osx_image: xcode7 - env: MAC_WHEELS=1 PYTHONWARNINGS=ignore + env: MAC_WHEELS=1 PYTHONWARNINGS=ignore RAY_USE_CMAKE=1 install: - ./ci/travis/install-dependencies.sh # This command should be kept in sync with ray/python/README-building-wheels.md. @@ -128,7 +128,8 @@ matrix: - PYTHON=3.5 - RAY_USE_NEW_GCS=on - PYTHONWARNINGS=ignore - + - RAY_USE_CMAKE=1 + install: - ./ci/suppress_output ./ci/travis/install-bazel.sh From 85641aee5e83c478e9643946834c7f6625ce26c1 Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 20:12:37 +0800 Subject: [PATCH 049/115] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7ca4741189bc..e426882eb2fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -129,7 +129,7 @@ matrix: - RAY_USE_NEW_GCS=on - PYTHONWARNINGS=ignore - RAY_USE_CMAKE=1 - + install: - ./ci/suppress_output ./ci/travis/install-bazel.sh From 12abf44f4d49d12ef9427f97e2f3c72774f56a5f Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 20:16:53 +0800 Subject: [PATCH 050/115] Update testng.xml Delete useless comments --- java/testng.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/java/testng.xml b/java/testng.xml index 5ccbb15a19b8..81503edf4b85 100644 --- a/java/testng.xml +++ b/java/testng.xml @@ -1,7 +1,6 @@ - From 9f352898b1b50a02830c3b80cca2119a9d8bf6a9 Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 22:31:50 +0800 Subject: [PATCH 051/115] Update BUILD.bazel fix the error with target copy_java_gcs_fbs --- java/BUILD.bazel | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 170b04b04cbf..90ab4f8a53f6 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -122,7 +122,7 @@ genrule( srcs = [":copy_java_gcs_fbs"], outs = ["generate_java_gcs_fbs.out"], cmd = """ - python3 $$(pwd)/java/modify_generated_java_flatbuffers_files.py $$(pwd) + python $$(pwd)/java/modify_generated_java_flatbuffers_files.py $$(pwd) echo $$(pwd) > $@ """, local = 1, @@ -179,7 +179,8 @@ genrule( GENERATED_DIR=$$WORK_DIR/java/runtime/src/main/java/org/ray/runtime/generated mkdir -p $$GENERATED_DIR for f in $(locations //java:java_gcs_fbs); do - cp $$f $$GENERATED_DIR/; + chmod +w $$f + cp -f $$f $$GENERATED_DIR/ done echo $$(pwd) >> $@ """, From bfe211d0043c0da4772c77141e7c3e5a32ed1a80 Mon Sep 17 00:00:00 2001 From: alegithub111 Date: Wed, 6 Mar 2019 22:40:38 +0800 Subject: [PATCH 052/115] Update ray_java.bzl add a TODO note --- bazel/ray_java.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/bazel/ray_java.bzl b/bazel/ray_java.bzl index c66feda3f227..aefbc1df7466 100644 --- a/bazel/ray_java.bzl +++ b/bazel/ray_java.bzl @@ -1,6 +1,7 @@ # The following dependencies were calculated from: # # generate_workspace --maven_project=/home/admin/ruifang.crf/code/X/arrow/java --repositories=http://mvn.dev.alipay.net:8080/artifactory/repo +# TODO: now this file is not automatically generated def generated_maven_jars(): From ee495bb6fe6e9a37f7345f66ac7b5a1d0dd1a03c Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Thu, 7 Mar 2019 11:34:13 +0800 Subject: [PATCH 053/115] modify the java/test.sh --- java/test.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/java/test.sh b/java/test.sh index 0c173232e95a..4bf037aa0666 100755 --- a/java/test.sh +++ b/java/test.sh @@ -8,18 +8,19 @@ set -x ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) pushd $ROOT_DIR/../java -mvn clean install -Dmaven.test.skip -check_style=$(mvn checkstyle:check) -echo "${check_style}" -[[ ${check_style} =~ "BUILD FAILURE" ]] && exit 1 +echo "Checking code format." +mvn checkstyle:check +popd pushd $ROOT_DIR/.. bazel build -c opt //java:all +popd + pushd $ROOT_DIR/../java/test -# test raylet -java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml +echo "Running tests under cluster mode." +ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml -# test raylet under SINGLE_PROCESS mode +echo "Running tests under single-process mode." java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml popd From 43dfbcf918809a743f7de982e18dfd727a96f9be Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Thu, 7 Mar 2019 11:58:01 +0800 Subject: [PATCH 054/115] delete the 'test bazel build' and add install-bazel.sh to installation steps --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e426882eb2fe..595a1fe6f64d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,7 @@ matrix: - PYTHON=3.5 PYTHONWARNINGS=ignore - RAY_INSTALL_JAVA=1 install: + - ./ci/travis/install-bazel.sh - ./ci/travis/install-dependencies.sh - export PATH="$HOME/miniconda/bin:$PATH" - ./ci/travis/install-ray.sh From 86b7de02a9dbd438c2a27c28f425f79940e1e863 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Mon, 11 Mar 2019 19:23:19 +0800 Subject: [PATCH 055/115] Add java's checkstyle with bazel and modify with some other suggestions --- BUILD.bazel | 2 +- WORKSPACE | 7 +- bazel/BUILD.plasma | 11 +- bazel/ray_java.bzl | 1578 ----------------- java/BUILD.bazel | 103 +- java/build.sh | 3 - java/dependencies.yaml | 116 ++ java/generate_deps.sh | 11 + java/test.sh | 10 +- java/third_party/BUILD | 0 java/third_party/workspace.bzl | 73 + java/tools/checkstyle/BUILD | 7 + java/tools/checkstyle/README.md | 70 + .../checkstyle/check-for-missing-targets.sh | 19 + .../checkstyle}/checkstyle-suppressions.xml | 0 java/tools/checkstyle/checkstyle.bzl | 127 ++ java/tools/checkstyle/checkstyle.sh | 17 + java/{ => tools/checkstyle}/checkstyle.xml | 0 java/tools/checkstyle/license-header.txt | 15 + java/tools/checkstyle/repo.bzl | 21 + 20 files changed, 563 insertions(+), 1627 deletions(-) delete mode 100644 bazel/ray_java.bzl delete mode 100755 java/build.sh create mode 100644 java/dependencies.yaml create mode 100644 java/generate_deps.sh create mode 100644 java/third_party/BUILD create mode 100644 java/third_party/workspace.bzl create mode 100644 java/tools/checkstyle/BUILD create mode 100644 java/tools/checkstyle/README.md create mode 100755 java/tools/checkstyle/check-for-missing-targets.sh rename java/{ => tools/checkstyle}/checkstyle-suppressions.xml (100%) create mode 100644 java/tools/checkstyle/checkstyle.bzl create mode 100755 java/tools/checkstyle/checkstyle.sh rename java/{ => tools/checkstyle}/checkstyle.xml (100%) create mode 100644 java/tools/checkstyle/license-header.txt create mode 100644 java/tools/checkstyle/repo.bzl diff --git a/BUILD.bazel b/BUILD.bazel index 30e3945afe00..2fcc49f32141 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -396,8 +396,8 @@ cc_binary( linkshared = 1, linkstatic = 0, deps = [ + "//:raylet_lib", "@plasma//:plasma_client", - "//:raylet_lib" ], ) diff --git a/WORKSPACE b/WORKSPACE index 58bf0dfb0bc8..3d8fc696e130 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,8 +1,9 @@ -workspace(name = "com_github_riselab_ray") +workspace(name = "com_github_ray_project_ray") load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository") -load("//bazel:ray_java.bzl", "generated_maven_jars") -generated_maven_jars() +load("//java:tools/checkstyle/repo.bzl", "checkstyle_repositories") + +checkstyle_repositories() git_repository( name = "com_github_nelhage_rules_boost", diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index 16960674726b..c57f3dd1990d 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -1,15 +1,15 @@ load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library") -load("@com_github_riselab_ray//bazel:ray_java.bzl", "generated_java_libraries") +load("@com_github_ray_project_ray//bazel:ray_java.bzl", "generated_java_libraries") generated_java_libraries() java_library( name = "org_apache_arrow_arrow_plasma", srcs = glob(["java/plasma/src/main/java/**/*.java"]), - visibility = ["//visibility:public"], data = [":plasma_client_java"], + visibility = ["//visibility:public"], deps = [ - "@org_slf4j_slf4j_api//jar", + "@mvnorg_slf4j_slf4j_api//jar", ], ) @@ -18,7 +18,10 @@ java_binary( srcs = ["java/plasma/src/test/java/org/apache/arrow/plasma/PlasmaClientTest.java"], main_class = "org.apache.arrow.plasma.PlasmaClientTest", visibility = ["//visibility:public"], - deps = [":org_apache_arrow_arrow_plasma", "@junit_junit//jar",], + deps = [ + ":org_apache_arrow_arrow_plasma", + "@mvnjunit_junit//jar", + ], ) cc_library( diff --git a/bazel/ray_java.bzl b/bazel/ray_java.bzl deleted file mode 100644 index aefbc1df7466..000000000000 --- a/bazel/ray_java.bzl +++ /dev/null @@ -1,1578 +0,0 @@ -# The following dependencies were calculated from: -# -# generate_workspace --maven_project=/home/admin/ruifang.crf/code/X/arrow/java --repositories=http://mvn.dev.alipay.net:8080/artifactory/repo -# TODO: now this file is not automatically generated - - -def generated_maven_jars(): - # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT - # pom.xml got requested version - # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 wanted version 1.1.2 - # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 wanted version 1.1.2 - # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "ch_qos_logback_logback_classic", - artifact = "ch.qos.logback:logback-classic:1.2.3", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "7c4f3c474fb2c041d8028740440937705ebb473a", - ) - - - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "com_google_code_findbugs_jsr305", - artifact = "com.google.code.findbugs:jsr305:3.0.2", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "25ea2e8b0c338a877313bd4672d3fe056ea78f0d", - ) - - - # io.grpc:grpc-protobuf:jar:1.14.0 - native.maven_jar( - name = "io_grpc_grpc_protobuf_lite", - artifact = "io.grpc:grpc-protobuf-lite:1.14.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "2ac8c28ca927f954eaa228a931d9c163cf3d860f", - ) - - - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT wanted version 2.10 - native.maven_jar( - name = "joda_time_joda_time", - artifact = "joda-time:joda-time:2.9.9", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "f7b520c458572890807d143670c9b24f4de90897", - ) - - - # org.mockito:mockito-core:jar:2.7.22 - native.maven_jar( - name = "net_bytebuddy_byte_buddy_agent", - artifact = "net.bytebuddy:byte-buddy-agent:1.6.11", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "0200d9c012befccd211ff91082a151257b1dc084", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 - native.maven_jar( - name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_converter_classic", - artifact = "de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:0.9.44", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "6f08f9bfd65869a51edc922432f46024d94f3ccd", - ) - - - # io.netty:netty-handler-proxy:jar:4.1.22.Final got requested version - # io.netty:netty-handler:jar:4.1.22.Final got requested version - # io.netty:netty-codec:jar:4.1.22.Final - native.maven_jar( - name = "io_netty_netty_transport", - artifact = "io.netty:netty-transport:4.1.22.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "3bd455cd9e5e5fb2e08fd9cd0acfa54c079ca989", - ) - - - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "com_carrotsearch_hppc", - artifact = "com.carrotsearch:hppc:0.7.2", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "710398361f2ae8fd594a133e3619045c16b24137", - ) - - - # io.netty:netty-codec-http2:jar:4.1.22.Final - native.maven_jar( - name = "io_netty_netty_handler", - artifact = "io.netty:netty-handler:4.1.22.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "a3a16b17d5a5ed6f784b0daba95e28d940356109", - ) - - - # io.grpc:grpc-netty:jar:1.14.0 - native.maven_jar( - name = "io_netty_netty_handler_proxy", - artifact = "io.netty:netty-handler-proxy:4.1.27.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "1a822ce7760bc6eb4937b7e448c9e081fedcc807", - ) - - - # org.apache.arrow:arrow-java-root:pom:0.12.0HOT - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "org_mockito_mockito_core", - artifact = "org.mockito:mockito-core:2.7.22", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "fcf63bc8010ca77991e3cadd8d33ad1a40603404", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-core:jar:0.9.44 got requested version - # de.huxhorn.lilith:de.huxhorn.lilith.data.logging.protobuf:jar:0.9.44 got requested version - # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 - # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 got requested version - native.maven_jar( - name = "de_huxhorn_sulky_de_huxhorn_sulky_codec", - artifact = "de.huxhorn.sulky:de.huxhorn.sulky.codec:0.9.17", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "4a2a917f1cc4b75d71aadeef4f4c178f229febc2", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.data.logging:jar:0.9.44 - native.maven_jar( - name = "de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", - artifact = "de.huxhorn.lilith:de.huxhorn.lilith.data.eventsource:0.9.44", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "618f853d75d59855c955b3a1bf0f447476f8027f", - ) - - - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "com_google_flatbuffers_flatbuffers_java", - artifact = "com.google.flatbuffers:flatbuffers-java:1.9.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "460bc8f6411768659c1ffb529592e251a808b9f2", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version - # de.huxhorn.lilith:de.huxhorn.lilith.data.logging.protobuf:jar:0.9.44 wanted version 2.5.0 - # io.grpc:grpc-protobuf:jar:1.14.0 - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT wanted version 2.5.0 - native.maven_jar( - name = "com_google_protobuf_protobuf_java", - artifact = "com.google.protobuf:protobuf-java:3.5.1", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "8c3492f7662fa1cbf8ca76a0f5eb1146f7725acd", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "io_netty_netty_tcnative_boringssl_static", - artifact = "io.netty:netty-tcnative-boringssl-static:2.0.12.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "b884be1450a7fd0854b98743836b8ccb0dfd75a4", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "com_h2database_h2", - artifact = "com.h2database:h2:1.4.196", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "dd0034398d593aa3588c6773faac429bbd9aea0e", - ) - - - # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "org_slf4j_jul_to_slf4j", - artifact = "org.slf4j:jul-to-slf4j:1.7.25", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "0af5364cd6679bfffb114f0dec8a157aaa283b76", - ) - - - native.maven_jar( - name = "net_java_dev_jna_jna", - artifact = "net.java.dev.jna:jna:4.1.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "1c12d070e602efd8021891cdd7fd18bc129372d4", - ) - - - native.maven_jar( - name = "org_mockito_mockito_all", - artifact = "org.mockito:mockito-all:1.10.19", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "539df70269cc254a58cccc5d8e43286b4a73bf30", - ) - - - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "commons_codec_commons_codec", - artifact = "commons-codec:commons-codec:1.10", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "4b95f4897fa13f2cd904aee711aeafc0c5295cd8", - ) - - - native.maven_jar( - name = "com_beust_jcommander", - artifact = "com.beust:jcommander:1.72", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "6375e521c1e11d6563d4f25a07ce124ccf8cd171", - ) - - - # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "org_slf4j_log4j_over_slf4j", - artifact = "org.slf4j:log4j-over-slf4j:1.7.25", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "a87bb47468f47ee7aabbd54f93e133d4215769c3", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "commons_cli_commons_cli", - artifact = "commons-cli:commons-cli:1.4", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "c51c00206bb913cd8612b24abd9fa98ae89719b1", - ) - - - # io.grpc:grpc-core:jar:1.14.0 - native.maven_jar( - name = "com_google_errorprone_error_prone_annotations", - artifact = "com.google.errorprone:error_prone_annotations:2.1.2", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "6dcc08f90f678ac33e5ef78c3c752b6f59e63e0c", - ) - - - # io.netty:netty-transport:jar:4.1.22.Final - native.maven_jar( - name = "io_netty_netty_resolver", - artifact = "io.netty:netty-resolver:4.1.22.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "b5484d17a97cb57b07d2a1ac092c249e47234c17", - ) - - - # io.grpc:grpc-core:jar:1.14.0 - native.maven_jar( - name = "io_opencensus_opencensus_contrib_grpc_metrics", - artifact = "io.opencensus:opencensus-contrib-grpc-metrics:0.12.3", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "a4c7ff238a91b901c8b459889b6d0d7a9d889b4d", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "io_grpc_grpc_stub", - artifact = "io.grpc:grpc-stub:1.14.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "74bfe83c0dc69bf903fff8df3568cbeb8b387d35", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "io_grpc_grpc_protobuf", - artifact = "io.grpc:grpc-protobuf:1.14.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "d429fdc2e0d288b34ea7588bb762eb458f385bd5", - ) - - - # io.netty:netty-handler-proxy:jar:4.1.22.Final - native.maven_jar( - name = "io_netty_netty_codec_socks", - artifact = "io.netty:netty-codec-socks:4.1.22.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "d077b39da2dedc5dc5db50a44e5f4c30353e86f3", - ) - - - # io.netty:netty-codec-http:jar:4.1.22.Final - # io.netty:netty-handler:jar:4.1.22.Final got requested version - # io.netty:netty-codec-socks:jar:4.1.22.Final got requested version - native.maven_jar( - name = "io_netty_netty_codec", - artifact = "io.netty:netty-codec:4.1.22.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "239c0af275952e70bb4adf7cf8c03d88ddc394c9", - ) - - - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version - # io.netty:netty-transport:jar:4.1.22.Final got requested version - # io.netty:netty-handler:jar:4.1.22.Final got requested version - native.maven_jar( - name = "io_netty_netty_buffer", - artifact = "io.netty:netty-buffer:4.1.22.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "15e964a2095031364f534a6e21977f5ee9ca32a9", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 - native.maven_jar( - name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_core", - artifact = "de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-core:0.9.44", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "636cc11e0e09d363af87766550c1deef8ab0c5bf", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "io_grpc_grpc_netty", - artifact = "io.grpc:grpc-netty:1.14.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "21c6edadd45b6869384f8aa0df1663d62c503617", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-core:jar:0.9.44 wanted version 1.1.2 - # ch.qos.logback:logback-classic:jar:1.2.3 - native.maven_jar( - name = "ch_qos_logback_logback_core", - artifact = "ch.qos.logback:logback-core:1.2.3", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "864344400c3d4d92dfeb0a305dc87d953677c03c", - ) - - - # org.mockito:mockito-core:jar:2.7.22 - native.maven_jar( - name = "net_bytebuddy_byte_buddy", - artifact = "net.bytebuddy:byte-buddy:1.6.11", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "8a8f9409e27f1d62c909c7eef2aa7b3a580b4901", - ) - - - # io.opencensus:opencensus-contrib-grpc-metrics:jar:0.12.3 got requested version - # io.grpc:grpc-core:jar:1.14.0 - native.maven_jar( - name = "io_opencensus_opencensus_api", - artifact = "io.opencensus:opencensus-api:0.12.3", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "743f074095f29aa985517299545e72cc99c87de0", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-core:jar:0.9.44 - native.maven_jar( - name = "de_huxhorn_lilith_de_huxhorn_lilith_sender", - artifact = "de.huxhorn.lilith:de.huxhorn.lilith.sender:0.9.44", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "fdeb2f1aaba42cad76c460ac4fd7072b8642520f", - ) - - - native.maven_jar( - name = "org_javassist_javassist", - artifact = "org.javassist:javassist:3.19.0-GA", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "50120f69224dd8684b445a6f3a5b08fe9b5c60f6", - ) - - - # org.mockito:mockito-core:jar:2.7.22 - native.maven_jar( - name = "org_objenesis_objenesis", - artifact = "org.objenesis:objenesis:2.5", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "612ecb799912ccf77cba9b3ed8c813da086076e9", - ) - - - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT got requested version - # io.netty:netty-resolver:jar:4.1.22.Final got requested version - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT got requested version - # io.netty:netty-buffer:jar:4.1.22.Final - native.maven_jar( - name = "io_netty_netty_common", - artifact = "io.netty:netty-common:4.1.22.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "56ff4deca53fc791ed59ac2b72eb6718714a4de9", - ) - - - # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT wanted version 4.12 - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "junit_junit", - artifact = "junit:junit:4.11", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "4e031bb61df09069aeb2bffb4019e7a5034a4ee0", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:jar:0.9.44 - native.maven_jar( - name = "de_huxhorn_lilith_de_huxhorn_lilith_data_logging_protobuf", - artifact = "de.huxhorn.lilith:de.huxhorn.lilith.data.logging.protobuf:0.9.44", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "56a037b365c7ff9b1c348e6d663fd3d778e2b0be", - ) - - - # de.huxhorn.sulky:de.huxhorn.sulky.codec:jar:0.9.17 - # de.huxhorn.lilith:de.huxhorn.lilith.sender:jar:0.9.44 got requested version - native.maven_jar( - name = "de_huxhorn_sulky_de_huxhorn_sulky_io", - artifact = "de.huxhorn.sulky:de.huxhorn.sulky.io:0.9.17", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "6baf1e17f80ba40f9a4b4b97235e72488047db02", - ) - - - native.maven_jar( - name = "log4j_log4j", - artifact = "log4j:log4j:1.2.17", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "5af35056b4d257e4b64b9e8069c0746e8b08629f", - ) - - - native.maven_jar( - name = "org_slf4j_slf4j_log4j12", - artifact = "org.slf4j:slf4j-log4j12:1.7.25", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "110cefe2df103412849d72ef7a67e4e91e4266b4", - ) - - - # org.slf4j:jcl-over-slf4j:jar:1.7.25 got requested version - # org.slf4j:jul-to-slf4j:jar:1.7.25 - # de.huxhorn.lilith:de.huxhorn.lilith.sender:jar:0.9.44 wanted version 1.7.7 - # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 wanted version 1.7.7 - # org.slf4j:log4j-over-slf4j:jar:1.7.25 got requested version - # ch.qos.logback:logback-classic:jar:1.2.3 got requested version - # de.huxhorn.sulky:de.huxhorn.sulky.codec:jar:0.9.17 wanted version 1.7.7 - # de.huxhorn.sulky:de.huxhorn.sulky.formatting:jar:0.9.17 wanted version 1.7.7 - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "org_slf4j_slf4j_api", - artifact = "org.slf4j:slf4j-api:1.7.25", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "da76ca59f6a57ee3102f8f9bd9cee742973efa8a", - ) - - - native.maven_jar( - name = "com_typesafe_config", - artifact = "com.typesafe:config:1.3.2", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "d6ac0ce079f114adce620f2360c92a70b2cb36dc", - ) - - - native.maven_jar( - name = "org_apache_commons_commons_lang3", - artifact = "org.apache.commons:commons-lang3:3.4", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "5fe28b9518e58819180a43a850fbc0dd24b7c050", - ) - - - # io.netty:netty-handler-proxy:jar:4.1.22.Final got requested version - # io.netty:netty-codec-http2:jar:4.1.22.Final - native.maven_jar( - name = "io_netty_netty_codec_http", - artifact = "io.netty:netty-codec-http:4.1.22.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "3805f3ca0d57630200defc7f9bb6ed3382dcb10b", - ) - - - # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOTi - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_classic", - artifact = "de.huxhorn.lilith:de.huxhorn.lilith.logback.appender.multiplex-classic:0.9.44", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "1ac6ffe5fc63f7900f8af54b0dfebf2508b41db7", - ) - - - native.maven_jar( - name = "commons_collections_commons_collections", - artifact = "commons-collections:commons-collections:3.2.2", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "8ad72fe39fa8c91eaaf12aadb21e0c3661fe26d5", - ) - - - # junit:junit:jar:4.11 - native.maven_jar( - name = "org_hamcrest_hamcrest_core", - artifact = "org.hamcrest:hamcrest-core:1.3", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "42a25dc3219429f0e5d060061f71acb49bf010a0", - ) - - - # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "org_jmockit_jmockit", - artifact = "org.jmockit:jmockit:1.33", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "8a2746af90706d8d68f3919e1a87d5e47b8c94d7", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.data.logging:jar:0.9.44 - native.maven_jar( - name = "de_huxhorn_sulky_de_huxhorn_sulky_formatting", - artifact = "de.huxhorn.sulky:de.huxhorn.sulky.formatting:0.9.17", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "0501b143ee18ca9e7807438cc1b0acfa144f657f", - ) - - - # io.grpc:grpc-netty:jar:1.14.0 - native.maven_jar( - name = "io_netty_netty_codec_http2", - artifact = "io.netty:netty-codec-http2:4.1.27.Final", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "3769790a2033667d663f9a526d5b63cfecdbdf4e", - ) - - - native.maven_jar( - name = "net_lingala_zip4j_zip4j", - artifact = "net.lingala.zip4j:zip4j:1.3.2", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "4ba84e98ee017b74cb52f45962f929a221f3074c", - ) - - - native.maven_jar( - name = "org_testng_testng", - artifact = "org.testng:testng:6.9.9", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "1bf509349476d6a48978cc2b04af9caa907781ab", - ) - - native.maven_jar( - name = "org_ini4j_ini4j", - artifact = "org.ini4j:ini4j:0.5.4", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "4a3ee4146a90c619b20977d65951825f5675b560", - ) - - - native.maven_jar( - name = "org_ow2_asm_asm", - artifact = "org.ow2.asm:asm:6.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "bc6fa6b19424bb9592fe43bbc20178f92d403105", - ) - - - # io.grpc:grpc-core:jar:1.14.0 - native.maven_jar( - name = "com_google_code_gson_gson", - artifact = "com.google.code.gson:gson:2.7",#2.8.5 - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "751f548c85fa49f330cecbb1875893f971b33c4e", - ) - - - # org.apache.arrow:arrow-java-root:pom:0.12.0-SNAPSHOT - # pom.xml got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-format:jar:0.12.0-SNAPSHOT - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-memory:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-plasma:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "org_slf4j_jcl_over_slf4j", - artifact = "org.slf4j:jcl-over-slf4j:1.7.25", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "f8c32b13ff142a513eeb5b6330b1588dcb2c0461", - ) - - - # pom.xml got requested version - # com.fasterxml.jackson.core:jackson-databind:bundle:2.7.9 wanted version 2.7.0 - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version - native.maven_jar( - name = "com_fasterxml_jackson_core_jackson_annotations", - artifact = "com.fasterxml.jackson.core:jackson-annotations:2.7.9", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "eb356e825cb73da42f7c902a3fe0276fe32b26c8", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.data.logging.protobuf:jar:0.9.44 got requested version - # de.huxhorn.lilith:de.huxhorn.lilith.logback.classic:jar:0.9.44 - # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 got requested version - native.maven_jar( - name = "de_huxhorn_lilith_de_huxhorn_lilith_data_logging", - artifact = "de.huxhorn.lilith:de.huxhorn.lilith.data.logging:0.9.44", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "3574a72628f59d78ff17210f10de057188043a4c", - ) - - - # io.grpc:grpc-core:jar:1.14.0 - native.maven_jar( - name = "io_grpc_grpc_context", - artifact = "io.grpc:grpc-context:1.14.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "77252b5f926875891aaae5629e6ab2ef968cd6c6", - ) - - - # io.grpc:grpc-protobuf:jar:1.14.0 - native.maven_jar( - name = "com_google_api_grpc_proto_google_common_protos", - artifact = "com.google.api.grpc:proto-google-common-protos:1.0.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "86f070507e28b930e50d218ee5b6788ef0dd05e6", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 - native.maven_jar( - name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_classic", - artifact = "de.huxhorn.lilith:de.huxhorn.lilith.logback.classic:0.9.44", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "f84da242a1988240222f6a06b511a18ff9c64344", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "com_fasterxml_jackson_core_jackson_databind", - artifact = "com.fasterxml.jackson.core:jackson-databind:2.7.9", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "a4c0b14c7dd85bdf4d25da074e90a10fa4b9b88b", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "com_fasterxml_jackson_dataformat_jackson_dataformat_yaml", - artifact = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.7.9", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "163a1bd63ace592d2a449023a1c07056b885ff12", - ) - - - native.maven_jar( - name = "com_github_davidmoten_flatbuffers_java", - artifact = "com.github.davidmoten:flatbuffers-java:1.9.0.1", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "0eaa1b27095bb5127879458bc2ee1b78e00d4a20", - ) - - - native.maven_jar( - name = "redis_clients_jedis", - artifact = "redis.clients:jedis:2.8.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "a9486e200b2195f31ce9b8779a70a5e2450a73ba", - ) - - - native.maven_jar( - name = "org_apache_commons_commons_pool2", - artifact = "org.apache.commons:commons-pool2:2.3", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "62a559a025fd890c30364296ece14643ba9c8c5b", - ) - - - native.maven_jar( - name = "commons_io_commons_io", - artifact = "commons-io:commons-io:2.5", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "2852e6e05fbb95076fc091f6d1780f1f8fe35e0f", - ) - - - # pom.xml got requested version - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT - # io.grpc:grpc-protobuf:jar:1.14.0 got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version - # io.grpc:grpc-protobuf-lite:jar:1.14.0 got requested version - # io.grpc:grpc-core:jar:1.14.0 - # org.apache.arrow.gandiva:arrow-gandiva:jar:0.12.0-SNAPSHOT wanted version 23.0 - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "com_google_guava_guava", - artifact = "com.google.guava:guava:20.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "89507701249388e1ed5ddcf8c41f4ce1be7831ef", - ) - - - # de.huxhorn.lilith:de.huxhorn.lilith.logback.converter-classic:jar:0.9.44 - native.maven_jar( - name = "de_huxhorn_lilith_de_huxhorn_lilith_data_converter", - artifact = "de.huxhorn.lilith:de.huxhorn.lilith.data.converter:0.9.44", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "31e4994f9bc768c43491c3c0d12237ae478d21e3", - ) - - - native.maven_jar( - name = "de_ruedigermoeller_fst", - artifact = "de.ruedigermoeller:fst:2.47", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "281d390ebed24a3621d3053825affc13d0eead8b", - ) - - - # pom.xml got requested version - # com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:bundle:2.7.9 got requested version - # org.apache.arrow:arrow-vector:jar:0.12.0-SNAPSHOT - # com.fasterxml.jackson.core:jackson-databind:bundle:2.7.9 got requested version - # org.apache.arrow:arrow-jdbc:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version - # org.apache.arrow:arrow-tools:jar:0.12.0-SNAPSHOT - native.maven_jar( - name = "com_fasterxml_jackson_core_jackson_core", - artifact = "com.fasterxml.jackson.core:jackson-core:2.7.9", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "09b530cec4fd2eb841ab8e79f19fc7cf0ec487b2", - ) - - - # pom.xml got requested version - # io.grpc:grpc-protobuf:jar:1.14.0 got requested version - # org.apache.arrow:arrow-flight:jar:0.12.0-SNAPSHOT got requested version - # io.grpc:grpc-protobuf-lite:jar:1.14.0 got requested version - # io.grpc:grpc-stub:jar:1.14.0 got requested version - # io.grpc:grpc-netty:jar:1.14.0 - native.maven_jar( - name = "io_grpc_grpc_core", - artifact = "io.grpc:grpc-core:1.14.0", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "526e5be291c96e248789d769c108a084febda07f", - ) - - - # com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:bundle:2.7.9 - native.maven_jar( - name = "org_yaml_snakeyaml", - artifact = "org.yaml:snakeyaml:1.15", - repository = "http://mvn.dev.alipay.net:8080/artifactory/repo/", - sha1 = "3b132bea69e8ee099f416044970997bde80f4ea6", - ) - - - - -def generated_java_libraries(): - native.java_library( - name = "log4j_log4j", - visibility = ["//visibility:public"], - exports = ["@log4j_log4j//jar"], - ) - - - native.java_library( - name = "org_slf4j_slf4j_log4j12", - visibility = ["//visibility:public"], - exports = ["@org_slf4j_slf4j_log4j12//jar"], - runtime_deps = [ - ":log4j_log4j", - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "com_typesafe_config", - visibility = ["//visibility:public"], - exports = ["@com_typesafe_config//jar"], - ) - - - native.java_library( - name = "org_apache_commons_commons_lang3", - visibility = ["//visibility:public"], - exports = ["@org_apache_commons_commons_lang3//jar"], - ) - - - native.java_library( - name = "com_google_code_findbugs_jsr305", - visibility = ["//visibility:public"], - exports = ["@com_google_code_findbugs_jsr305//jar"], - ) - - - native.java_library( - name = "io_grpc_grpc_core", - visibility = ["//visibility:public"], - exports = ["@io_grpc_grpc_core//jar"], - runtime_deps = [ - ":com_google_code_findbugs_jsr305", - ":com_google_code_gson_gson", - ":com_google_errorprone_error_prone_annotations", - ":com_google_guava_guava", - ":io_grpc_grpc_context", - ":io_opencensus_opencensus_api", - ":io_opencensus_opencensus_contrib_grpc_metrics", - ], - ) - - - native.java_library( - name = "ch_qos_logback_logback_classic", - visibility = ["//visibility:public"], - exports = ["@ch_qos_logback_logback_classic//jar"], - runtime_deps = [ - ":ch_qos_logback_logback_core", - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "joda_time_joda_time", - visibility = ["//visibility:public"], - exports = ["@joda_time_joda_time//jar"], - ) - - - native.java_library( - name = "net_bytebuddy_byte_buddy_agent", - visibility = ["//visibility:public"], - exports = ["@net_bytebuddy_byte_buddy_agent//jar"], - ) - - - native.java_library( - name = "io_grpc_grpc_protobuf_lite", - visibility = ["//visibility:public"], - exports = ["@io_grpc_grpc_protobuf_lite//jar"], - runtime_deps = [ - ":com_google_guava_guava", - ":io_grpc_grpc_core", - ], - ) - - - native.java_library( - name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_converter_classic", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_logback_converter_classic//jar"], - runtime_deps = [ - ":ch_qos_logback_logback_classic", - ":de_huxhorn_lilith_de_huxhorn_lilith_data_converter", - ":de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", - ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging", - ":de_huxhorn_lilith_de_huxhorn_lilith_logback_classic", - ":de_huxhorn_sulky_de_huxhorn_sulky_codec", - ":de_huxhorn_sulky_de_huxhorn_sulky_formatting", - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "io_netty_netty_transport", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_transport//jar"], - runtime_deps = [ - ":io_netty_netty_buffer", - ":io_netty_netty_common", - ":io_netty_netty_resolver", - ], - ) - - - native.java_library( - name = "com_carrotsearch_hppc", - visibility = ["//visibility:public"], - exports = ["@com_carrotsearch_hppc//jar"], - ) - - - native.java_library( - name = "org_mockito_mockito_core", - visibility = ["//visibility:public"], - exports = ["@org_mockito_mockito_core//jar"], - runtime_deps = [ - ":net_bytebuddy_byte_buddy", - ":net_bytebuddy_byte_buddy_agent", - ":org_objenesis_objenesis", - ], - ) - - - native.java_library( - name = "io_netty_netty_handler_proxy", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_handler_proxy//jar"], - runtime_deps = [ - ":io_netty_netty_codec", - ":io_netty_netty_codec_http", - ":io_netty_netty_codec_socks", - ":io_netty_netty_transport", - ], - ) - - - native.java_library( - name = "de_huxhorn_sulky_de_huxhorn_sulky_codec", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_sulky_de_huxhorn_sulky_codec//jar"], - runtime_deps = [ - ":de_huxhorn_sulky_de_huxhorn_sulky_io", - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource//jar"], - ) - - - native.java_library( - name = "com_google_flatbuffers_flatbuffers_java", - visibility = ["//visibility:public"], - exports = ["@com_google_flatbuffers_flatbuffers_java//jar"], - ) - - - native.java_library( - name = "com_google_protobuf_protobuf_java", - visibility = ["//visibility:public"], - exports = ["@com_google_protobuf_protobuf_java//jar"], - ) - - - native.java_library( - name = "com_h2database_h2", - visibility = ["//visibility:public"], - exports = ["@com_h2database_h2//jar"], - ) - - - native.java_library( - name = "org_slf4j_jul_to_slf4j", - visibility = ["//visibility:public"], - exports = ["@org_slf4j_jul_to_slf4j//jar"], - runtime_deps = [ - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "commons_codec_commons_codec", - visibility = ["//visibility:public"], - exports = ["@commons_codec_commons_codec//jar"], - ) - - - native.java_library( - name = "org_slf4j_log4j_over_slf4j", - visibility = ["//visibility:public"], - exports = ["@org_slf4j_log4j_over_slf4j//jar"], - runtime_deps = [ - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "commons_cli_commons_cli", - visibility = ["//visibility:public"], - exports = ["@commons_cli_commons_cli//jar"], - ) - - - native.java_library( - name = "io_netty_netty_tcnative_boringssl_static", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_tcnative_boringssl_static//jar"], - ) - - - native.java_library( - name = "io_grpc_grpc_stub", - visibility = ["//visibility:public"], - exports = ["@io_grpc_grpc_stub//jar"], - runtime_deps = [ - ":io_grpc_grpc_core", - ], - ) - - - native.java_library( - name = "io_grpc_grpc_protobuf", - visibility = ["//visibility:public"], - exports = ["@io_grpc_grpc_protobuf//jar"], - runtime_deps = [ - ":com_google_api_grpc_proto_google_common_protos", - ":com_google_guava_guava", - ":com_google_protobuf_protobuf_java", - ":io_grpc_grpc_core", - ":io_grpc_grpc_protobuf_lite", - ], - ) - - - native.java_library( - name = "io_netty_netty_codec_socks", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_codec_socks//jar"], - runtime_deps = [ - ":io_netty_netty_codec", - ], - ) - - - native.java_library( - name = "io_netty_netty_codec", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_codec//jar"], - runtime_deps = [ - ":io_netty_netty_buffer", - ":io_netty_netty_common", - ":io_netty_netty_resolver", - ":io_netty_netty_transport", - ], - ) - - - native.java_library( - name = "io_netty_netty_buffer", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_buffer//jar"], - runtime_deps = [ - ":io_netty_netty_common", - ], - ) - - - native.java_library( - name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_core", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_core//jar"], - runtime_deps = [ - ":ch_qos_logback_logback_core", - ":de_huxhorn_lilith_de_huxhorn_lilith_sender", - ":de_huxhorn_sulky_de_huxhorn_sulky_codec", - ":de_huxhorn_sulky_de_huxhorn_sulky_io", - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "io_grpc_grpc_netty", - visibility = ["//visibility:public"], - exports = ["@io_grpc_grpc_netty//jar"], - runtime_deps = [ - ":com_google_code_findbugs_jsr305", - ":com_google_code_gson_gson", - ":com_google_errorprone_error_prone_annotations", - ":com_google_guava_guava", - ":io_grpc_grpc_context", - ":io_grpc_grpc_core", - ":io_netty_netty_buffer", - ":io_netty_netty_codec", - ":io_netty_netty_codec_http", - ":io_netty_netty_codec_http2", - ":io_netty_netty_codec_socks", - ":io_netty_netty_common", - ":io_netty_netty_handler", - ":io_netty_netty_handler_proxy", - ":io_netty_netty_resolver", - ":io_netty_netty_transport", - ":io_opencensus_opencensus_api", - ":io_opencensus_opencensus_contrib_grpc_metrics", - ], - ) - - - native.java_library( - name = "ch_qos_logback_logback_core", - visibility = ["//visibility:public"], - exports = ["@ch_qos_logback_logback_core//jar"], - ) - - - native.java_library( - name = "net_bytebuddy_byte_buddy", - visibility = ["//visibility:public"], - exports = ["@net_bytebuddy_byte_buddy//jar"], - ) - - - native.java_library( - name = "io_opencensus_opencensus_api", - visibility = ["//visibility:public"], - exports = ["@io_opencensus_opencensus_api//jar"], - ) - - - native.java_library( - name = "de_huxhorn_lilith_de_huxhorn_lilith_sender", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_sender//jar"], - runtime_deps = [ - ":de_huxhorn_sulky_de_huxhorn_sulky_io", - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "com_google_errorprone_error_prone_annotations", - visibility = ["//visibility:public"], - exports = ["@com_google_errorprone_error_prone_annotations//jar"], - ) - - - native.java_library( - name = "io_netty_netty_resolver", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_resolver//jar"], - runtime_deps = [ - ":io_netty_netty_common", - ], - ) - - - native.java_library( - name = "io_opencensus_opencensus_contrib_grpc_metrics", - visibility = ["//visibility:public"], - exports = ["@io_opencensus_opencensus_contrib_grpc_metrics//jar"], - runtime_deps = [ - ":io_opencensus_opencensus_api", - ], - ) - - - native.java_library( - name = "org_javassist_javassist", - visibility = ["//visibility:public"], - exports = ["@org_javassist_javassist//jar"], - ) - - - native.java_library( - name = "org_objenesis_objenesis", - visibility = ["//visibility:public"], - exports = ["@org_objenesis_objenesis//jar"], - ) - - - native.java_library( - name = "io_netty_netty_common", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_common//jar"], - ) - - - native.java_library( - name = "junit_junit", - visibility = ["//visibility:public"], - exports = ["@junit_junit//jar"], - runtime_deps = [ - ":org_hamcrest_hamcrest_core", - ], - ) - - - native.java_library( - name = "de_huxhorn_lilith_de_huxhorn_lilith_data_logging_protobuf", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_data_logging_protobuf//jar"], - runtime_deps = [ - ":com_google_protobuf_protobuf_java", - ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging", - ":de_huxhorn_sulky_de_huxhorn_sulky_codec", - ], - ) - - - native.java_library( - name = "de_huxhorn_sulky_de_huxhorn_sulky_io", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_sulky_de_huxhorn_sulky_io//jar"], - ) - - - native.java_library( - name = "org_slf4j_slf4j_api", - visibility = ["//visibility:public"], - exports = ["@org_slf4j_slf4j_api//jar"], - ) - - - native.java_library( - name = "io_netty_netty_codec_http", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_codec_http//jar"], - runtime_deps = [ - ":io_netty_netty_buffer", - ":io_netty_netty_codec", - ":io_netty_netty_common", - ":io_netty_netty_resolver", - ":io_netty_netty_transport", - ], - ) - - - native.java_library( - name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_classic", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_classic//jar"], - runtime_deps = [ - ":ch_qos_logback_logback_classic", - ":ch_qos_logback_logback_core", - ":com_google_protobuf_protobuf_java", - ":de_huxhorn_lilith_de_huxhorn_lilith_data_converter", - ":de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", - ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging", - ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging_protobuf", - ":de_huxhorn_lilith_de_huxhorn_lilith_logback_appender_multiplex_core", - ":de_huxhorn_lilith_de_huxhorn_lilith_logback_classic", - ":de_huxhorn_lilith_de_huxhorn_lilith_logback_converter_classic", - ":de_huxhorn_lilith_de_huxhorn_lilith_sender", - ":de_huxhorn_sulky_de_huxhorn_sulky_codec", - ":de_huxhorn_sulky_de_huxhorn_sulky_formatting", - ":de_huxhorn_sulky_de_huxhorn_sulky_io", - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "org_hamcrest_hamcrest_core", - visibility = ["//visibility:public"], - exports = ["@org_hamcrest_hamcrest_core//jar"], - ) - - - native.java_library( - name = "io_netty_netty_handler", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_handler//jar"], - runtime_deps = [ - ":io_netty_netty_buffer", - ":io_netty_netty_codec", - ":io_netty_netty_transport", - ], - ) - - - native.java_library( - name = "org_jmockit_jmockit", - visibility = ["//visibility:public"], - exports = ["@org_jmockit_jmockit//jar"], - ) - - - native.java_library( - name = "de_huxhorn_sulky_de_huxhorn_sulky_formatting", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_sulky_de_huxhorn_sulky_formatting//jar"], - runtime_deps = [ - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "io_netty_netty_codec_http2", - visibility = ["//visibility:public"], - exports = ["@io_netty_netty_codec_http2//jar"], - runtime_deps = [ - ":io_netty_netty_buffer", - ":io_netty_netty_codec", - ":io_netty_netty_codec_http", - ":io_netty_netty_common", - ":io_netty_netty_handler", - ":io_netty_netty_resolver", - ":io_netty_netty_transport", - ], - ) - - - native.java_library( - name = "net_lingala_zip4j_zip4j", - visibility = ["//visibility:public"], - exports = ["@net_lingala_zip4j_zip4j//jar"], - ) - - - native.java_library( - name = "org_ini4j_ini4j", - visibility = ["//visibility:public"], - exports = ["@org_ini4j_ini4j//jar"], - ) - - - native.java_library( - name = "org_ow2_asm_asm", - visibility = ["//visibility:public"], - exports = ["@org_ow2_asm_asm//jar"], - ) - - - native.java_library( - name = "com_google_code_gson_gson", - visibility = ["//visibility:public"], - exports = ["@com_google_code_gson_gson//jar"], - ) - - - native.java_library( - name = "org_slf4j_jcl_over_slf4j", - visibility = ["//visibility:public"], - exports = ["@org_slf4j_jcl_over_slf4j//jar"], - runtime_deps = [ - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "com_fasterxml_jackson_core_jackson_annotations", - visibility = ["//visibility:public"], - exports = ["@com_fasterxml_jackson_core_jackson_annotations//jar"], - ) - - - native.java_library( - name = "de_huxhorn_lilith_de_huxhorn_lilith_data_logging", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_data_logging//jar"], - runtime_deps = [ - ":de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", - ":de_huxhorn_sulky_de_huxhorn_sulky_formatting", - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "de_huxhorn_lilith_de_huxhorn_lilith_logback_classic", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_logback_classic//jar"], - runtime_deps = [ - ":de_huxhorn_lilith_de_huxhorn_lilith_data_eventsource", - ":de_huxhorn_lilith_de_huxhorn_lilith_data_logging", - ":de_huxhorn_sulky_de_huxhorn_sulky_formatting", - ":org_slf4j_slf4j_api", - ], - ) - - - native.java_library( - name = "com_fasterxml_jackson_core_jackson_databind", - visibility = ["//visibility:public"], - exports = ["@com_fasterxml_jackson_core_jackson_databind//jar"], - runtime_deps = [ - ":com_fasterxml_jackson_core_jackson_annotations", - ":com_fasterxml_jackson_core_jackson_core", - ], - ) - - - native.java_library( - name = "com_fasterxml_jackson_dataformat_jackson_dataformat_yaml", - visibility = ["//visibility:public"], - exports = ["@com_fasterxml_jackson_dataformat_jackson_dataformat_yaml//jar"], - runtime_deps = [ - ":com_fasterxml_jackson_core_jackson_core", - ":org_yaml_snakeyaml", - ], - ) - - - native.java_library( - name = "io_grpc_grpc_context", - visibility = ["//visibility:public"], - exports = ["@io_grpc_grpc_context//jar"], - ) - - - native.java_library( - name = "com_google_api_grpc_proto_google_common_protos", - visibility = ["//visibility:public"], - exports = ["@com_google_api_grpc_proto_google_common_protos//jar"], - ) - - - native.java_library( - name = "com_github_davidmoten_flatbuffers_java", - visibility = ["//visibility:public"], - exports = ["@com_github_davidmoten_flatbuffers_java//jar"], - ) - - - native.java_library( - name = "redis_clients_jedis", - visibility = ["//visibility:public"], - exports = ["@redis_clients_jedis//jar"], - runtime_deps = [ - ":org_apache_commons_commons_pool2", - ], - ) - - - native.java_library( - name = "org_apache_commons_commons_pool2", - visibility = ["//visibility:public"], - exports = ["@org_apache_commons_commons_pool2//jar"], - ) - - - native.java_library( - name = "com_google_guava_guava", - visibility = ["//visibility:public"], - exports = ["@com_google_guava_guava//jar"], - ) - - - native.java_library( - name = "de_huxhorn_lilith_de_huxhorn_lilith_data_converter", - visibility = ["//visibility:public"], - exports = ["@de_huxhorn_lilith_de_huxhorn_lilith_data_converter//jar"], - ) - - - native.java_library( - name = "de_ruedigermoeller_fst", - visibility = ["//visibility:public"], - exports = ["@de_ruedigermoeller_fst//jar"], - runtime_deps = [ - ":com_fasterxml_jackson_core_jackson_core", - ":org_javassist_javassist", - ":org_objenesis_objenesis", - ], - ) - - - native.java_library( - name = "com_fasterxml_jackson_core_jackson_core", - visibility = ["//visibility:public"], - exports = ["@com_fasterxml_jackson_core_jackson_core//jar"], - ) - - - native.java_library( - name = "org_yaml_snakeyaml", - visibility = ["//visibility:public"], - exports = ["@org_yaml_snakeyaml//jar"], - ) - - - native.java_library( - name = "net_java_dev_jna_jna", - visibility = ["//visibility:public"], - exports = ["@net_java_dev_jna_jna//jar"], - ) - - - native.java_library( - name = "org_mockito_mockito_all", - visibility = ["//visibility:public"], - exports = ["@org_mockito_mockito_all//jar"], - ) - - - native.java_library( - name = "com_beust_jcommander", - visibility = ["//visibility:public"], - exports = ["@com_beust_jcommander//jar"], - ) - - native.java_library( - name = "org_testng_testng", - visibility = ["//visibility:public"], - exports = ["@org_testng_testng//jar"], - ) diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 90ab4f8a53f6..43eec664b94f 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -1,8 +1,11 @@ - +load("//java/tools/checkstyle:checkstyle.bzl", "checkstyle_test") load("//bazel:ray.bzl", "flatbuffer_java_library") -load("//bazel:ray_java.bzl", "generated_java_libraries") -generated_java_libraries() +java_binary( + name = "bazel_deps", + main_class = "com.github.johnynek.bazel_deps.ParseProject", + runtime_deps = ["@bazel_deps//jar"], +) java_import( name = "org_ray_ray_java_root", @@ -33,11 +36,20 @@ java_library( name = "org_ray_ray_api", srcs = glob(["api/src/main/java/**/*.java"]), deps = [ - "@org_slf4j_slf4j_log4j12//jar", - "@log4j_log4j//jar", + "@mvnorg_slf4j_slf4j_log4j12//jar", + "@mvnlog4j_log4j//jar", + "@mvnjavax_xml_bind_jaxb_api//jar", + "@mvncom_sun_xml_bind_jaxb_core//jar", + "@mvncom_sun_xml_bind_jaxb_impl//jar", ], ) +checkstyle_test( + name = "org_ray_ray_api-checkstyle", + target = ":org_ray_ray_api", + size = "small", +) + java_library( name = "org_ray_ray_runtime", srcs = glob(["runtime/src/main/java/**/*.java"]), @@ -46,27 +58,33 @@ java_library( data = ["//:raylet_library_java", ":generate_java_gcs_fbs"], deps = [ ":org_ray_ray_api", - "@com_typesafe_config//jar", - "@org_apache_commons_commons_lang3//jar", - "@de_ruedigermoeller_fst//jar", - "@com_github_davidmoten_flatbuffers_java//jar", - "@redis_clients_jedis//jar", "@plasma//:org_apache_arrow_arrow_plasma", - "@commons_io_commons_io//jar", - "@com_google_guava_guava//jar", - "@net_lingala_zip4j_zip4j//jar", - "@org_slf4j_slf4j_log4j12//jar", - "@org_slf4j_slf4j_api//jar", - "@org_ini4j_ini4j//jar", - "@org_ow2_asm_asm//jar", - "@com_google_code_gson_gson//jar", - "@com_fasterxml_jackson_core_jackson_core//jar", - "@org_apache_commons_commons_pool2//jar", - "@org_javassist_javassist//jar", - "@org_objenesis_objenesis//jar", + "@mvnorg_slf4j_slf4j_api//jar", + #"@mvncom_google_code_gson_gson//jar", + "@mvncom_fasterxml_jackson_core_jackson_core//jar", + "@mvnorg_apache_commons_commons_pool2//jar", + "@mvnorg_javassist_javassist//jar", + "@mvnorg_objenesis_objenesis//jar", + "@mvncom_typesafe_config//jar", + "@mvnorg_apache_commons_commons_lang3//jar", + "@mvnde_ruedigermoeller_fst//jar", + "@mvncom_github_davidmoten_flatbuffers_java//jar", + "@mvnredis_clients_jedis//jar", + "@mvncommons_io_commons_io//jar", + "@mvncom_google_guava_guava//jar", + "@mvnnet_lingala_zip4j_zip4j//jar", + "@mvnorg_slf4j_slf4j_log4j12//jar", + "@mvnorg_ini4j_ini4j//jar", + "@mvnorg_ow2_asm_asm//jar", ], ) +checkstyle_test( + name = "org_ray_ray_runtime-checkstyle", + target = ":org_ray_ray_runtime", + size = "small", +) + java_binary( name = "org_ray_ray_tutorial", srcs = glob(["tutorial/src/main/java/**/*.java"]), @@ -75,40 +93,59 @@ java_binary( deps = [ ":org_ray_ray_api", ":org_ray_ray_runtime", - "@com_google_guava_guava//jar", + "@mvncom_google_guava_guava//jar", ], ) +checkstyle_test( + name = "org_ray_ray_tutorial-checkstyle", + target = ":org_ray_ray_tutorial", + size = "small", +) + java_library( name = "org_ray_ray_test", srcs = glob(["test/src/main/java/**/*.java"]), deps = [ - "@com_google_guava_guava//jar", - "@commons_collections_commons_collections//jar", ":org_ray_ray_api", ":org_ray_ray_runtime", - "@org_apache_commons_commons_lang3//jar", - "@org_slf4j_slf4j_api//jar", + "@mvnorg_apache_commons_commons_lang3//jar", + "@mvnorg_slf4j_slf4j_api//jar", "@plasma//:org_apache_arrow_arrow_plasma", - "@org_testng_testng//jar", + "@mvnorg_testng_testng//jar", + "@mvncommons_collections_commons_collections//jar", + "@mvncom_google_guava_guava//jar", ], ) +checkstyle_test( + name = "org_ray_ray_test-checkstyle", + target = ":org_ray_ray_test", + size = "small", +) + java_library( name = "org_ray_ray_runtime_test", srcs = glob(["runtime/src/test/java/**/*.java"]), deps = [ ":org_ray_ray_api", ":org_ray_ray_runtime", - "@org_testng_testng//jar", - "@org_apache_commons_commons_lang3//jar", - "@com_beust_jcommander//jar", + "@mvnorg_apache_commons_commons_lang3//jar", + "@mvncom_beust_jcommander//jar", + "@mvnorg_testng_testng//jar", ], ) -java_test( - name = "AllTests", +checkstyle_test( + name = "org_ray_ray_runtime_test-checkstyle", + target = ":org_ray_ray_runtime_test", size = "small", +) + +exports_files(["testng.xml"]) + +java_binary( + name = "AllTests", main_class = "org.testng.TestNG", data = ["testng.xml"], runtime_deps = [ diff --git a/java/build.sh b/java/build.sh deleted file mode 100755 index 9fc0aa7ab742..000000000000 --- a/java/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -bazel build -c opt //java:all diff --git a/java/dependencies.yaml b/java/dependencies.yaml new file mode 100644 index 000000000000..f37adc748047 --- /dev/null +++ b/java/dependencies.yaml @@ -0,0 +1,116 @@ +options: + languages: [ "java" ] + licenses: [ "permissive" ] + namePrefix: "mvn" + resolverType: "coursier" + resolvers: + - id: "mavencentral" + type: "default" + url: https://repo.maven.apache.org/maven2/ + thirdPartyDirectory: "java/third_party/" + transitivity: runtime_deps + versionConflictPolicy: highest + +dependencies: + com.puppycrawl.tools: + checkstyle: + lang: java + version: "8.15" + + de.ruedigermoeller: + fst: + lang: java + version: "2.47" + + org.ini4j: + ini4j: + lang: java + version: "0.5.2" + + org.ow2.asm: + asm: + lang: java + version: "6.0" + + com.github.davidmoten: + flatbuffers-java: + lang: java + version: "1.9.0.1" + + com.beust: + jcommander: + lang: java + version: "1.72" + + redis.clients: + jedis: + lang: java + version: "2.8.0" + + commons-io: + commons-io: + lang: java + version: "2.5" + + org.apache.commons: + commons-lang3: + lang: java + version: "3.4" + + commons-codec: + commons-codec: + lang: java + version: "1.4" + + net.lingala.zip4j: + zip4j: + lang: java + version: "1.3.2" + + com.google.guava: + guava: + lang: java + version: "19.0" + + commons-collections: + commons-collections: + lang: java + version: "3.2.2" + + org.slf4j: + slf4j-log4j12: + lang: java + version: "1.7.25" + + com.typesafe: + config: + lang: java + version: "1.3.2" + + net.java.dev.jna: + jna: + lang: java + version: "4.1.0" + + org.mockito: + mockito-all: + lang: java + version: "1.10.19" + + org.testng: + testng: + lang: java + version: "6.9.9" + + javax.xml.bind: + jaxb-api: + lang: java + version: "2.3.0" + + com.sun.xml.bind: + jaxb-core: + lang: java + version: "2.3.0" + jaxb-impl: + lang: java + version: "2.3.0" diff --git a/java/generate_deps.sh b/java/generate_deps.sh new file mode 100644 index 000000000000..3c512c97db25 --- /dev/null +++ b/java/generate_deps.sh @@ -0,0 +1,11 @@ +#/bin/bash + +set -e +# Show explicitly which commands are currently running. +set -x + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +pushd $ROOT_DIR/.. +bazel run //java:bazel_deps -- generate -r `pwd` -s java/third_party/workspace.bzl -d java/dependencies.yaml +popd diff --git a/java/test.sh b/java/test.sh index 4bf037aa0666..6f8f29605aa8 100755 --- a/java/test.sh +++ b/java/test.sh @@ -7,13 +7,13 @@ set -x ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -pushd $ROOT_DIR/../java -echo "Checking code format." -mvn checkstyle:check -popd +# run this file before compile the targets +sh $ROOT_DIR/generate_deps.sh pushd $ROOT_DIR/.. -bazel build -c opt //java:all +bazel test -c opt //java:all + +bazel build -c opt //java:org_ray_ray_java_root popd pushd $ROOT_DIR/../java/test diff --git a/java/third_party/BUILD b/java/third_party/BUILD new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/java/third_party/workspace.bzl b/java/third_party/workspace.bzl new file mode 100644 index 000000000000..4aa7c40b298f --- /dev/null +++ b/java/third_party/workspace.bzl @@ -0,0 +1,73 @@ +# Do not edit. bazel-deps autogenerates this file from java/dependencies.yaml. +def _jar_artifact_impl(ctx): + jar_name = "%s.jar" % ctx.name + ctx.download( + output=ctx.path("jar/%s" % jar_name), + url=ctx.attr.urls, + sha256=ctx.attr.sha256, + executable=False + ) + src_name="%s-sources.jar" % ctx.name + srcjar_attr="" + has_sources = len(ctx.attr.src_urls) != 0 + if has_sources: + ctx.download( + output=ctx.path("jar/%s" % src_name), + url=ctx.attr.src_urls, + sha256=ctx.attr.src_sha256, + executable=False + ) + srcjar_attr ='\n srcjar = ":%s",' % src_name + + build_file_contents = """ +package(default_visibility = ['//visibility:public']) +java_import( + name = 'jar', + jars = ['{jar_name}'],{srcjar_attr} +) +filegroup( + name = 'file', + srcs = [ + '{jar_name}', + '{src_name}' + ], + visibility = ['//visibility:public'] +)\n""".format(jar_name = jar_name, src_name = src_name, srcjar_attr = srcjar_attr) + ctx.file(ctx.path("jar/BUILD"), build_file_contents, False) + return None + +jar_artifact = repository_rule( + attrs = { + "artifact": attr.string(mandatory = True), + "sha256": attr.string(mandatory = True), + "urls": attr.string_list(mandatory = True), + "src_sha256": attr.string(mandatory = False, default=""), + "src_urls": attr.string_list(mandatory = False, default=[]), + }, + implementation = _jar_artifact_impl +) + +def jar_artifact_callback(hash): + src_urls = [] + src_sha256 = "" + source=hash.get("source", None) + if source != None: + src_urls = [source["url"]] + src_sha256 = source["sha256"] + jar_artifact( + artifact = hash["artifact"], + name = hash["name"], + urls = [hash["url"]], + sha256 = hash["sha256"], + src_urls = src_urls, + src_sha256 = src_sha256 + ) + native.bind(name = hash["bind"], actual = hash["actual"]) + +def list_dependencies(): + return [ + ] + +def maven_dependencies(callback = jar_artifact_callback): + for hash in list_dependencies(): + callback(hash) diff --git a/java/tools/checkstyle/BUILD b/java/tools/checkstyle/BUILD new file mode 100644 index 000000000000..f580ac937ca2 --- /dev/null +++ b/java/tools/checkstyle/BUILD @@ -0,0 +1,7 @@ +exports_files([ + "checkstyle.sh", + "license-header.txt", + "checkstyle.xml", + "checkstyle-suppressions.xml", + "repo.bzl", +]) diff --git a/java/tools/checkstyle/README.md b/java/tools/checkstyle/README.md new file mode 100644 index 000000000000..0a83260d8081 --- /dev/null +++ b/java/tools/checkstyle/README.md @@ -0,0 +1,70 @@ +# Checkstyle integration + +## What is Checkstyle + +>[checkstyle](http://checkstyle.sourceforge.net/) is a development tool to +help programmers write Java code that adheres to a coding standard. +It automates the process of checking Java code to spare humans of this +boring (but important) task. This makes it ideal for projects that want to +enforce a coding standard. + +## Integrating with `bazel` + +To apply `checkstyle` check to Java code built with `bazel`, +we use the `checkstyle_test` rule. + +Example: + +`WORKSPACE`: +``` +load("//tools/checkstyle:checkstyle.bzl", "checkstyle_dependencies") +checkstyle_dependencies() +``` + +`BUILD`: + +``` +load("//tools/checkstyle:checkstyle.bzl", "checkstyle_test") + +java_library( + name = "lib", + srcs = ["Lib.java"], +) + +java_library( + name = "ignored_lib", + srcs = ["Lib.java"], + tags = ["checkstyle_ignore"] +) + +checkstyle_test( + name = "lib-checkstyle", + allow_failure = 1, + target = ":lib", +) + +``` + +Sample code illustrates several points: +* To apply Checkstyle to Java target named `lib`, create a `checkstyle_test` +named `lib-checkstyle` (with a dash). Such naming convention is enforced at rule level. +* `checkstyle_test` **is** a test in Bazel context, which means you can do +`bazel test //:lib-checkstyle` and get result of that inspection. +* To perform initial integration, `allow_failure` attribute can be temporarily set, +which will _still_ perform Checkstyle test, but will succeed (as Bazel test) +independently of check's result. +* Targets that do not need to be checked (e.g. generated code), such as `ignored_lib` +can be ignored by assigning `tags = ["checkstyle_ignore"]` +* Checkstyle test **completeness** (that is, all Java code that _needs_ checking +**is** being checked) is enforced via the shell script named `check-for-missing-targets.sh` +ran by CI. If you omit **both** `checkstyle_test` target and `checkstyle_ignore` tag, the script +will exit with non-zero code and print out non-covered targets. + +## Configuring Checkstyle + +Checkstyle is configured by an XML file. By default, current rule implementation uses +`//tools/checkstyle:config.xml` which is based on Google Java style +with added checks of license header +(via [`RegexpHeader`](http://checkstyle.sourceforge.net/config_header.html#RegexpHeader) +and `//tools/checkstyle:license-header.txt`) and package header +(via [`PackageName`](http://checkstyle.sourceforge.net/config_naming.html#PackageName)) diff --git a/java/tools/checkstyle/check-for-missing-targets.sh b/java/tools/checkstyle/check-for-missing-targets.sh new file mode 100755 index 000000000000..a28b6d2a1db4 --- /dev/null +++ b/java/tools/checkstyle/check-for-missing-targets.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# This script verifies that all Java targets except those having assigned `checkstyle_ignore` tag +# do have a accompanying Checkstyle target + +# [1] Get all Java targets except those having assigned `checkstyle_ignore` tag +bazel query 'kind(java_*, //...) - kind(java_proto_library, //...) - //third_party/... - attr("tags", "checkstyle_ignore", //...)' | sort >/tmp/all_java_targets +# [2] Get all checkstyle targets' names, replace '-checkstyle' with nothing so it matches Java target name +bazel query 'kind(checkstyle_test, //...)' | sed -e 's|-checkstyle||g' | sort >/tmp/checkstyle_covered_java_targets + +# [3] Targets in [1] which are not in [2] +NON_COVERED_JAVA_TARGETS=$(comm -23 /tmp/all_java_targets /tmp/checkstyle_covered_java_targets) + +# Having non-empty [3] is an error +if [[ ! -z "$NON_COVERED_JAVA_TARGETS" ]]; then + echo "$(tput setaf 1)[!] These java targets do not have accompanying checkstyle targets:"$(tput sgr0) + echo ${NON_COVERED_JAVA_TARGETS} + exit 1 +fi diff --git a/java/checkstyle-suppressions.xml b/java/tools/checkstyle/checkstyle-suppressions.xml similarity index 100% rename from java/checkstyle-suppressions.xml rename to java/tools/checkstyle/checkstyle-suppressions.xml diff --git a/java/tools/checkstyle/checkstyle.bzl b/java/tools/checkstyle/checkstyle.bzl new file mode 100644 index 000000000000..ec44d48d5a27 --- /dev/null +++ b/java/tools/checkstyle/checkstyle.bzl @@ -0,0 +1,127 @@ +#Structure representing info about Java source files +JavaSourceFiles = provider( + fields = { + 'files' : 'java source files' + } +) + +# This aspect is responsible for collecting Java sources for target +# When applied to target, `JavaSourceFiles` struct will be attached to +# target info +def collect_sources_impl(target, ctx): + files = [] + if hasattr(ctx.rule.attr, 'srcs'): + for src in ctx.rule.attr.srcs: + for file in src.files: + if file.extension == 'java': + files.append(file) + return [JavaSourceFiles(files = files)] + + +collect_sources = aspect( + implementation = collect_sources_impl, +) + + +# `ctx` is rule context: https://docs.bazel.build/versions/master/skylark/lib/ctx.html +def _checkstyle_test_impl(ctx): + # verify target name matches naming conventions + if "{}-checkstyle".format(ctx.attr.target.label.name) != ctx.attr.name: + fail("target should follow `{java_library target name}-checkstyle` pattern") + + suppressions = ctx.file.suppressions + opts = ctx.attr.opts + sopts = ctx.attr.string_opts + + # Checkstyle and its dependencies + checkstyle_dependencies = ctx.attr._checkstyle.java.transitive_runtime_deps + classpath = ":".join([file.path for file in checkstyle_dependencies]) + + args = "" + inputs = [] + if ctx.file.config: + args += " -c %s" % ctx.file.config.path + inputs.append(ctx.file.config) + if suppressions: + inputs.append(suppressions) + + # Build command to run Checkstyle test + cmd = " ".join( + ["java -cp %s com.puppycrawl.tools.checkstyle.Main" % classpath] + + [args] + + ["--%s" % x for x in opts] + + ["--%s %s" % (k, sopts[k]) for k in sopts] + + [file.path for file in ctx.attr.target[JavaSourceFiles].files] + ) + + # Wrap checkstyle command in a shell script so allow_failure is supported + ctx.actions.expand_template( + template = ctx.file._checkstyle_sh_template, + output = ctx.outputs.checkstyle_script, + substitutions = { + "{command}" : cmd, + "{allow_failure}": str(int(ctx.attr.allow_failure)), + }, + is_executable = True, + ) + + files = [ctx.outputs.checkstyle_script, ctx.file.license] + ctx.attr.target[JavaSourceFiles].files + checkstyle_dependencies.to_list() + inputs + runfiles = ctx.runfiles( + files = files, + collect_data = True + ) + return DefaultInfo( + executable = ctx.outputs.checkstyle_script, + files = depset(files), + runfiles = runfiles, + ) + +checkstyle_test = rule( + implementation = _checkstyle_test_impl, + test = True, + attrs = { + "config": attr.label( + allow_single_file=True, + doc = "A checkstyle configuration file", + default = "//java/tools/checkstyle:checkstyle.xml", + ), + "suppressions": attr.label( + allow_single_file=True, + doc = ("A file for specifying files and lines " + + "that should be suppressed from checks." + + "Example: https://github.com/checkstyle/checkstyle/blob/master/config/suppressions.xml"), + default = "//java/tools/checkstyle:checkstyle-suppressions.xml", + ), + "license": attr.label( + allow_single_file=True, + doc = "A license file that can be used with the checkstyle license target", + default = "//java/tools/checkstyle:license-header.txt", + ), + "opts": attr.string_list( + doc = "Options to be passed on the command line that have no argument" + ), + "string_opts": attr.string_dict( + doc = "Options to be passed on the command line that have an argument" + ), + "target": attr.label( + doc = "The java_library target to check sources on", + aspects = [collect_sources], + mandatory = True + ), + "allow_failure": attr.bool( + default = False, + doc = "Successfully finish the test even if checkstyle failed" + ), + "_checkstyle_sh_template": attr.label( + allow_single_file = True, + default = "//java/tools/checkstyle:checkstyle.sh" + ), + "_checkstyle": attr.label( + default = "//java/third_party/com/puppycrawl/tools:checkstyle" + ), + }, + outputs = { + "checkstyle_script": "%{name}.sh", + }, +) + diff --git a/java/tools/checkstyle/checkstyle.sh b/java/tools/checkstyle/checkstyle.sh new file mode 100755 index 000000000000..622a9cda9db9 --- /dev/null +++ b/java/tools/checkstyle/checkstyle.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +COMMAND="{command}" +ALLOW_FAILURE="{allow_failure}" + +echo "ALLOW_FAILURE IS $ALLOW_FAILURE" + +source ~/.bash_profile + +${COMMAND} +result=$? + +if [[ ${ALLOW_FAILURE} -eq 1 ]]; then + exit 0 +fi + +exit ${result} diff --git a/java/checkstyle.xml b/java/tools/checkstyle/checkstyle.xml similarity index 100% rename from java/checkstyle.xml rename to java/tools/checkstyle/checkstyle.xml diff --git a/java/tools/checkstyle/license-header.txt b/java/tools/checkstyle/license-header.txt new file mode 100644 index 000000000000..2d4ccdb2ee66 --- /dev/null +++ b/java/tools/checkstyle/license-header.txt @@ -0,0 +1,15 @@ +^\W* +^\W*Copyright 20[0-9]{2} The StartupOS Authors. +^\W* +^\W*Licensed under the Apache License, Version 2.0 \(the "License"\); +^\W*you may not use this file except in compliance with the License. +^\W*You may obtain a copy of the License at +^\W* +^\W*https://www.apache.org/licenses/LICENSE-2.0 +^\W* +^\W*Unless required by applicable law or agreed to in writing, software +^\W*distributed under the License is distributed on an "AS IS" BASIS, +^\W*WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +^\W*See the License for the specific language governing permissions and +^\W*limitations under the License. +^\W* \ No newline at end of file diff --git a/java/tools/checkstyle/repo.bzl b/java/tools/checkstyle/repo.bzl new file mode 100644 index 000000000000..8119b8d4fe4f --- /dev/null +++ b/java/tools/checkstyle/repo.bzl @@ -0,0 +1,21 @@ +load("//java/third_party:workspace.bzl", "maven_dependencies") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_jar") + +def __maybe(repo_rule, name, **kwargs): + if name not in native.existing_rules(): + repo_rule(name = name, **kwargs) + +def __maven_repositories(): + maven_dependencies() + +def __bazel_deps(): + __maybe( + http_jar, + name = "bazel_deps", + sha256 = "98b05c2826f2248f70e7356dc6c78bc52395904bb932fbb409a5abf5416e4292", + urls = ["https://github.com/oferb/startupos-binaries/releases/download/0.1.01/bazel_deps.jar"], + ) + +def checkstyle_repositories(): + __maven_repositories() + __bazel_deps() From dcb1c21881de4ec2511538f8b2deda5ce387b3ec Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Mon, 11 Mar 2019 19:47:34 +0800 Subject: [PATCH 056/115] modify bazel format --- bazel/BUILD.plasma | 2 +- bazel/ray.bzl | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index c57f3dd1990d..d89868628d7e 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -19,7 +19,7 @@ java_binary( main_class = "org.apache.arrow.plasma.PlasmaClientTest", visibility = ["//visibility:public"], deps = [ - ":org_apache_arrow_arrow_plasma", + ":org_apache_arrow_arrow_plasma", "@mvnjunit_junit//jar", ], ) diff --git a/bazel/ray.bzl b/bazel/ray.bzl index 2f68b25127b6..85ae845ef1c0 100644 --- a/bazel/ray.bzl +++ b/bazel/ray.bzl @@ -21,4 +21,3 @@ def flatbuffer_java_library(name, srcs, outs, out_prefix, includes = [], include include_paths = include_paths, includes = includes, ) - From b62e8c808637df2314eb6c4b4a16cd460be517d1 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Mon, 11 Mar 2019 20:03:42 +0800 Subject: [PATCH 057/115] Some code errors --- WORKSPACE | 4 ++-- java/{tools/checkstyle => }/repo.bzl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename java/{tools/checkstyle => }/repo.bzl (95%) diff --git a/WORKSPACE b/WORKSPACE index 3d8fc696e130..7c18beac161c 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,9 +1,9 @@ workspace(name = "com_github_ray_project_ray") load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository") -load("//java:tools/checkstyle/repo.bzl", "checkstyle_repositories") +load("//java:repo.bzl", "java_repositories") -checkstyle_repositories() +java_repositories() git_repository( name = "com_github_nelhage_rules_boost", diff --git a/java/tools/checkstyle/repo.bzl b/java/repo.bzl similarity index 95% rename from java/tools/checkstyle/repo.bzl rename to java/repo.bzl index 8119b8d4fe4f..e0989b0edba0 100644 --- a/java/tools/checkstyle/repo.bzl +++ b/java/repo.bzl @@ -16,6 +16,6 @@ def __bazel_deps(): urls = ["https://github.com/oferb/startupos-binaries/releases/download/0.1.01/bazel_deps.jar"], ) -def checkstyle_repositories(): +def java_repositories(): __maven_repositories() __bazel_deps() From 04397ea67df351e6a339ad999bb4f76d24cad68e Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Mon, 11 Mar 2019 20:17:09 +0800 Subject: [PATCH 058/115] code errors --- bazel/BUILD.plasma | 3 --- 1 file changed, 3 deletions(-) diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index d89868628d7e..e73798741044 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -1,7 +1,4 @@ load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library") -load("@com_github_ray_project_ray//bazel:ray_java.bzl", "generated_java_libraries") - -generated_java_libraries() java_library( name = "org_apache_arrow_arrow_plasma", From 61d0eb9b2594dbd2f04dab48b64086ffeb183578 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Mon, 11 Mar 2019 20:55:23 +0800 Subject: [PATCH 059/115] modify the error 'pushd not find' --- java/generate_deps.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/java/generate_deps.sh b/java/generate_deps.sh index 3c512c97db25..39ecc9f488ae 100644 --- a/java/generate_deps.sh +++ b/java/generate_deps.sh @@ -1,11 +1,12 @@ -#/bin/bash +#!/usr/bin/env bash set -e -# Show explicitly which commands are currently running. set -x ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -pushd $ROOT_DIR/.. +cd $ROOT_DIR/.. bazel run //java:bazel_deps -- generate -r `pwd` -s java/third_party/workspace.bzl -d java/dependencies.yaml -popd + +set +x +set +e From eaae232d748b1799ef517272e4e8d3cafed74a39 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Mon, 11 Mar 2019 21:34:24 +0800 Subject: [PATCH 060/115] modify compile error --- java/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/test.sh b/java/test.sh index 6f8f29605aa8..f02979a7aeda 100755 --- a/java/test.sh +++ b/java/test.sh @@ -11,9 +11,9 @@ ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) sh $ROOT_DIR/generate_deps.sh pushd $ROOT_DIR/.. -bazel test -c opt //java:all +bazel build -c opt //java:all -bazel build -c opt //java:org_ray_ray_java_root +bazel test -c opt //java:all popd pushd $ROOT_DIR/../java/test From 547064755ee896f3cec9d6c5b37a51b5ce75b006 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Tue, 12 Mar 2019 11:48:35 +0800 Subject: [PATCH 061/115] modify generate_java_gcs_fbs to ensure runtime's compile is successful --- java/BUILD.bazel | 73 +++++++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 43eec664b94f..403c8a1dbf64 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -52,10 +52,14 @@ checkstyle_test( java_library( name = "org_ray_ray_runtime", - srcs = glob(["runtime/src/main/java/**/*.java"]), + srcs = glob([ + "runtime/src/main/java/**/*.java" + ]) + [ + ":generate_java_gcs_fbs", + ], resources = glob(["runtime/src/main/resources/**"]), visibility = ["//visibility:public"], - data = ["//:raylet_library_java", ":generate_java_gcs_fbs"], + data = ["//:raylet_library_java"], deps = [ ":org_ray_ray_api", "@plasma//:org_apache_arrow_arrow_plasma", @@ -156,11 +160,45 @@ java_binary( genrule( name = "generate_java_gcs_fbs", - srcs = [":copy_java_gcs_fbs"], - outs = ["generate_java_gcs_fbs.out"], + srcs = [":java_gcs_fbs"], + outs = [ + "runtime/src/main/java/org/ray/runtime/generated/ActorCheckpointData.java", + "runtime/src/main/java/org/ray/runtime/generated/ActorCheckpointIdData.java", + "runtime/src/main/java/org/ray/runtime/generated/ActorState.java", + "runtime/src/main/java/org/ray/runtime/generated/ActorTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/Arg.java", + "runtime/src/main/java/org/ray/runtime/generated/ClassTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/ClientTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/ConfigTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/CustomSerializerData.java", + "runtime/src/main/java/org/ray/runtime/generated/DriverTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/ErrorTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/ErrorType.java", + "runtime/src/main/java/org/ray/runtime/generated/FunctionTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/GcsTableEntry.java", + "runtime/src/main/java/org/ray/runtime/generated/HeartbeatBatchTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/HeartbeatTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/Language.java", + "runtime/src/main/java/org/ray/runtime/generated/ObjectTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/ProfileEvent.java", + "runtime/src/main/java/org/ray/runtime/generated/ProfileTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/RayResource.java", + "runtime/src/main/java/org/ray/runtime/generated/ResourcePair.java", + "runtime/src/main/java/org/ray/runtime/generated/SchedulingState.java", + "runtime/src/main/java/org/ray/runtime/generated/TablePrefix.java", + "runtime/src/main/java/org/ray/runtime/generated/TablePubsub.java", + "runtime/src/main/java/org/ray/runtime/generated/TaskInfo.java", + "runtime/src/main/java/org/ray/runtime/generated/TaskLeaseData.java", + "runtime/src/main/java/org/ray/runtime/generated/TaskReconstructionData.java", + "runtime/src/main/java/org/ray/runtime/generated/TaskTableData.java", + "runtime/src/main/java/org/ray/runtime/generated/TaskTableTestAndUpdate.java", + ], cmd = """ - python $$(pwd)/java/modify_generated_java_flatbuffers_files.py $$(pwd) - echo $$(pwd) > $@ + for f in $(locations //java:java_gcs_fbs); do + chmod +w $$f + cp -f $$f $(@D)/runtime/src/main/java/org/ray/runtime/generated + done + python $$(pwd)/java/modify_generated_java_flatbuffers_files.py $(@D)/.. """, local = 1, ) @@ -200,26 +238,5 @@ flatbuffer_java_library( "TaskTableData.java", "TaskTableTestAndUpdate.java", ], - out_prefix = "java/runtime/src/main/java/org/ray/runtime/generated", -) - -genrule( - name = "copy_java_gcs_fbs", - srcs = [ - "//java:java_gcs_fbs", - ], - outs = [ - "java_gcs_fbs.out", - ], - cmd = """ - WORK_DIR=$$(pwd) - GENERATED_DIR=$$WORK_DIR/java/runtime/src/main/java/org/ray/runtime/generated - mkdir -p $$GENERATED_DIR - for f in $(locations //java:java_gcs_fbs); do - chmod +w $$f - cp -f $$f $$GENERATED_DIR/ - done - echo $$(pwd) >> $@ - """, - local = 1, + out_prefix = "runtime/src/main/java/org/ray/runtime/generated", ) From 621ad88443801feeaba11f5e96e7d313b929e888 Mon Sep 17 00:00:00 2001 From: Yuhong Guo Date: Tue, 12 Mar 2019 15:34:31 +0800 Subject: [PATCH 062/115] Update Arrow to plasma-client-separation-on-c0a2e73 --- WORKSPACE | 2 +- bazel/BUILD.plasma | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 7c18beac161c..4a3526a9036a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -42,7 +42,7 @@ git_repository( new_git_repository( name = "plasma", build_file = "@//bazel:BUILD.plasma", - commit = "6a27c660ea700febf6fd73b2e851ab96e9315134", + commit = "fa8967cb9ec228f6544b2eecc0f1bd78f29daa4d", remote = "https://github.com/ray-project/arrow", ) diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index e73798741044..efe69098d710 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -29,6 +29,7 @@ cc_library( "cpp/src/arrow/status.cc", "cpp/src/arrow/util/io-util.cc", "cpp/src/arrow/util/logging.cc", + "cpp/src/arrow/util/memory.cc", "cpp/src/arrow/util/thread-pool.cc", ], hdrs = [ @@ -40,6 +41,7 @@ cc_library( "cpp/src/arrow/util/io-util.h", "cpp/src/arrow/util/logging.h", "cpp/src/arrow/util/macros.h", + "cpp/src/arrow/util/memory.h", "cpp/src/arrow/util/string_builder.h", "cpp/src/arrow/util/string_view.h", "cpp/src/arrow/util/thread-pool.h", @@ -62,6 +64,7 @@ cc_library( "cpp/src/plasma/io.cc", "cpp/src/plasma/malloc.cc", "cpp/src/plasma/plasma.cc", + "cpp/src/plasma/plasma_allocator.cc", "cpp/src/plasma/protocol.cc", ], hdrs = [ @@ -73,6 +76,7 @@ cc_library( "cpp/src/plasma/io.h", "cpp/src/plasma/malloc.h", "cpp/src/plasma/plasma.h", + "cpp/src/plasma/plasma_allocator.h", "cpp/src/plasma/plasma_generated.h", "cpp/src/plasma/protocol.h", ], @@ -143,10 +147,12 @@ cc_library( "cpp/src/plasma/dlmalloc.cc", "cpp/src/plasma/events.cc", "cpp/src/plasma/eviction_policy.cc", + "cpp/src/plasma/external_store.cc", ], hdrs = [ "cpp/src/plasma/events.h", "cpp/src/plasma/eviction_policy.h", + "cpp/src/plasma/external_store.h", "cpp/src/plasma/store.h", "cpp/src/plasma/thirdparty/ae/ae.h", "cpp/src/plasma/thirdparty/ae/ae_epoll.c", From 5b835324e886368e8c9a0697458040175d58c4be Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Tue, 12 Mar 2019 16:02:33 +0800 Subject: [PATCH 063/115] modify code with raulchen's and pcmoritz's suggestions --- .gitignore | 1 + bazel/BUILD.plasma | 4 +- build.sh | 11 +- java/BUILD.bazel | 178 ++++++++---------- .../checkstyle-suppressions.xml | 0 java/{tools/checkstyle => }/checkstyle.xml | 0 java/dependencies.yaml | 2 +- java/repo.bzl | 15 +- java/test.sh | 11 +- java/tools/checkstyle/BUILD | 1 - java/tools/checkstyle/checkstyle.bzl | 4 +- 11 files changed, 98 insertions(+), 129 deletions(-) rename java/{tools/checkstyle => }/checkstyle-suppressions.xml (100%) rename java/{tools/checkstyle => }/checkstyle.xml (100%) diff --git a/.gitignore b/.gitignore index 21d8edfc4f96..ca13360c5f55 100644 --- a/.gitignore +++ b/.gitignore @@ -149,6 +149,7 @@ java/**/.settings java/**/.classpath java/**/.project java/runtime/native_dependencies/ +java/third_party/workspace.bzl # python virtual env venv diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index efe69098d710..f221c8af17fd 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -6,7 +6,7 @@ java_library( data = [":plasma_client_java"], visibility = ["//visibility:public"], deps = [ - "@mvnorg_slf4j_slf4j_api//jar", + "@org_slf4j_slf4j_api//jar", ], ) @@ -17,7 +17,7 @@ java_binary( visibility = ["//visibility:public"], deps = [ ":org_apache_arrow_arrow_plasma", - "@mvnjunit_junit//jar", + "@junit_junit//jar", ], ) diff --git a/build.sh b/build.sh index 590c0da1713f..aa0f7b0e2b1b 100755 --- a/build.sh +++ b/build.sh @@ -121,10 +121,11 @@ else $PYTHON_EXECUTABLE -m pip install \ --target=$ROOT_DIR/python/ray/pyarrow_files pyarrow==0.12.0.RAY \ --find-links https://s3-us-west-2.amazonaws.com/arrow-wheels/9357dc130789ee42f8181d8724bee1d5d1509060/index.html - -if [ "$RAY_BUILD_JAVA" == "YES" ]; then - bazel build //:ray_java_pkg -c opt - #soft link + + if [ "$RAY_BUILD_JAVA" == "YES" ]; then + bazel build //:ray_java_pkg -c opt --verbose_failures + # The following are soft links. + # TODO: remove this once cmake is removed mkdir -p $ROOT_DIR/build/src/ray/raylet/ mkdir -p $ROOT_DIR/build/src/ray/gcs/redis_module/ mkdir -p $ROOT_DIR/build/src/ray/thirdparty/redis/src/ @@ -138,7 +139,7 @@ if [ "$RAY_BUILD_JAVA" == "YES" ]; then fi if [ "$RAY_BUILD_PYTHON" == "YES" ]; then - bazel build //:ray_pkg -c opt + bazel build //:ray_pkg -c opt --verbose_failures # Copy files and keep them writeable. This is a workaround, as Bazel # marks all generated files non-writeable. If we would just copy them # over without adding write permission, the copy would fail the next time. diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 403c8a1dbf64..26307706cd42 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -20,8 +20,8 @@ java_import( "liborg_ray_ray_test-src.jar", "org_ray_ray_tutorial_deploy.jar", "org_ray_ray_tutorial_deploy-src.jar", - "AllTests_deploy.jar", - "AllTests_deploy-src.jar", + "all_tests_deploy.jar", + "all_tests_deploy-src.jar", ], deps = [ ":org_ray_ray_api", @@ -36,11 +36,11 @@ java_library( name = "org_ray_ray_api", srcs = glob(["api/src/main/java/**/*.java"]), deps = [ - "@mvnorg_slf4j_slf4j_log4j12//jar", - "@mvnlog4j_log4j//jar", - "@mvnjavax_xml_bind_jaxb_api//jar", - "@mvncom_sun_xml_bind_jaxb_core//jar", - "@mvncom_sun_xml_bind_jaxb_impl//jar", + "@org_slf4j_slf4j_log4j12//jar", + "@log4j_log4j//jar", + "@javax_xml_bind_jaxb_api//jar", + "@com_sun_xml_bind_jaxb_core//jar", + "@com_sun_xml_bind_jaxb_impl//jar", ], ) @@ -58,28 +58,26 @@ java_library( ":generate_java_gcs_fbs", ], resources = glob(["runtime/src/main/resources/**"]), - visibility = ["//visibility:public"], data = ["//:raylet_library_java"], deps = [ ":org_ray_ray_api", "@plasma//:org_apache_arrow_arrow_plasma", - "@mvnorg_slf4j_slf4j_api//jar", - #"@mvncom_google_code_gson_gson//jar", - "@mvncom_fasterxml_jackson_core_jackson_core//jar", - "@mvnorg_apache_commons_commons_pool2//jar", - "@mvnorg_javassist_javassist//jar", - "@mvnorg_objenesis_objenesis//jar", - "@mvncom_typesafe_config//jar", - "@mvnorg_apache_commons_commons_lang3//jar", - "@mvnde_ruedigermoeller_fst//jar", - "@mvncom_github_davidmoten_flatbuffers_java//jar", - "@mvnredis_clients_jedis//jar", - "@mvncommons_io_commons_io//jar", - "@mvncom_google_guava_guava//jar", - "@mvnnet_lingala_zip4j_zip4j//jar", - "@mvnorg_slf4j_slf4j_log4j12//jar", - "@mvnorg_ini4j_ini4j//jar", - "@mvnorg_ow2_asm_asm//jar", + "@org_slf4j_slf4j_api//jar", + "@com_fasterxml_jackson_core_jackson_core//jar", + "@org_apache_commons_commons_pool2//jar", + "@org_javassist_javassist//jar", + "@org_objenesis_objenesis//jar", + "@com_typesafe_config//jar", + "@org_apache_commons_commons_lang3//jar", + "@de_ruedigermoeller_fst//jar", + "@com_github_davidmoten_flatbuffers_java//jar", + "@redis_clients_jedis//jar", + "@commons_io_commons_io//jar", + "@com_google_guava_guava//jar", + "@net_lingala_zip4j_zip4j//jar", + "@org_slf4j_slf4j_log4j12//jar", + "@org_ini4j_ini4j//jar", + "@org_ow2_asm_asm//jar", ], ) @@ -93,11 +91,10 @@ java_binary( name = "org_ray_ray_tutorial", srcs = glob(["tutorial/src/main/java/**/*.java"]), resources = glob(["tutorial/src/main/resources/**"]), - visibility = ["//visibility:public"], deps = [ ":org_ray_ray_api", ":org_ray_ray_runtime", - "@mvncom_google_guava_guava//jar", + "@com_google_guava_guava//jar", ], ) @@ -113,12 +110,12 @@ java_library( deps = [ ":org_ray_ray_api", ":org_ray_ray_runtime", - "@mvnorg_apache_commons_commons_lang3//jar", - "@mvnorg_slf4j_slf4j_api//jar", + "@org_apache_commons_commons_lang3//jar", + "@org_slf4j_slf4j_api//jar", "@plasma//:org_apache_arrow_arrow_plasma", - "@mvnorg_testng_testng//jar", - "@mvncommons_collections_commons_collections//jar", - "@mvncom_google_guava_guava//jar", + "@org_testng_testng//jar", + "@commons_collections_commons_collections//jar", + "@com_google_guava_guava//jar", ], ) @@ -134,9 +131,9 @@ java_library( deps = [ ":org_ray_ray_api", ":org_ray_ray_runtime", - "@mvnorg_apache_commons_commons_lang3//jar", - "@mvncom_beust_jcommander//jar", - "@mvnorg_testng_testng//jar", + "@org_apache_commons_commons_lang3//jar", + "@com_beust_jcommander//jar", + "@org_testng_testng//jar", ], ) @@ -149,7 +146,7 @@ checkstyle_test( exports_files(["testng.xml"]) java_binary( - name = "AllTests", + name = "all_tests", main_class = "org.testng.TestNG", data = ["testng.xml"], runtime_deps = [ @@ -158,40 +155,51 @@ java_binary( ], ) +flatbuffers_generate_files = [ + "ActorCheckpointData.java", + "ActorCheckpointIdData.java", + "ActorState.java", + "ActorTableData.java", + "Arg.java", + "ClassTableData.java", + "ClientTableData.java", + "ConfigTableData.java", + "CustomSerializerData.java", + "DriverTableData.java", + "ErrorTableData.java", + "ErrorType.java", + "FunctionTableData.java", + "GcsTableEntry.java", + "HeartbeatBatchTableData.java", + "HeartbeatTableData.java", + "Language.java", + "ObjectTableData.java", + "ProfileEvent.java", + "ProfileTableData.java", + "RayResource.java", + "ResourcePair.java", + "SchedulingState.java", + "TablePrefix.java", + "TablePubsub.java", + "TaskInfo.java", + "TaskLeaseData.java", + "TaskReconstructionData.java", + "TaskTableData.java", + "TaskTableTestAndUpdate.java", +] + +flatbuffer_java_library( + name = "java_gcs_fbs", + srcs = ["//:gcs_fbs_file"], + outs = flatbuffers_generate_files, + out_prefix = "", +) + genrule( name = "generate_java_gcs_fbs", srcs = [":java_gcs_fbs"], outs = [ - "runtime/src/main/java/org/ray/runtime/generated/ActorCheckpointData.java", - "runtime/src/main/java/org/ray/runtime/generated/ActorCheckpointIdData.java", - "runtime/src/main/java/org/ray/runtime/generated/ActorState.java", - "runtime/src/main/java/org/ray/runtime/generated/ActorTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/Arg.java", - "runtime/src/main/java/org/ray/runtime/generated/ClassTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/ClientTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/ConfigTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/CustomSerializerData.java", - "runtime/src/main/java/org/ray/runtime/generated/DriverTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/ErrorTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/ErrorType.java", - "runtime/src/main/java/org/ray/runtime/generated/FunctionTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/GcsTableEntry.java", - "runtime/src/main/java/org/ray/runtime/generated/HeartbeatBatchTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/HeartbeatTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/Language.java", - "runtime/src/main/java/org/ray/runtime/generated/ObjectTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/ProfileEvent.java", - "runtime/src/main/java/org/ray/runtime/generated/ProfileTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/RayResource.java", - "runtime/src/main/java/org/ray/runtime/generated/ResourcePair.java", - "runtime/src/main/java/org/ray/runtime/generated/SchedulingState.java", - "runtime/src/main/java/org/ray/runtime/generated/TablePrefix.java", - "runtime/src/main/java/org/ray/runtime/generated/TablePubsub.java", - "runtime/src/main/java/org/ray/runtime/generated/TaskInfo.java", - "runtime/src/main/java/org/ray/runtime/generated/TaskLeaseData.java", - "runtime/src/main/java/org/ray/runtime/generated/TaskReconstructionData.java", - "runtime/src/main/java/org/ray/runtime/generated/TaskTableData.java", - "runtime/src/main/java/org/ray/runtime/generated/TaskTableTestAndUpdate.java", + "runtime/src/main/java/org/ray/runtime/generated/" + file for file in flatbuffers_generate_files ], cmd = """ for f in $(locations //java:java_gcs_fbs); do @@ -202,41 +210,3 @@ genrule( """, local = 1, ) - -flatbuffer_java_library( - name = "java_gcs_fbs", - srcs = ["//:gcs_fbs_file"], - outs = [ - "ActorCheckpointData.java", - "ActorCheckpointIdData.java", - "ActorState.java", - "ActorTableData.java", - "Arg.java", - "ClassTableData.java", - "ClientTableData.java", - "ConfigTableData.java", - "CustomSerializerData.java", - "DriverTableData.java", - "ErrorTableData.java", - "ErrorType.java", - "FunctionTableData.java", - "GcsTableEntry.java", - "HeartbeatBatchTableData.java", - "HeartbeatTableData.java", - "Language.java", - "ObjectTableData.java", - "ProfileEvent.java", - "ProfileTableData.java", - "RayResource.java", - "ResourcePair.java", - "SchedulingState.java", - "TablePrefix.java", - "TablePubsub.java", - "TaskInfo.java", - "TaskLeaseData.java", - "TaskReconstructionData.java", - "TaskTableData.java", - "TaskTableTestAndUpdate.java", - ], - out_prefix = "runtime/src/main/java/org/ray/runtime/generated", -) diff --git a/java/tools/checkstyle/checkstyle-suppressions.xml b/java/checkstyle-suppressions.xml similarity index 100% rename from java/tools/checkstyle/checkstyle-suppressions.xml rename to java/checkstyle-suppressions.xml diff --git a/java/tools/checkstyle/checkstyle.xml b/java/checkstyle.xml similarity index 100% rename from java/tools/checkstyle/checkstyle.xml rename to java/checkstyle.xml diff --git a/java/dependencies.yaml b/java/dependencies.yaml index f37adc748047..62ca6af2eb36 100644 --- a/java/dependencies.yaml +++ b/java/dependencies.yaml @@ -1,7 +1,7 @@ options: languages: [ "java" ] licenses: [ "permissive" ] - namePrefix: "mvn" + namePrefix: "" resolverType: "coursier" resolvers: - id: "mavencentral" diff --git a/java/repo.bzl b/java/repo.bzl index e0989b0edba0..a06c285f234b 100644 --- a/java/repo.bzl +++ b/java/repo.bzl @@ -1,15 +1,12 @@ load("//java/third_party:workspace.bzl", "maven_dependencies") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_jar") -def __maybe(repo_rule, name, **kwargs): +def create_if_needed(rule, name, **kwargs): if name not in native.existing_rules(): - repo_rule(name = name, **kwargs) + rule(name = name, **kwargs) -def __maven_repositories(): - maven_dependencies() - -def __bazel_deps(): - __maybe( +def bazel_deps(): + create_if_needed( http_jar, name = "bazel_deps", sha256 = "98b05c2826f2248f70e7356dc6c78bc52395904bb932fbb409a5abf5416e4292", @@ -17,5 +14,5 @@ def __bazel_deps(): ) def java_repositories(): - __maven_repositories() - __bazel_deps() + maven_dependencies() + bazel_deps() diff --git a/java/test.sh b/java/test.sh index f02979a7aeda..4079f1652335 100755 --- a/java/test.sh +++ b/java/test.sh @@ -11,16 +11,17 @@ ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) sh $ROOT_DIR/generate_deps.sh pushd $ROOT_DIR/.. -bazel build -c opt //java:all - -bazel test -c opt //java:all +# bazel checkstyle for java +bazel test //java:all -c opt +# compile all the targets +bazel build //java:all -c opt --verbose_failures popd pushd $ROOT_DIR/../java/test echo "Running tests under cluster mode." -ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml +ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/../java/testng.xml echo "Running tests under single-process mode." -java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/AllTests_deploy.jar $ROOT_DIR/../java/testng.xml +java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/../java/testng.xml popd diff --git a/java/tools/checkstyle/BUILD b/java/tools/checkstyle/BUILD index f580ac937ca2..b323bb6066f8 100644 --- a/java/tools/checkstyle/BUILD +++ b/java/tools/checkstyle/BUILD @@ -3,5 +3,4 @@ exports_files([ "license-header.txt", "checkstyle.xml", "checkstyle-suppressions.xml", - "repo.bzl", ]) diff --git a/java/tools/checkstyle/checkstyle.bzl b/java/tools/checkstyle/checkstyle.bzl index ec44d48d5a27..51129aeb830a 100644 --- a/java/tools/checkstyle/checkstyle.bzl +++ b/java/tools/checkstyle/checkstyle.bzl @@ -83,14 +83,14 @@ checkstyle_test = rule( "config": attr.label( allow_single_file=True, doc = "A checkstyle configuration file", - default = "//java/tools/checkstyle:checkstyle.xml", + default = "//java:checkstyle.xml", ), "suppressions": attr.label( allow_single_file=True, doc = ("A file for specifying files and lines " + "that should be suppressed from checks." + "Example: https://github.com/checkstyle/checkstyle/blob/master/config/suppressions.xml"), - default = "//java/tools/checkstyle:checkstyle-suppressions.xml", + default = "//java:checkstyle-suppressions.xml", ), "license": attr.label( allow_single_file=True, From af00ed900a08af63620b2b83898c71c64b7ac3ca Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Tue, 12 Mar 2019 21:08:30 +0800 Subject: [PATCH 064/115] add soft link to some path that cmake used --- java/test.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/java/test.sh b/java/test.sh index 4079f1652335..170e089bec8c 100755 --- a/java/test.sh +++ b/java/test.sh @@ -17,7 +17,14 @@ bazel test //java:all -c opt bazel build //java:all -c opt --verbose_failures popd -pushd $ROOT_DIR/../java/test +# The following are soft links +# TODO: remove this once cmake is removed +mkdir -p $ROOT_DIR/../build/java/ +ln -sf $ROOT_DIR/../bazel-bin/java/* $ROOT_DIR/../build/java/ +mkdir -p $ROOT_DIR/tutorial/target/ +ln -sf $ROOT_DIR/../bazel-bin/java/org_ray_ray_tutorial_deploy.jar $ROOT_DIR/tutorial/target/ray-tutorial-0.1-SNAPSHOT.jar + +pushd $ROOT_DIR/test echo "Running tests under cluster mode." ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/../java/testng.xml From 87f183ccc6e34145fc89124bfd9703a9d37f0aed Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Wed, 13 Mar 2019 18:17:29 +0800 Subject: [PATCH 065/115] modify java/test.sh with the exitcode 2 --- java/test.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/java/test.sh b/java/test.sh index 170e089bec8c..9df89fdc4b7d 100755 --- a/java/test.sh +++ b/java/test.sh @@ -26,9 +26,13 @@ ln -sf $ROOT_DIR/../bazel-bin/java/org_ray_ray_tutorial_deploy.jar $ROOT_DIR/tut pushd $ROOT_DIR/test echo "Running tests under cluster mode." -ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/../java/testng.xml +ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml || exit_code=$? echo "Running tests under single-process mode." -java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/../java/testng.xml +java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml || exit_code=$? +# exit_code == 2 means there are some tests skiped. +if [ $exit_code -eq 2 ] && [ $exit_code -eq 0 ] ; then + exit $exit_code +fi popd From 2d645153a50fe742669ea4c355e5de92472768e0 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Wed, 13 Mar 2019 18:32:39 +0800 Subject: [PATCH 066/115] modify java/test.sh --- java/test.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/java/test.sh b/java/test.sh index 9df89fdc4b7d..b6f4973109e7 100755 --- a/java/test.sh +++ b/java/test.sh @@ -26,12 +26,17 @@ ln -sf $ROOT_DIR/../bazel-bin/java/org_ray_ray_tutorial_deploy.jar $ROOT_DIR/tut pushd $ROOT_DIR/test echo "Running tests under cluster mode." -ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml || exit_code=$? +ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml || cluster_exit_code=$? + +# exit_code == 2 means there are some tests skiped. +if [ $cluster_exit_code -eq 2 ] && [ $cluster_exit_code -eq 0 ] ; then + exit $exit_code +fi echo "Running tests under single-process mode." -java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml || exit_code=$? +java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml || single_exit_code=$? # exit_code == 2 means there are some tests skiped. -if [ $exit_code -eq 2 ] && [ $exit_code -eq 0 ] ; then +if [ $single_exit_code -eq 2 ] && [ $single_exit_code -eq 0 ] ; then exit $exit_code fi From 755efc7d87a4a2f5a80e0c4d86423b5430875a73 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Thu, 14 Mar 2019 21:40:08 +0800 Subject: [PATCH 067/115] [Java] Package binary dependencies into jar --- .../src/main/java/org/ray/runtime/RayNativeRuntime.java | 6 ++---- java/test/src/main/java/org/ray/api/test/BaseTest.java | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java b/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java index a0fbdf01f28d..2949a27c5f20 100644 --- a/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java +++ b/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java @@ -3,7 +3,6 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import java.io.File; -import java.io.InputStream; import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.file.Files; @@ -88,9 +87,8 @@ public void start() throws Exception { // Copy the file from resources to a temp dir, and load the native library. File file = File.createTempFile(fileName, ""); file.deleteOnExit(); - InputStream in = RayNativeRuntime.class.getResourceAsStream("/" + fileName); - Preconditions.checkNotNull(in, "{} doesn't exist.", fileName); - Files.copy(in, Paths.get(file.getAbsolutePath()), StandardCopyOption.REPLACE_EXISTING); + Files.copy(RayNativeRuntime.class.getResourceAsStream("/" + fileName), + Paths.get(file.getAbsolutePath()), StandardCopyOption.REPLACE_EXISTING); System.load(file.getAbsolutePath()); } } catch (Exception e) { diff --git a/java/test/src/main/java/org/ray/api/test/BaseTest.java b/java/test/src/main/java/org/ray/api/test/BaseTest.java index 366ee9a6737f..259540722d83 100644 --- a/java/test/src/main/java/org/ray/api/test/BaseTest.java +++ b/java/test/src/main/java/org/ray/api/test/BaseTest.java @@ -33,6 +33,7 @@ public void setUpBase(Method method) { @AfterMethod public void tearDownBase() { + Ray.shutdown(); for (File file : filesToDelete) { From 2324c67289c002ad5cbfd7d70b71caef916533e7 Mon Sep 17 00:00:00 2001 From: Jovan Date: Fri, 15 Mar 2019 16:24:06 +0800 Subject: [PATCH 068/115] Fix function manager test --- java/runtime/src/main/resources/DemoApp.java | 27 ++++++++ .../functionmanager/FunctionManagerTest.java | 63 ++++++++++++++----- 2 files changed, 76 insertions(+), 14 deletions(-) create mode 100644 java/runtime/src/main/resources/DemoApp.java diff --git a/java/runtime/src/main/resources/DemoApp.java b/java/runtime/src/main/resources/DemoApp.java new file mode 100644 index 000000000000..f9c493d5a337 --- /dev/null +++ b/java/runtime/src/main/resources/DemoApp.java @@ -0,0 +1,27 @@ +package org.ray.demo; + +import org.ray.api.Ray; +import org.ray.api.RayObject; +import org.ray.api.annotation.RayRemote; + +public class DemoApp { + + @RayRemote + public static String sayHello() { + String ret = "hello"; + return ret; + } + + public static void main(String []args) { + try { + Ray.init(); + RayObject hello = Ray.call(DemoApp::sayHello); + System.out.println(hello.get()); + } catch (Throwable t) { + t.printStackTrace(); + } finally { + Ray.shutdown(); + } + } + +} diff --git a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java index e0307635a9aa..24b2a09a42af 100644 --- a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java +++ b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java @@ -5,6 +5,7 @@ import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.Map; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.ray.api.annotation.RayRemote; @@ -15,6 +16,8 @@ import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import javax.tools.JavaCompiler; +import javax.tools.ToolProvider; /** * Tests for {@link FunctionManager} @@ -117,25 +120,57 @@ public void testLoadFunctionTableForClass() { //TODO(qwang): This is an integration test case, and we should move it to test folder in the future. @Test - public void testGetFunctionFromLocalResource() throws Exception{ - UniqueId driverId = UniqueId.fromHexString("0123456789012345678901234567890123456789"); - - //TODO(qwang): We should use a independent app demo instead of `tutorial`. - final String resourcePath = "/tmp/ray/test/resource"; - final String srcJarPath = System.getProperty("user.dir") + - "/../tutorial/target/ray-tutorial-0.1-SNAPSHOT.jar"; - final String destJarPath = resourcePath + "/" + driverId.toString() + - "/ray-tutorial-0.1-SNAPSHOT.jar"; - - File file = new File(resourcePath + "/" + driverId.toString()); + public void testGetFunctionFromLocalResource() throws Exception { + final String resourcePath = "/tmp/ray/java_test/resource"; + UniqueId driverId = UniqueId.randomId(); + final String driverResourcePath = resourcePath + "/" + driverId.toString(); + final String destDir = System.getProperty("user.dir") + "/org/ray/demo"; + compileAndPack("../runtime/src/main/resources/DemoApp.java", destDir); + + final String sourcePath = destDir + "/DemoApp.jar"; + final String destPath = driverResourcePath + "/DemoApp.jar"; + File file = new File(destPath); file.mkdirs(); - Files.copy(Paths.get(srcJarPath), Paths.get(destJarPath), StandardCopyOption.REPLACE_EXISTING); + Files.move(Paths.get(sourcePath), Paths.get(destPath), + StandardCopyOption.REPLACE_EXISTING); + FunctionDescriptor sayHelloDescriptor = new FunctionDescriptor( + "org.ray.demo.DemoApp", "sayHello", "()Ljava/lang/String;"); final FunctionManager functionManager = new FunctionManager(resourcePath); - FunctionDescriptor sayHelloDescriptor = new FunctionDescriptor("org.ray.exercise.Exercise02", - "sayHello", "()Ljava/lang/String;"); + RayFunction func = functionManager.getFunction(driverId, sayHelloDescriptor); Assert.assertEquals(func.getFunctionDescriptor(), sayHelloDescriptor); } + private static void compileAndPack(String sourceFilePath, String destDir) throws Exception { + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + int result = compiler.run(null, null, null, sourceFilePath); + + if (result != 0) { + throw new RuntimeException(String.format("Failed to compile file : %s", sourceFilePath)); + } + + + String[] packJarCommand = new String[]{ + "jar", + "-cvf", + destDir + "/DemoApp.jar", + "org/ray/demo/DemoApp.class" + }; + + Runtime.getRuntime().exec(packJarCommand); + + // Move to destination directory. + final String sourceFileDir = (new File(sourceFilePath)).getParent(); + final String classFileSourcePath = sourceFileDir + "/DemoApp.class"; + final String classFileDestPath = destDir + "/DemoApp.class"; + File file = new File(classFileDestPath); + file.mkdirs(); + Files.move(Paths.get(classFileSourcePath), Paths.get(classFileDestPath), + StandardCopyOption.REPLACE_EXISTING); + + // Sleep for waiting for jar. + TimeUnit.SECONDS.sleep(1); + } + } From c9a6361b75efe955bb6e4b6e0356833d481d6508 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Fri, 15 Mar 2019 17:00:41 +0800 Subject: [PATCH 069/115] Test1 (#104) --- .gitignore | 1 + .travis.yml | 2 +- BUILD.bazel | 35 ++- WORKSPACE | 7 +- bazel/BUILD.plasma | 37 ++- bazel/ray.bzl | 11 + build.sh | 35 ++- java/BUILD.bazel | 212 ++++++++++++++++++ java/build.sh | 3 - java/dependencies.yaml | 116 ++++++++++ java/generate_deps.sh | 12 + java/repo.bzl | 18 ++ java/test.sh | 36 ++- .../org/ray/api/test/ClientExceptionTest.java | 4 +- java/testng.xml | 10 + java/third_party/BUILD | 0 java/third_party/workspace.bzl | 73 ++++++ java/tools/checkstyle/BUILD | 6 + java/tools/checkstyle/README.md | 70 ++++++ .../checkstyle/check-for-missing-targets.sh | 19 ++ java/tools/checkstyle/checkstyle.bzl | 127 +++++++++++ java/tools/checkstyle/checkstyle.sh | 17 ++ java/tools/checkstyle/license-header.txt | 15 ++ 23 files changed, 831 insertions(+), 35 deletions(-) create mode 100644 java/BUILD.bazel delete mode 100755 java/build.sh create mode 100644 java/dependencies.yaml create mode 100644 java/generate_deps.sh create mode 100644 java/repo.bzl create mode 100644 java/testng.xml create mode 100644 java/third_party/BUILD create mode 100644 java/third_party/workspace.bzl create mode 100644 java/tools/checkstyle/BUILD create mode 100644 java/tools/checkstyle/README.md create mode 100755 java/tools/checkstyle/check-for-missing-targets.sh create mode 100644 java/tools/checkstyle/checkstyle.bzl create mode 100755 java/tools/checkstyle/checkstyle.sh create mode 100644 java/tools/checkstyle/license-header.txt diff --git a/.gitignore b/.gitignore index 21d8edfc4f96..a5c9754c27d6 100644 --- a/.gitignore +++ b/.gitignore @@ -148,6 +148,7 @@ java/**/lib java/**/.settings java/**/.classpath java/**/.project +java/third_party/workspace.bzl java/runtime/native_dependencies/ # python virtual env diff --git a/.travis.yml b/.travis.yml index 62fb4e43d397..dee43a97523c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,9 +25,9 @@ matrix: env: - JDK='Oracle JDK 8' - PYTHON=3.5 PYTHONWARNINGS=ignore - - RAY_USE_CMAKE=1 - RAY_INSTALL_JAVA=1 install: + - ./ci/travis/install-bazel.sh - ./ci/travis/install-dependencies.sh - export PATH="$HOME/miniconda/bin:$PATH" - ./ci/travis/install-ray.sh diff --git a/BUILD.bazel b/BUILD.bazel index b597939896b9..b0331298cb27 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -99,6 +99,7 @@ cc_test( args = ["--gtest_filter=PrintLogTest*"], copts = COPTS, deps = [ + "@boost//:asio", ":ray_util", "@com_google_googletest//:gtest_main", ], @@ -372,7 +373,7 @@ pyx_library( ) cc_binary( - name = "raylet_library_java.so", + name = "libraylet_library_java.so", srcs = [ "src/ray/raylet/lib/java/org_ray_runtime_raylet_RayletClientImpl.h", "src/ray/raylet/lib/java/org_ray_runtime_raylet_RayletClientImpl.cc", @@ -403,8 +404,8 @@ cc_binary( genrule( name = "raylet-jni-darwin-compat", - srcs = [":raylet_library_java.so"], - outs = ["raylet_library_java.dylib"], + srcs = [":libraylet_library_java.so"], + outs = ["libraylet_library_java.dylib"], cmd = "cp $< $@", output_to_bindir = 1, ) @@ -412,16 +413,22 @@ genrule( filegroup( name = "raylet_library_java", srcs = select({ - "@bazel_tools//src/conditions:darwin": [":raylet_library_java.dylib"], - "//conditions:default": [":raylet_library_java.so"], + "@bazel_tools//src/conditions:darwin": [":libraylet_library_java.dylib"], + "//conditions:default": [":libraylet_library_java.so"], }), visibility = ["//visibility:public"], ) +filegroup( + name = "gcs_fbs_file", + srcs = ["src/ray/gcs/format/gcs.fbs"], + visibility = ["//visibility:public"], +) + flatbuffer_py_library( name = "python_gcs_fbs", srcs = [ - "src/ray/gcs/format/gcs.fbs", + ":gcs_fbs_file", ], outs = [ "ActorCheckpointIdData.py", @@ -578,3 +585,19 @@ genrule( mv python $(location ray_pkg) """, ) + +genrule( + name = "ray_java_pkg", + srcs = [ + "//:redis-server", + "//:ray_redis_module.so", + "//:raylet", + "//:raylet_library_java", + "@plasma//:plasma_store_server", + "@plasma//:plasma_client_java", + ], + outs = ["ray_java_pkg.out"], + cmd = """ + echo $$(pwd) > $@ + """, +) diff --git a/WORKSPACE b/WORKSPACE index 450bc6a65567..4a3526a9036a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,4 +1,9 @@ +workspace(name = "com_github_ray_project_ray") + load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository") +load("//java:repo.bzl", "java_repositories") + +java_repositories() git_repository( name = "com_github_nelhage_rules_boost", @@ -37,7 +42,7 @@ git_repository( new_git_repository( name = "plasma", build_file = "@//bazel:BUILD.plasma", - commit = "6a27c660ea700febf6fd73b2e851ab96e9315134", + commit = "fa8967cb9ec228f6544b2eecc0f1bd78f29daa4d", remote = "https://github.com/ray-project/arrow", ) diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index 33353caaf6f7..f221c8af17fd 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -1,5 +1,26 @@ load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library") +java_library( + name = "org_apache_arrow_arrow_plasma", + srcs = glob(["java/plasma/src/main/java/**/*.java"]), + data = [":plasma_client_java"], + visibility = ["//visibility:public"], + deps = [ + "@org_slf4j_slf4j_api//jar", + ], +) + +java_binary( + name = "org_apache_arrow_arrow_plasma_test", + srcs = ["java/plasma/src/test/java/org/apache/arrow/plasma/PlasmaClientTest.java"], + main_class = "org.apache.arrow.plasma.PlasmaClientTest", + visibility = ["//visibility:public"], + deps = [ + ":org_apache_arrow_arrow_plasma", + "@junit_junit//jar", + ], +) + cc_library( name = "arrow", srcs = [ @@ -8,6 +29,7 @@ cc_library( "cpp/src/arrow/status.cc", "cpp/src/arrow/util/io-util.cc", "cpp/src/arrow/util/logging.cc", + "cpp/src/arrow/util/memory.cc", "cpp/src/arrow/util/thread-pool.cc", ], hdrs = [ @@ -19,6 +41,7 @@ cc_library( "cpp/src/arrow/util/io-util.h", "cpp/src/arrow/util/logging.h", "cpp/src/arrow/util/macros.h", + "cpp/src/arrow/util/memory.h", "cpp/src/arrow/util/string_builder.h", "cpp/src/arrow/util/string_view.h", "cpp/src/arrow/util/thread-pool.h", @@ -41,6 +64,7 @@ cc_library( "cpp/src/plasma/io.cc", "cpp/src/plasma/malloc.cc", "cpp/src/plasma/plasma.cc", + "cpp/src/plasma/plasma_allocator.cc", "cpp/src/plasma/protocol.cc", ], hdrs = [ @@ -52,6 +76,7 @@ cc_library( "cpp/src/plasma/io.h", "cpp/src/plasma/malloc.h", "cpp/src/plasma/plasma.h", + "cpp/src/plasma/plasma_allocator.h", "cpp/src/plasma/plasma_generated.h", "cpp/src/plasma/protocol.h", ], @@ -65,7 +90,7 @@ cc_library( ) cc_binary( - name = "plasma_client_java.so", + name = "libplasma_java.so", srcs = [ "cpp/src/plasma/lib/java/org_apache_arrow_plasma_PlasmaClientJNI.cc", "cpp/src/plasma/lib/java/org_apache_arrow_plasma_PlasmaClientJNI.h", @@ -101,8 +126,8 @@ genrule( genrule( name = "plasma-jni-darwin-compat", - srcs = [":plasma_client_java.so"], - outs = ["plasma_client_java.dylib"], + srcs = [":libplasma_java.so"], + outs = ["libplasma_java.dylib"], cmd = "cp $< $@", output_to_bindir = 1, ) @@ -110,8 +135,8 @@ genrule( filegroup( name = "plasma_client_java", srcs = select({ - "@bazel_tools//src/conditions:darwin": [":plasma_client_java.dylib"], - "//conditions:default": [":plasma_client_java.so"], + "@bazel_tools//src/conditions:darwin": [":libplasma_java.dylib"], + "//conditions:default": [":libplasma_java.so"], }), visibility = ["//visibility:public"], ) @@ -122,10 +147,12 @@ cc_library( "cpp/src/plasma/dlmalloc.cc", "cpp/src/plasma/events.cc", "cpp/src/plasma/eviction_policy.cc", + "cpp/src/plasma/external_store.cc", ], hdrs = [ "cpp/src/plasma/events.h", "cpp/src/plasma/eviction_policy.h", + "cpp/src/plasma/external_store.h", "cpp/src/plasma/store.h", "cpp/src/plasma/thirdparty/ae/ae.h", "cpp/src/plasma/thirdparty/ae/ae_epoll.c", diff --git a/bazel/ray.bzl b/bazel/ray.bzl index 4a1e4f1063e4..85ae845ef1c0 100644 --- a/bazel/ray.bzl +++ b/bazel/ray.bzl @@ -10,3 +10,14 @@ def flatbuffer_py_library(name, srcs, outs, out_prefix, includes = [], include_p include_paths = include_paths, includes = includes, ) + +def flatbuffer_java_library(name, srcs, outs, out_prefix, includes = [], include_paths = []): + flatbuffer_library_public( + name = name, + srcs = srcs, + outs = outs, + language_flag = "-j", + out_prefix = out_prefix, + include_paths = include_paths, + includes = includes, + ) diff --git a/build.sh b/build.sh index 283f16757e49..c2d8282d1d8e 100755 --- a/build.sh +++ b/build.sh @@ -121,14 +121,33 @@ else $PYTHON_EXECUTABLE -m pip install \ --target=$ROOT_DIR/python/ray/pyarrow_files pyarrow==0.12.0.RAY \ --find-links https://s3-us-west-2.amazonaws.com/arrow-wheels/9357dc130789ee42f8181d8724bee1d5d1509060/index.html - bazel build //:ray_pkg -c opt --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHON_EXECUTABLE - # Copy files and keep them writeable. This is a workaround, as Bazel - # marks all generated files non-writeable. If we would just copy them - # over without adding write permission, the copy would fail the next time. - # TODO(pcm): It would be great to have a solution here that does not - # require us to copy the files. - find $ROOT_DIR/bazel-genfiles/ray_pkg/ -exec chmod +w {} \; - cp -r $ROOT_DIR/bazel-genfiles/ray_pkg/ray $ROOT_DIR/python || true + + if [ "$RAY_BUILD_JAVA" == "YES" ]; then + bazel build //:ray_java_pkg -c opt --verbose_failures + # The following are soft links. + # TODO: remove this once cmake is removed + mkdir -p $ROOT_DIR/build/src/ray/raylet/ + mkdir -p $ROOT_DIR/build/src/ray/gcs/redis_module/ + mkdir -p $ROOT_DIR/build/src/ray/thirdparty/redis/src/ + mkdir -p $ROOT_DIR/build/src/plasma/ + ln -sf $ROOT_DIR/bazel-bin/* $ROOT_DIR/build/src/ray/raylet/ + ln -sf $ROOT_DIR/bazel-bin/external/plasma/* $ROOT_DIR/build/src/plasma/ + ln -sf $ROOT_DIR/bazel-genfiles/redis-server $ROOT_DIR/build/src/ray/thirdparty/redis/src/redis-server + ln -sf $ROOT_DIR/bazel-bin/ray_redis_module.so $ROOT_DIR/build/src/ray/gcs/redis_module/libray_redis_module.so + ln -sf $ROOT_DIR/bazel-bin/external/plasma/plasma_store_server $ROOT_DIR/build/src/plasma/plasma_store_server + ln -sf $ROOT_DIR/bazel-bin/raylet $ROOT_DIR/build/src/ray/raylet/raylet + fi + + if [ "$RAY_BUILD_PYTHON" == "YES" ]; then + bazel build //:ray_pkg -c opt --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHON_EXECUTABLE + # Copy files and keep them writeable. This is a workaround, as Bazel + # marks all generated files non-writeable. If we would just copy them + # over without adding write permission, the copy would fail the next time. + # TODO(pcm): It would be great to have a solution here that does not + # require us to copy the files. + find $ROOT_DIR/bazel-genfiles/ray_pkg/ -exec chmod +w {} \; + cp -r $ROOT_DIR/bazel-genfiles/ray_pkg/ray $ROOT_DIR/python || true + fi fi popd diff --git a/java/BUILD.bazel b/java/BUILD.bazel new file mode 100644 index 000000000000..26307706cd42 --- /dev/null +++ b/java/BUILD.bazel @@ -0,0 +1,212 @@ +load("//java/tools/checkstyle:checkstyle.bzl", "checkstyle_test") +load("//bazel:ray.bzl", "flatbuffer_java_library") + +java_binary( + name = "bazel_deps", + main_class = "com.github.johnynek.bazel_deps.ParseProject", + runtime_deps = ["@bazel_deps//jar"], +) + +java_import( + name = "org_ray_ray_java_root", + jars = [ + "liborg_ray_ray_api.jar", + "liborg_ray_ray_api-src.jar", + "liborg_ray_ray_runtime.jar", + "liborg_ray_ray_runtime-src.jar", + "liborg_ray_ray_runtime_test.jar", + "liborg_ray_ray_runtime_test-src.jar", + "liborg_ray_ray_test.jar", + "liborg_ray_ray_test-src.jar", + "org_ray_ray_tutorial_deploy.jar", + "org_ray_ray_tutorial_deploy-src.jar", + "all_tests_deploy.jar", + "all_tests_deploy-src.jar", + ], + deps = [ + ":org_ray_ray_api", + ":org_ray_ray_runtime", + ":org_ray_ray_tutorial", + ":org_ray_ray_test", + ":org_ray_ray_runtime_test", + ], +) + +java_library( + name = "org_ray_ray_api", + srcs = glob(["api/src/main/java/**/*.java"]), + deps = [ + "@org_slf4j_slf4j_log4j12//jar", + "@log4j_log4j//jar", + "@javax_xml_bind_jaxb_api//jar", + "@com_sun_xml_bind_jaxb_core//jar", + "@com_sun_xml_bind_jaxb_impl//jar", + ], +) + +checkstyle_test( + name = "org_ray_ray_api-checkstyle", + target = ":org_ray_ray_api", + size = "small", +) + +java_library( + name = "org_ray_ray_runtime", + srcs = glob([ + "runtime/src/main/java/**/*.java" + ]) + [ + ":generate_java_gcs_fbs", + ], + resources = glob(["runtime/src/main/resources/**"]), + data = ["//:raylet_library_java"], + deps = [ + ":org_ray_ray_api", + "@plasma//:org_apache_arrow_arrow_plasma", + "@org_slf4j_slf4j_api//jar", + "@com_fasterxml_jackson_core_jackson_core//jar", + "@org_apache_commons_commons_pool2//jar", + "@org_javassist_javassist//jar", + "@org_objenesis_objenesis//jar", + "@com_typesafe_config//jar", + "@org_apache_commons_commons_lang3//jar", + "@de_ruedigermoeller_fst//jar", + "@com_github_davidmoten_flatbuffers_java//jar", + "@redis_clients_jedis//jar", + "@commons_io_commons_io//jar", + "@com_google_guava_guava//jar", + "@net_lingala_zip4j_zip4j//jar", + "@org_slf4j_slf4j_log4j12//jar", + "@org_ini4j_ini4j//jar", + "@org_ow2_asm_asm//jar", + ], +) + +checkstyle_test( + name = "org_ray_ray_runtime-checkstyle", + target = ":org_ray_ray_runtime", + size = "small", +) + +java_binary( + name = "org_ray_ray_tutorial", + srcs = glob(["tutorial/src/main/java/**/*.java"]), + resources = glob(["tutorial/src/main/resources/**"]), + deps = [ + ":org_ray_ray_api", + ":org_ray_ray_runtime", + "@com_google_guava_guava//jar", + ], +) + +checkstyle_test( + name = "org_ray_ray_tutorial-checkstyle", + target = ":org_ray_ray_tutorial", + size = "small", +) + +java_library( + name = "org_ray_ray_test", + srcs = glob(["test/src/main/java/**/*.java"]), + deps = [ + ":org_ray_ray_api", + ":org_ray_ray_runtime", + "@org_apache_commons_commons_lang3//jar", + "@org_slf4j_slf4j_api//jar", + "@plasma//:org_apache_arrow_arrow_plasma", + "@org_testng_testng//jar", + "@commons_collections_commons_collections//jar", + "@com_google_guava_guava//jar", + ], +) + +checkstyle_test( + name = "org_ray_ray_test-checkstyle", + target = ":org_ray_ray_test", + size = "small", +) + +java_library( + name = "org_ray_ray_runtime_test", + srcs = glob(["runtime/src/test/java/**/*.java"]), + deps = [ + ":org_ray_ray_api", + ":org_ray_ray_runtime", + "@org_apache_commons_commons_lang3//jar", + "@com_beust_jcommander//jar", + "@org_testng_testng//jar", + ], +) + +checkstyle_test( + name = "org_ray_ray_runtime_test-checkstyle", + target = ":org_ray_ray_runtime_test", + size = "small", +) + +exports_files(["testng.xml"]) + +java_binary( + name = "all_tests", + main_class = "org.testng.TestNG", + data = ["testng.xml"], + runtime_deps = [ + ":org_ray_ray_test", + ":org_ray_ray_runtime_test", + ], +) + +flatbuffers_generate_files = [ + "ActorCheckpointData.java", + "ActorCheckpointIdData.java", + "ActorState.java", + "ActorTableData.java", + "Arg.java", + "ClassTableData.java", + "ClientTableData.java", + "ConfigTableData.java", + "CustomSerializerData.java", + "DriverTableData.java", + "ErrorTableData.java", + "ErrorType.java", + "FunctionTableData.java", + "GcsTableEntry.java", + "HeartbeatBatchTableData.java", + "HeartbeatTableData.java", + "Language.java", + "ObjectTableData.java", + "ProfileEvent.java", + "ProfileTableData.java", + "RayResource.java", + "ResourcePair.java", + "SchedulingState.java", + "TablePrefix.java", + "TablePubsub.java", + "TaskInfo.java", + "TaskLeaseData.java", + "TaskReconstructionData.java", + "TaskTableData.java", + "TaskTableTestAndUpdate.java", +] + +flatbuffer_java_library( + name = "java_gcs_fbs", + srcs = ["//:gcs_fbs_file"], + outs = flatbuffers_generate_files, + out_prefix = "", +) + +genrule( + name = "generate_java_gcs_fbs", + srcs = [":java_gcs_fbs"], + outs = [ + "runtime/src/main/java/org/ray/runtime/generated/" + file for file in flatbuffers_generate_files + ], + cmd = """ + for f in $(locations //java:java_gcs_fbs); do + chmod +w $$f + cp -f $$f $(@D)/runtime/src/main/java/org/ray/runtime/generated + done + python $$(pwd)/java/modify_generated_java_flatbuffers_files.py $(@D)/.. + """, + local = 1, +) diff --git a/java/build.sh b/java/build.sh deleted file mode 100755 index 60b361ae8eb9..000000000000 --- a/java/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -mvn clean install -Dmaven.test.skip diff --git a/java/dependencies.yaml b/java/dependencies.yaml new file mode 100644 index 000000000000..62ca6af2eb36 --- /dev/null +++ b/java/dependencies.yaml @@ -0,0 +1,116 @@ +options: + languages: [ "java" ] + licenses: [ "permissive" ] + namePrefix: "" + resolverType: "coursier" + resolvers: + - id: "mavencentral" + type: "default" + url: https://repo.maven.apache.org/maven2/ + thirdPartyDirectory: "java/third_party/" + transitivity: runtime_deps + versionConflictPolicy: highest + +dependencies: + com.puppycrawl.tools: + checkstyle: + lang: java + version: "8.15" + + de.ruedigermoeller: + fst: + lang: java + version: "2.47" + + org.ini4j: + ini4j: + lang: java + version: "0.5.2" + + org.ow2.asm: + asm: + lang: java + version: "6.0" + + com.github.davidmoten: + flatbuffers-java: + lang: java + version: "1.9.0.1" + + com.beust: + jcommander: + lang: java + version: "1.72" + + redis.clients: + jedis: + lang: java + version: "2.8.0" + + commons-io: + commons-io: + lang: java + version: "2.5" + + org.apache.commons: + commons-lang3: + lang: java + version: "3.4" + + commons-codec: + commons-codec: + lang: java + version: "1.4" + + net.lingala.zip4j: + zip4j: + lang: java + version: "1.3.2" + + com.google.guava: + guava: + lang: java + version: "19.0" + + commons-collections: + commons-collections: + lang: java + version: "3.2.2" + + org.slf4j: + slf4j-log4j12: + lang: java + version: "1.7.25" + + com.typesafe: + config: + lang: java + version: "1.3.2" + + net.java.dev.jna: + jna: + lang: java + version: "4.1.0" + + org.mockito: + mockito-all: + lang: java + version: "1.10.19" + + org.testng: + testng: + lang: java + version: "6.9.9" + + javax.xml.bind: + jaxb-api: + lang: java + version: "2.3.0" + + com.sun.xml.bind: + jaxb-core: + lang: java + version: "2.3.0" + jaxb-impl: + lang: java + version: "2.3.0" diff --git a/java/generate_deps.sh b/java/generate_deps.sh new file mode 100644 index 000000000000..39ecc9f488ae --- /dev/null +++ b/java/generate_deps.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e +set -x + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +cd $ROOT_DIR/.. +bazel run //java:bazel_deps -- generate -r `pwd` -s java/third_party/workspace.bzl -d java/dependencies.yaml + +set +x +set +e diff --git a/java/repo.bzl b/java/repo.bzl new file mode 100644 index 000000000000..a06c285f234b --- /dev/null +++ b/java/repo.bzl @@ -0,0 +1,18 @@ +load("//java/third_party:workspace.bzl", "maven_dependencies") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_jar") + +def create_if_needed(rule, name, **kwargs): + if name not in native.existing_rules(): + rule(name = name, **kwargs) + +def bazel_deps(): + create_if_needed( + http_jar, + name = "bazel_deps", + sha256 = "98b05c2826f2248f70e7356dc6c78bc52395904bb932fbb409a5abf5416e4292", + urls = ["https://github.com/oferb/startupos-binaries/releases/download/0.1.01/bazel_deps.jar"], + ) + +def java_repositories(): + maven_dependencies() + bazel_deps() diff --git a/java/test.sh b/java/test.sh index b3e889371bcd..6200d632264d 100755 --- a/java/test.sh +++ b/java/test.sh @@ -7,20 +7,38 @@ set -x ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -pushd $ROOT_DIR/../java +# run this file before compile the targets +sh $ROOT_DIR/generate_deps.sh + echo "Compiling Java code." -mvn clean install -Dmaven.test.skip +pushd $ROOT_DIR/.. +# bazel checkstyle for java +bazel test //java:all -c opt +# compile all the targets +bazel build //java:all -c opt --verbose_failures +popd -echo "Checking code format." -mvn checkstyle:check +# The following are soft links +# TODO: remove this once cmake is removed +mkdir -p $ROOT_DIR/../build/java/ +ln -sf $ROOT_DIR/../bazel-bin/java/* $ROOT_DIR/../build/java/ +mkdir -p $ROOT_DIR/tutorial/target/ +ln -sf $ROOT_DIR/../bazel-bin/java/org_ray_ray_tutorial_deploy.jar $ROOT_DIR/tutorial/target/ray-tutorial-0.1-SNAPSHOT.jar +pushd $ROOT_DIR/test echo "Running tests under cluster mode." -ENABLE_MULTI_LANGUAGE_TESTS=1 mvn test +ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml || cluster_exit_code=$? -echo "Running tests under single-process mode." -mvn test -Dray.run-mode=SINGLE_PROCESS +# exit_code == 2 means there are some tests skiped. +if [ $cluster_exit_code -eq 2 ] && [ $cluster_exit_code -eq 0 ] ; then + exit $exit_code +fi -set +x -set +e +echo "Running tests under single-process mode." +java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml || single_exit_code=$? +# exit_code == 2 means there are some tests skiped. +if [ $single_exit_code -eq 2 ] && [ $single_exit_code -eq 0 ] ; then + exit $exit_code +fi popd diff --git a/java/test/src/main/java/org/ray/api/test/ClientExceptionTest.java b/java/test/src/main/java/org/ray/api/test/ClientExceptionTest.java index e9f53dddd794..b588822712c5 100644 --- a/java/test/src/main/java/org/ray/api/test/ClientExceptionTest.java +++ b/java/test/src/main/java/org/ray/api/test/ClientExceptionTest.java @@ -37,12 +37,12 @@ public void testWaitAndCrash() { Ray.wait(ImmutableList.of(notExisting), 1, 2000); Assert.fail("Should not reach here"); } catch (RayException e) { - LOGGER.debug(String.format("Expected runtime exception: {}", e)); + LOGGER.debug("Expected runtime exception: {}", e); } try { thread.join(); } catch (Exception e) { - LOGGER.error(String.format("Excpetion caught: {}", e)); + LOGGER.error("Excpetion caught: {}", e); } } } diff --git a/java/testng.xml b/java/testng.xml new file mode 100644 index 000000000000..81503edf4b85 --- /dev/null +++ b/java/testng.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/java/third_party/BUILD b/java/third_party/BUILD new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/java/third_party/workspace.bzl b/java/third_party/workspace.bzl new file mode 100644 index 000000000000..4aa7c40b298f --- /dev/null +++ b/java/third_party/workspace.bzl @@ -0,0 +1,73 @@ +# Do not edit. bazel-deps autogenerates this file from java/dependencies.yaml. +def _jar_artifact_impl(ctx): + jar_name = "%s.jar" % ctx.name + ctx.download( + output=ctx.path("jar/%s" % jar_name), + url=ctx.attr.urls, + sha256=ctx.attr.sha256, + executable=False + ) + src_name="%s-sources.jar" % ctx.name + srcjar_attr="" + has_sources = len(ctx.attr.src_urls) != 0 + if has_sources: + ctx.download( + output=ctx.path("jar/%s" % src_name), + url=ctx.attr.src_urls, + sha256=ctx.attr.src_sha256, + executable=False + ) + srcjar_attr ='\n srcjar = ":%s",' % src_name + + build_file_contents = """ +package(default_visibility = ['//visibility:public']) +java_import( + name = 'jar', + jars = ['{jar_name}'],{srcjar_attr} +) +filegroup( + name = 'file', + srcs = [ + '{jar_name}', + '{src_name}' + ], + visibility = ['//visibility:public'] +)\n""".format(jar_name = jar_name, src_name = src_name, srcjar_attr = srcjar_attr) + ctx.file(ctx.path("jar/BUILD"), build_file_contents, False) + return None + +jar_artifact = repository_rule( + attrs = { + "artifact": attr.string(mandatory = True), + "sha256": attr.string(mandatory = True), + "urls": attr.string_list(mandatory = True), + "src_sha256": attr.string(mandatory = False, default=""), + "src_urls": attr.string_list(mandatory = False, default=[]), + }, + implementation = _jar_artifact_impl +) + +def jar_artifact_callback(hash): + src_urls = [] + src_sha256 = "" + source=hash.get("source", None) + if source != None: + src_urls = [source["url"]] + src_sha256 = source["sha256"] + jar_artifact( + artifact = hash["artifact"], + name = hash["name"], + urls = [hash["url"]], + sha256 = hash["sha256"], + src_urls = src_urls, + src_sha256 = src_sha256 + ) + native.bind(name = hash["bind"], actual = hash["actual"]) + +def list_dependencies(): + return [ + ] + +def maven_dependencies(callback = jar_artifact_callback): + for hash in list_dependencies(): + callback(hash) diff --git a/java/tools/checkstyle/BUILD b/java/tools/checkstyle/BUILD new file mode 100644 index 000000000000..b323bb6066f8 --- /dev/null +++ b/java/tools/checkstyle/BUILD @@ -0,0 +1,6 @@ +exports_files([ + "checkstyle.sh", + "license-header.txt", + "checkstyle.xml", + "checkstyle-suppressions.xml", +]) diff --git a/java/tools/checkstyle/README.md b/java/tools/checkstyle/README.md new file mode 100644 index 000000000000..0a83260d8081 --- /dev/null +++ b/java/tools/checkstyle/README.md @@ -0,0 +1,70 @@ +# Checkstyle integration + +## What is Checkstyle + +>[checkstyle](http://checkstyle.sourceforge.net/) is a development tool to +help programmers write Java code that adheres to a coding standard. +It automates the process of checking Java code to spare humans of this +boring (but important) task. This makes it ideal for projects that want to +enforce a coding standard. + +## Integrating with `bazel` + +To apply `checkstyle` check to Java code built with `bazel`, +we use the `checkstyle_test` rule. + +Example: + +`WORKSPACE`: +``` +load("//tools/checkstyle:checkstyle.bzl", "checkstyle_dependencies") +checkstyle_dependencies() +``` + +`BUILD`: + +``` +load("//tools/checkstyle:checkstyle.bzl", "checkstyle_test") + +java_library( + name = "lib", + srcs = ["Lib.java"], +) + +java_library( + name = "ignored_lib", + srcs = ["Lib.java"], + tags = ["checkstyle_ignore"] +) + +checkstyle_test( + name = "lib-checkstyle", + allow_failure = 1, + target = ":lib", +) + +``` + +Sample code illustrates several points: +* To apply Checkstyle to Java target named `lib`, create a `checkstyle_test` +named `lib-checkstyle` (with a dash). Such naming convention is enforced at rule level. +* `checkstyle_test` **is** a test in Bazel context, which means you can do +`bazel test //:lib-checkstyle` and get result of that inspection. +* To perform initial integration, `allow_failure` attribute can be temporarily set, +which will _still_ perform Checkstyle test, but will succeed (as Bazel test) +independently of check's result. +* Targets that do not need to be checked (e.g. generated code), such as `ignored_lib` +can be ignored by assigning `tags = ["checkstyle_ignore"]` +* Checkstyle test **completeness** (that is, all Java code that _needs_ checking +**is** being checked) is enforced via the shell script named `check-for-missing-targets.sh` +ran by CI. If you omit **both** `checkstyle_test` target and `checkstyle_ignore` tag, the script +will exit with non-zero code and print out non-covered targets. + +## Configuring Checkstyle + +Checkstyle is configured by an XML file. By default, current rule implementation uses +`//tools/checkstyle:config.xml` which is based on Google Java style +with added checks of license header +(via [`RegexpHeader`](http://checkstyle.sourceforge.net/config_header.html#RegexpHeader) +and `//tools/checkstyle:license-header.txt`) and package header +(via [`PackageName`](http://checkstyle.sourceforge.net/config_naming.html#PackageName)) diff --git a/java/tools/checkstyle/check-for-missing-targets.sh b/java/tools/checkstyle/check-for-missing-targets.sh new file mode 100755 index 000000000000..a28b6d2a1db4 --- /dev/null +++ b/java/tools/checkstyle/check-for-missing-targets.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# This script verifies that all Java targets except those having assigned `checkstyle_ignore` tag +# do have a accompanying Checkstyle target + +# [1] Get all Java targets except those having assigned `checkstyle_ignore` tag +bazel query 'kind(java_*, //...) - kind(java_proto_library, //...) - //third_party/... - attr("tags", "checkstyle_ignore", //...)' | sort >/tmp/all_java_targets +# [2] Get all checkstyle targets' names, replace '-checkstyle' with nothing so it matches Java target name +bazel query 'kind(checkstyle_test, //...)' | sed -e 's|-checkstyle||g' | sort >/tmp/checkstyle_covered_java_targets + +# [3] Targets in [1] which are not in [2] +NON_COVERED_JAVA_TARGETS=$(comm -23 /tmp/all_java_targets /tmp/checkstyle_covered_java_targets) + +# Having non-empty [3] is an error +if [[ ! -z "$NON_COVERED_JAVA_TARGETS" ]]; then + echo "$(tput setaf 1)[!] These java targets do not have accompanying checkstyle targets:"$(tput sgr0) + echo ${NON_COVERED_JAVA_TARGETS} + exit 1 +fi diff --git a/java/tools/checkstyle/checkstyle.bzl b/java/tools/checkstyle/checkstyle.bzl new file mode 100644 index 000000000000..51129aeb830a --- /dev/null +++ b/java/tools/checkstyle/checkstyle.bzl @@ -0,0 +1,127 @@ +#Structure representing info about Java source files +JavaSourceFiles = provider( + fields = { + 'files' : 'java source files' + } +) + +# This aspect is responsible for collecting Java sources for target +# When applied to target, `JavaSourceFiles` struct will be attached to +# target info +def collect_sources_impl(target, ctx): + files = [] + if hasattr(ctx.rule.attr, 'srcs'): + for src in ctx.rule.attr.srcs: + for file in src.files: + if file.extension == 'java': + files.append(file) + return [JavaSourceFiles(files = files)] + + +collect_sources = aspect( + implementation = collect_sources_impl, +) + + +# `ctx` is rule context: https://docs.bazel.build/versions/master/skylark/lib/ctx.html +def _checkstyle_test_impl(ctx): + # verify target name matches naming conventions + if "{}-checkstyle".format(ctx.attr.target.label.name) != ctx.attr.name: + fail("target should follow `{java_library target name}-checkstyle` pattern") + + suppressions = ctx.file.suppressions + opts = ctx.attr.opts + sopts = ctx.attr.string_opts + + # Checkstyle and its dependencies + checkstyle_dependencies = ctx.attr._checkstyle.java.transitive_runtime_deps + classpath = ":".join([file.path for file in checkstyle_dependencies]) + + args = "" + inputs = [] + if ctx.file.config: + args += " -c %s" % ctx.file.config.path + inputs.append(ctx.file.config) + if suppressions: + inputs.append(suppressions) + + # Build command to run Checkstyle test + cmd = " ".join( + ["java -cp %s com.puppycrawl.tools.checkstyle.Main" % classpath] + + [args] + + ["--%s" % x for x in opts] + + ["--%s %s" % (k, sopts[k]) for k in sopts] + + [file.path for file in ctx.attr.target[JavaSourceFiles].files] + ) + + # Wrap checkstyle command in a shell script so allow_failure is supported + ctx.actions.expand_template( + template = ctx.file._checkstyle_sh_template, + output = ctx.outputs.checkstyle_script, + substitutions = { + "{command}" : cmd, + "{allow_failure}": str(int(ctx.attr.allow_failure)), + }, + is_executable = True, + ) + + files = [ctx.outputs.checkstyle_script, ctx.file.license] + ctx.attr.target[JavaSourceFiles].files + checkstyle_dependencies.to_list() + inputs + runfiles = ctx.runfiles( + files = files, + collect_data = True + ) + return DefaultInfo( + executable = ctx.outputs.checkstyle_script, + files = depset(files), + runfiles = runfiles, + ) + +checkstyle_test = rule( + implementation = _checkstyle_test_impl, + test = True, + attrs = { + "config": attr.label( + allow_single_file=True, + doc = "A checkstyle configuration file", + default = "//java:checkstyle.xml", + ), + "suppressions": attr.label( + allow_single_file=True, + doc = ("A file for specifying files and lines " + + "that should be suppressed from checks." + + "Example: https://github.com/checkstyle/checkstyle/blob/master/config/suppressions.xml"), + default = "//java:checkstyle-suppressions.xml", + ), + "license": attr.label( + allow_single_file=True, + doc = "A license file that can be used with the checkstyle license target", + default = "//java/tools/checkstyle:license-header.txt", + ), + "opts": attr.string_list( + doc = "Options to be passed on the command line that have no argument" + ), + "string_opts": attr.string_dict( + doc = "Options to be passed on the command line that have an argument" + ), + "target": attr.label( + doc = "The java_library target to check sources on", + aspects = [collect_sources], + mandatory = True + ), + "allow_failure": attr.bool( + default = False, + doc = "Successfully finish the test even if checkstyle failed" + ), + "_checkstyle_sh_template": attr.label( + allow_single_file = True, + default = "//java/tools/checkstyle:checkstyle.sh" + ), + "_checkstyle": attr.label( + default = "//java/third_party/com/puppycrawl/tools:checkstyle" + ), + }, + outputs = { + "checkstyle_script": "%{name}.sh", + }, +) + diff --git a/java/tools/checkstyle/checkstyle.sh b/java/tools/checkstyle/checkstyle.sh new file mode 100755 index 000000000000..622a9cda9db9 --- /dev/null +++ b/java/tools/checkstyle/checkstyle.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +COMMAND="{command}" +ALLOW_FAILURE="{allow_failure}" + +echo "ALLOW_FAILURE IS $ALLOW_FAILURE" + +source ~/.bash_profile + +${COMMAND} +result=$? + +if [[ ${ALLOW_FAILURE} -eq 1 ]]; then + exit 0 +fi + +exit ${result} diff --git a/java/tools/checkstyle/license-header.txt b/java/tools/checkstyle/license-header.txt new file mode 100644 index 000000000000..2d4ccdb2ee66 --- /dev/null +++ b/java/tools/checkstyle/license-header.txt @@ -0,0 +1,15 @@ +^\W* +^\W*Copyright 20[0-9]{2} The StartupOS Authors. +^\W* +^\W*Licensed under the Apache License, Version 2.0 \(the "License"\); +^\W*you may not use this file except in compliance with the License. +^\W*You may obtain a copy of the License at +^\W* +^\W*https://www.apache.org/licenses/LICENSE-2.0 +^\W* +^\W*Unless required by applicable law or agreed to in writing, software +^\W*distributed under the License is distributed on an "AS IS" BASIS, +^\W*WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +^\W*See the License for the specific language governing permissions and +^\W*limitations under the License. +^\W* \ No newline at end of file From d64452281a431860ada503397fdef5cf55e7456c Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Fri, 15 Mar 2019 18:18:50 +0800 Subject: [PATCH 070/115] delete local checkstyle and add .bazelrc file and modify java_test --- .bazelrc | 2 + .gitignore | 2 +- .travis.yml | 2 +- BUILD.bazel | 13 +- CMakeLists.txt | 4 +- WORKSPACE | 6 + build.sh | 18 +-- java/BUILD.bazel | 28 +++- java/dependencies.yaml | 2 +- java/runtime/pom.xml | 3 - .../org/ray/runtime/RayNativeRuntime.java | 2 +- .../org/ray/runtime/runner/RunManager.java | 10 +- java/test.sh | 13 +- java/tools/checkstyle/BUILD | 6 - java/tools/checkstyle/README.md | 70 ---------- .../checkstyle/check-for-missing-targets.sh | 19 --- java/tools/checkstyle/checkstyle.bzl | 127 ------------------ java/tools/checkstyle/checkstyle.sh | 17 --- java/tools/checkstyle/license-header.txt | 15 --- src/ray/test/run_gcs_tests.sh | 2 +- src/ray/test/run_object_manager_tests.sh | 8 +- src/ray/test/run_object_manager_valgrind.sh | 8 +- 22 files changed, 73 insertions(+), 304 deletions(-) create mode 100644 .bazelrc delete mode 100644 java/tools/checkstyle/BUILD delete mode 100644 java/tools/checkstyle/README.md delete mode 100755 java/tools/checkstyle/check-for-missing-targets.sh delete mode 100644 java/tools/checkstyle/checkstyle.bzl delete mode 100755 java/tools/checkstyle/checkstyle.sh delete mode 100644 java/tools/checkstyle/license-header.txt diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 000000000000..55bcd0242bd9 --- /dev/null +++ b/.bazelrc @@ -0,0 +1,2 @@ +# build config +build --compilation_mode=opt diff --git a/.gitignore b/.gitignore index a5c9754c27d6..290744290f05 100644 --- a/.gitignore +++ b/.gitignore @@ -149,7 +149,7 @@ java/**/.settings java/**/.classpath java/**/.project java/third_party/workspace.bzl -java/runtime/native_dependencies/ +java/runtime/src/main/resources/native/ # python virtual env venv diff --git a/.travis.yml b/.travis.yml index dee43a97523c..f778f64a11ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -142,7 +142,7 @@ install: - ./ci/suppress_output bash src/ray/test/run_gcs_tests.sh # Raylet tests. - ./ci/suppress_output bash src/ray/test/run_object_manager_tests.sh - - ./ci/suppress_output bazel test --build_tests_only --test_lang_filters=cc ... -c opt + - ./ci/suppress_output bazel test --build_tests_only --test_lang_filters=cc //:all script: diff --git a/BUILD.bazel b/BUILD.bazel index b0331298cb27..f3a57a7ab8f4 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -99,8 +99,8 @@ cc_test( args = ["--gtest_filter=PrintLogTest*"], copts = COPTS, deps = [ - "@boost//:asio", ":ray_util", + "@boost//:asio", "@com_google_googletest//:gtest_main", ], ) @@ -596,8 +596,15 @@ genrule( "@plasma//:plasma_store_server", "@plasma//:plasma_client_java", ], - outs = ["ray_java_pkg.out"], + outs = ["ray_java_pkg/java/runtime/src/main/resources/native"], cmd = """ - echo $$(pwd) > $@ + set -x && + mkdir -p $(@D)/native && + cp $(location //:redis-server) $(@D)/native/ && + cp $(location //:ray_redis_module.so) $(@D)/native/ && + cp $(location //:raylet) $(@D)/native && + cp $(location //:raylet_library_java) $(@D)/native && + cp $(location @plasma//:plasma_store_server) $(@D)/native && + cp $(location @plasma//:plasma_client_java) $(@D)/native """, ) diff --git a/CMakeLists.txt b/CMakeLists.txt index 84236d4e9ce5..b6d437c0cf27 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -154,7 +154,7 @@ if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES") # Copy java native dependencies. add_custom_command(TARGET copy_ray POST_BUILD - COMMAND mkdir -p ${CMAKE_SOURCE_DIR}/java/runtime/native_dependencies) + COMMAND mkdir -p ${CMAKE_SOURCE_DIR}/java/runtime/src/main/resources/native) set(java_native_dependencies "src/ray/thirdparty/redis/src/redis-server" "src/ray/gcs/redis_module/libray_redis_module.so" @@ -163,7 +163,7 @@ if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES") foreach(file ${java_native_dependencies}) add_custom_command(TARGET copy_ray POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/${file} - ${CMAKE_SOURCE_DIR}/java/runtime/native_dependencies) + ${CMAKE_SOURCE_DIR}/java/runtime/src/main/resources/native) endforeach() add_custom_command(TARGET copy_ray POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy ${ARROW_HOME}/bin/plasma_store_server diff --git a/WORKSPACE b/WORKSPACE index 4a3526a9036a..0bd857b90d36 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -5,6 +5,12 @@ load("//java:repo.bzl", "java_repositories") java_repositories() +git_repository( + name = "com_github_checkstyle_java", + remote = "https://github.com/alegithub111/checkstyle_java.git", + commit = "85f37871ca03b9d3fee63c69c8107f167e24e77b", +) + git_repository( name = "com_github_nelhage_rules_boost", commit = "6d6fd834281cb8f8e758dd9ad76df86304bf1869", diff --git a/build.sh b/build.sh index c2d8282d1d8e..3ecae0a25000 100755 --- a/build.sh +++ b/build.sh @@ -123,23 +123,13 @@ else --find-links https://s3-us-west-2.amazonaws.com/arrow-wheels/9357dc130789ee42f8181d8724bee1d5d1509060/index.html if [ "$RAY_BUILD_JAVA" == "YES" ]; then - bazel build //:ray_java_pkg -c opt --verbose_failures - # The following are soft links. - # TODO: remove this once cmake is removed - mkdir -p $ROOT_DIR/build/src/ray/raylet/ - mkdir -p $ROOT_DIR/build/src/ray/gcs/redis_module/ - mkdir -p $ROOT_DIR/build/src/ray/thirdparty/redis/src/ - mkdir -p $ROOT_DIR/build/src/plasma/ - ln -sf $ROOT_DIR/bazel-bin/* $ROOT_DIR/build/src/ray/raylet/ - ln -sf $ROOT_DIR/bazel-bin/external/plasma/* $ROOT_DIR/build/src/plasma/ - ln -sf $ROOT_DIR/bazel-genfiles/redis-server $ROOT_DIR/build/src/ray/thirdparty/redis/src/redis-server - ln -sf $ROOT_DIR/bazel-bin/ray_redis_module.so $ROOT_DIR/build/src/ray/gcs/redis_module/libray_redis_module.so - ln -sf $ROOT_DIR/bazel-bin/external/plasma/plasma_store_server $ROOT_DIR/build/src/plasma/plasma_store_server - ln -sf $ROOT_DIR/bazel-bin/raylet $ROOT_DIR/build/src/ray/raylet/raylet + bazel build //:ray_java_pkg --verbose_failures + find $ROOT_DIR/bazel-genfiles/ray_java_pkg/ -exec chmod +w {} \; + cp -r $ROOT_DIR/bazel-genfiles/ray_java_pkg/java $ROOT_DIR/ || true fi if [ "$RAY_BUILD_PYTHON" == "YES" ]; then - bazel build //:ray_pkg -c opt --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHON_EXECUTABLE + bazel build //:ray_pkg --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHON_EXECUTABLE # Copy files and keep them writeable. This is a workaround, as Bazel # marks all generated files non-writeable. If we would just copy them # over without adding write permission, the copy would fail the next time. diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 26307706cd42..dd2246899eea 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -1,6 +1,12 @@ -load("//java/tools/checkstyle:checkstyle.bzl", "checkstyle_test") +load("@com_github_checkstyle_java//checkstyle:checkstyle.bzl", "checkstyle_test") load("//bazel:ray.bzl", "flatbuffer_java_library") +exports_files([ + "testng.xml", + "checkstyle.xml", + "checkstyle-suppressions.xml", +]) + java_binary( name = "bazel_deps", main_class = "com.github.johnynek.bazel_deps.ParseProject", @@ -47,7 +53,10 @@ java_library( checkstyle_test( name = "org_ray_ray_api-checkstyle", target = ":org_ray_ray_api", + config = "//java:checkstyle.xml", + suppressions = "//java:checkstyle-suppressions.xml", size = "small", + tags = ["checkstyle"], ) java_library( @@ -84,7 +93,10 @@ java_library( checkstyle_test( name = "org_ray_ray_runtime-checkstyle", target = ":org_ray_ray_runtime", + config = "//java:checkstyle.xml", + suppressions = "//java:checkstyle-suppressions.xml", size = "small", + tags = ["checkstyle"], ) java_binary( @@ -101,7 +113,10 @@ java_binary( checkstyle_test( name = "org_ray_ray_tutorial-checkstyle", target = ":org_ray_ray_tutorial", + config = "//java:checkstyle.xml", + suppressions = "//java:checkstyle-suppressions.xml", size = "small", + tags = ["checkstyle"], ) java_library( @@ -122,7 +137,10 @@ java_library( checkstyle_test( name = "org_ray_ray_test-checkstyle", target = ":org_ray_ray_test", + config = "//java:checkstyle.xml", + suppressions = "//java:checkstyle-suppressions.xml", size = "small", + tags = ["checkstyle"], ) java_library( @@ -140,15 +158,17 @@ java_library( checkstyle_test( name = "org_ray_ray_runtime_test-checkstyle", target = ":org_ray_ray_runtime_test", + config = "//java:checkstyle.xml", + suppressions = "//java:checkstyle-suppressions.xml", size = "small", + tags = ["checkstyle"], ) -exports_files(["testng.xml"]) - -java_binary( +java_test( name = "all_tests", main_class = "org.testng.TestNG", data = ["testng.xml"], + args = ["java/testng.xml"], runtime_deps = [ ":org_ray_ray_test", ":org_ray_ray_runtime_test", diff --git a/java/dependencies.yaml b/java/dependencies.yaml index 62ca6af2eb36..0d4cdd68fae8 100644 --- a/java/dependencies.yaml +++ b/java/dependencies.yaml @@ -70,7 +70,7 @@ dependencies: com.google.guava: guava: lang: java - version: "19.0" + version: "27.0.1-jre" commons-collections: commons-collections: diff --git a/java/runtime/pom.xml b/java/runtime/pom.xml index 4305b074313f..a87f14cc471a 100644 --- a/java/runtime/pom.xml +++ b/java/runtime/pom.xml @@ -83,9 +83,6 @@ src/main/resources - - native_dependencies - diff --git a/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java b/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java index a0fbdf01f28d..ff5a34f2026b 100644 --- a/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java +++ b/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java @@ -88,7 +88,7 @@ public void start() throws Exception { // Copy the file from resources to a temp dir, and load the native library. File file = File.createTempFile(fileName, ""); file.deleteOnExit(); - InputStream in = RayNativeRuntime.class.getResourceAsStream("/" + fileName); + InputStream in = RayNativeRuntime.class.getResourceAsStream("/native/" + fileName); Preconditions.checkNotNull(in, "{} doesn't exist.", fileName); Files.copy(in, Paths.get(file.getAbsolutePath()), StandardCopyOption.REPLACE_EXISTING); System.load(file.getAbsolutePath()); diff --git a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java index bec4eea591e8..025cb82835a0 100644 --- a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java +++ b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java @@ -1,6 +1,7 @@ package org.ray.runtime.runner; import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import java.io.File; @@ -94,6 +95,7 @@ private File getTempFile(String fileName) { file = File.createTempFile(fileName, ""); file.deleteOnExit(); try (InputStream in = RunManager.class.getResourceAsStream(fileName)) { + Preconditions.checkNotNull(in, "{} doesn't exist.", fileName); Files.copy(in, Paths.get(file.getCanonicalPath()), StandardCopyOption.REPLACE_EXISTING); } file.setExecutable(true); @@ -203,7 +205,7 @@ private void startRedisServer() { private String startRedisInstance(String ip, int port, String password, Integer shard) { List command = Lists.newArrayList( // The redis-server executable file. - getTempFile("/redis-server").getAbsolutePath(), + getTempFile("/native/redis-server").getAbsolutePath(), "--protected-mode", "no", "--port", @@ -212,7 +214,7 @@ private String startRedisInstance(String ip, int port, String password, Integer "warning", "--loadmodule", // The redis module file. - getTempFile("/libray_redis_module.so").getAbsolutePath() + getTempFile("/native/libray_redis_module.so").getAbsolutePath() ); if (!StringUtil.isNullOrEmpty(password)) { @@ -250,7 +252,7 @@ private void startRaylet() { // See `src/ray/raylet/main.cc` for the meaning of each parameter. List command = ImmutableList.of( // The raylet executable file. - getTempFile("/raylet").getAbsolutePath(), + getTempFile("/native/raylet").getAbsolutePath(), rayConfig.rayletSocketName, rayConfig.objectStoreSocketName, "0", // The object manager port. @@ -326,7 +328,7 @@ private String buildWorkerCommandRaylet() { private void startObjectStore() { List command = ImmutableList.of( // The plasma store executable file. - getTempFile("/plasma_store_server").getAbsolutePath(), + getTempFile("/native/plasma_store_server").getAbsolutePath(), "-s", rayConfig.objectStoreSocketName, "-m", diff --git a/java/test.sh b/java/test.sh index 6200d632264d..c879fd06177a 100755 --- a/java/test.sh +++ b/java/test.sh @@ -12,11 +12,10 @@ sh $ROOT_DIR/generate_deps.sh echo "Compiling Java code." pushd $ROOT_DIR/.. -# bazel checkstyle for java -bazel test //java:all -c opt # compile all the targets -bazel build //java:all -c opt --verbose_failures -popd +bazel build //java:all --verbose_failures +# bazel checkstyle for java +bazel test //java:all --test_tag_filters="checkstyle" # The following are soft links # TODO: remove this once cmake is removed @@ -25,9 +24,8 @@ ln -sf $ROOT_DIR/../bazel-bin/java/* $ROOT_DIR/../build/java/ mkdir -p $ROOT_DIR/tutorial/target/ ln -sf $ROOT_DIR/../bazel-bin/java/org_ray_ray_tutorial_deploy.jar $ROOT_DIR/tutorial/target/ray-tutorial-0.1-SNAPSHOT.jar -pushd $ROOT_DIR/test echo "Running tests under cluster mode." -ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar -Dray.home=$ROOT_DIR/../ $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml || cluster_exit_code=$? +ENABLE_MULTI_LANGUAGE_TESTS=1 bazel test //java:all_tests || cluster_exit_code=$? # exit_code == 2 means there are some tests skiped. if [ $cluster_exit_code -eq 2 ] && [ $cluster_exit_code -eq 0 ] ; then @@ -35,7 +33,8 @@ if [ $cluster_exit_code -eq 2 ] && [ $cluster_exit_code -eq 0 ] ; then fi echo "Running tests under single-process mode." -java -jar -Dray.home=$ROOT_DIR/../ -Dray.run-mode=SINGLE_PROCESS $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml || single_exit_code=$? +bazel test //java:all_tests --jvmopt="-Dray.run-mode=SINGLE_PROCESS" || single_exit_code=$? + # exit_code == 2 means there are some tests skiped. if [ $single_exit_code -eq 2 ] && [ $single_exit_code -eq 0 ] ; then exit $exit_code diff --git a/java/tools/checkstyle/BUILD b/java/tools/checkstyle/BUILD deleted file mode 100644 index b323bb6066f8..000000000000 --- a/java/tools/checkstyle/BUILD +++ /dev/null @@ -1,6 +0,0 @@ -exports_files([ - "checkstyle.sh", - "license-header.txt", - "checkstyle.xml", - "checkstyle-suppressions.xml", -]) diff --git a/java/tools/checkstyle/README.md b/java/tools/checkstyle/README.md deleted file mode 100644 index 0a83260d8081..000000000000 --- a/java/tools/checkstyle/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# Checkstyle integration - -## What is Checkstyle - ->[checkstyle](http://checkstyle.sourceforge.net/) is a development tool to -help programmers write Java code that adheres to a coding standard. -It automates the process of checking Java code to spare humans of this -boring (but important) task. This makes it ideal for projects that want to -enforce a coding standard. - -## Integrating with `bazel` - -To apply `checkstyle` check to Java code built with `bazel`, -we use the `checkstyle_test` rule. - -Example: - -`WORKSPACE`: -``` -load("//tools/checkstyle:checkstyle.bzl", "checkstyle_dependencies") -checkstyle_dependencies() -``` - -`BUILD`: - -``` -load("//tools/checkstyle:checkstyle.bzl", "checkstyle_test") - -java_library( - name = "lib", - srcs = ["Lib.java"], -) - -java_library( - name = "ignored_lib", - srcs = ["Lib.java"], - tags = ["checkstyle_ignore"] -) - -checkstyle_test( - name = "lib-checkstyle", - allow_failure = 1, - target = ":lib", -) - -``` - -Sample code illustrates several points: -* To apply Checkstyle to Java target named `lib`, create a `checkstyle_test` -named `lib-checkstyle` (with a dash). Such naming convention is enforced at rule level. -* `checkstyle_test` **is** a test in Bazel context, which means you can do -`bazel test //:lib-checkstyle` and get result of that inspection. -* To perform initial integration, `allow_failure` attribute can be temporarily set, -which will _still_ perform Checkstyle test, but will succeed (as Bazel test) -independently of check's result. -* Targets that do not need to be checked (e.g. generated code), such as `ignored_lib` -can be ignored by assigning `tags = ["checkstyle_ignore"]` -* Checkstyle test **completeness** (that is, all Java code that _needs_ checking -**is** being checked) is enforced via the shell script named `check-for-missing-targets.sh` -ran by CI. If you omit **both** `checkstyle_test` target and `checkstyle_ignore` tag, the script -will exit with non-zero code and print out non-covered targets. - -## Configuring Checkstyle - -Checkstyle is configured by an XML file. By default, current rule implementation uses -`//tools/checkstyle:config.xml` which is based on Google Java style -with added checks of license header -(via [`RegexpHeader`](http://checkstyle.sourceforge.net/config_header.html#RegexpHeader) -and `//tools/checkstyle:license-header.txt`) and package header -(via [`PackageName`](http://checkstyle.sourceforge.net/config_naming.html#PackageName)) diff --git a/java/tools/checkstyle/check-for-missing-targets.sh b/java/tools/checkstyle/check-for-missing-targets.sh deleted file mode 100755 index a28b6d2a1db4..000000000000 --- a/java/tools/checkstyle/check-for-missing-targets.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -# This script verifies that all Java targets except those having assigned `checkstyle_ignore` tag -# do have a accompanying Checkstyle target - -# [1] Get all Java targets except those having assigned `checkstyle_ignore` tag -bazel query 'kind(java_*, //...) - kind(java_proto_library, //...) - //third_party/... - attr("tags", "checkstyle_ignore", //...)' | sort >/tmp/all_java_targets -# [2] Get all checkstyle targets' names, replace '-checkstyle' with nothing so it matches Java target name -bazel query 'kind(checkstyle_test, //...)' | sed -e 's|-checkstyle||g' | sort >/tmp/checkstyle_covered_java_targets - -# [3] Targets in [1] which are not in [2] -NON_COVERED_JAVA_TARGETS=$(comm -23 /tmp/all_java_targets /tmp/checkstyle_covered_java_targets) - -# Having non-empty [3] is an error -if [[ ! -z "$NON_COVERED_JAVA_TARGETS" ]]; then - echo "$(tput setaf 1)[!] These java targets do not have accompanying checkstyle targets:"$(tput sgr0) - echo ${NON_COVERED_JAVA_TARGETS} - exit 1 -fi diff --git a/java/tools/checkstyle/checkstyle.bzl b/java/tools/checkstyle/checkstyle.bzl deleted file mode 100644 index 51129aeb830a..000000000000 --- a/java/tools/checkstyle/checkstyle.bzl +++ /dev/null @@ -1,127 +0,0 @@ -#Structure representing info about Java source files -JavaSourceFiles = provider( - fields = { - 'files' : 'java source files' - } -) - -# This aspect is responsible for collecting Java sources for target -# When applied to target, `JavaSourceFiles` struct will be attached to -# target info -def collect_sources_impl(target, ctx): - files = [] - if hasattr(ctx.rule.attr, 'srcs'): - for src in ctx.rule.attr.srcs: - for file in src.files: - if file.extension == 'java': - files.append(file) - return [JavaSourceFiles(files = files)] - - -collect_sources = aspect( - implementation = collect_sources_impl, -) - - -# `ctx` is rule context: https://docs.bazel.build/versions/master/skylark/lib/ctx.html -def _checkstyle_test_impl(ctx): - # verify target name matches naming conventions - if "{}-checkstyle".format(ctx.attr.target.label.name) != ctx.attr.name: - fail("target should follow `{java_library target name}-checkstyle` pattern") - - suppressions = ctx.file.suppressions - opts = ctx.attr.opts - sopts = ctx.attr.string_opts - - # Checkstyle and its dependencies - checkstyle_dependencies = ctx.attr._checkstyle.java.transitive_runtime_deps - classpath = ":".join([file.path for file in checkstyle_dependencies]) - - args = "" - inputs = [] - if ctx.file.config: - args += " -c %s" % ctx.file.config.path - inputs.append(ctx.file.config) - if suppressions: - inputs.append(suppressions) - - # Build command to run Checkstyle test - cmd = " ".join( - ["java -cp %s com.puppycrawl.tools.checkstyle.Main" % classpath] + - [args] + - ["--%s" % x for x in opts] + - ["--%s %s" % (k, sopts[k]) for k in sopts] + - [file.path for file in ctx.attr.target[JavaSourceFiles].files] - ) - - # Wrap checkstyle command in a shell script so allow_failure is supported - ctx.actions.expand_template( - template = ctx.file._checkstyle_sh_template, - output = ctx.outputs.checkstyle_script, - substitutions = { - "{command}" : cmd, - "{allow_failure}": str(int(ctx.attr.allow_failure)), - }, - is_executable = True, - ) - - files = [ctx.outputs.checkstyle_script, ctx.file.license] + ctx.attr.target[JavaSourceFiles].files + checkstyle_dependencies.to_list() + inputs - runfiles = ctx.runfiles( - files = files, - collect_data = True - ) - return DefaultInfo( - executable = ctx.outputs.checkstyle_script, - files = depset(files), - runfiles = runfiles, - ) - -checkstyle_test = rule( - implementation = _checkstyle_test_impl, - test = True, - attrs = { - "config": attr.label( - allow_single_file=True, - doc = "A checkstyle configuration file", - default = "//java:checkstyle.xml", - ), - "suppressions": attr.label( - allow_single_file=True, - doc = ("A file for specifying files and lines " + - "that should be suppressed from checks." + - "Example: https://github.com/checkstyle/checkstyle/blob/master/config/suppressions.xml"), - default = "//java:checkstyle-suppressions.xml", - ), - "license": attr.label( - allow_single_file=True, - doc = "A license file that can be used with the checkstyle license target", - default = "//java/tools/checkstyle:license-header.txt", - ), - "opts": attr.string_list( - doc = "Options to be passed on the command line that have no argument" - ), - "string_opts": attr.string_dict( - doc = "Options to be passed on the command line that have an argument" - ), - "target": attr.label( - doc = "The java_library target to check sources on", - aspects = [collect_sources], - mandatory = True - ), - "allow_failure": attr.bool( - default = False, - doc = "Successfully finish the test even if checkstyle failed" - ), - "_checkstyle_sh_template": attr.label( - allow_single_file = True, - default = "//java/tools/checkstyle:checkstyle.sh" - ), - "_checkstyle": attr.label( - default = "//java/third_party/com/puppycrawl/tools:checkstyle" - ), - }, - outputs = { - "checkstyle_script": "%{name}.sh", - }, -) - diff --git a/java/tools/checkstyle/checkstyle.sh b/java/tools/checkstyle/checkstyle.sh deleted file mode 100755 index 622a9cda9db9..000000000000 --- a/java/tools/checkstyle/checkstyle.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -COMMAND="{command}" -ALLOW_FAILURE="{allow_failure}" - -echo "ALLOW_FAILURE IS $ALLOW_FAILURE" - -source ~/.bash_profile - -${COMMAND} -result=$? - -if [[ ${ALLOW_FAILURE} -eq 1 ]]; then - exit 0 -fi - -exit ${result} diff --git a/java/tools/checkstyle/license-header.txt b/java/tools/checkstyle/license-header.txt deleted file mode 100644 index 2d4ccdb2ee66..000000000000 --- a/java/tools/checkstyle/license-header.txt +++ /dev/null @@ -1,15 +0,0 @@ -^\W* -^\W*Copyright 20[0-9]{2} The StartupOS Authors. -^\W* -^\W*Licensed under the Apache License, Version 2.0 \(the "License"\); -^\W*you may not use this file except in compliance with the License. -^\W*You may obtain a copy of the License at -^\W* -^\W*https://www.apache.org/licenses/LICENSE-2.0 -^\W* -^\W*Unless required by applicable law or agreed to in writing, software -^\W*distributed under the License is distributed on an "AS IS" BASIS, -^\W*WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -^\W*See the License for the specific language governing permissions and -^\W*limitations under the License. -^\W* \ No newline at end of file diff --git a/src/ray/test/run_gcs_tests.sh b/src/ray/test/run_gcs_tests.sh index b564f5c7e1a3..335d7fe0dbed 100644 --- a/src/ray/test/run_gcs_tests.sh +++ b/src/ray/test/run_gcs_tests.sh @@ -6,7 +6,7 @@ set -e set -x -bazel build "//:gcs_client_test" "//:asio_test" "//:ray_redis_module.so" -c opt +bazel build "//:gcs_client_test" "//:asio_test" "//:ray_redis_module.so" # Start Redis. if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then diff --git a/src/ray/test/run_object_manager_tests.sh b/src/ray/test/run_object_manager_tests.sh index 37fe59f0aa30..af844a11e93b 100644 --- a/src/ray/test/run_object_manager_tests.sh +++ b/src/ray/test/run_object_manager_tests.sh @@ -6,7 +6,7 @@ set -e set -x -bazel build -c opt "//:object_manager_stress_test" "//:object_manager_test" "@plasma//:plasma_store_server" +bazel build "//:object_manager_stress_test" "//:object_manager_test" "@plasma//:plasma_store_server" # Get the directory in which this script is executing. SCRIPT_DIR="`dirname \"$0\"`" @@ -27,16 +27,16 @@ LOAD_MODULE_ARGS="--loadmodule ${REDIS_MODULE}" STORE_EXEC="./bazel-bin/external/plasma/plasma_store_server" # Allow cleanup commands to fail. -bazel run -c opt //:redis-cli -- -p 6379 shutdown || true +bazel run //:redis-cli -- -p 6379 shutdown || true sleep 1s -bazel run -c opt //:redis-server -- --loglevel warning ${LOAD_MODULE_ARGS} --port 6379 & +bazel run //:redis-server -- --loglevel warning ${LOAD_MODULE_ARGS} --port 6379 & sleep 1s # Run tests. ./bazel-bin/object_manager_stress_test $STORE_EXEC sleep 1s # Use timeout=1000ms for the Wait tests. ./bazel-bin/object_manager_test $STORE_EXEC 1000 -bazel run -c opt //:redis-cli -- -p 6379 shutdown +bazel run //:redis-cli -- -p 6379 shutdown sleep 1s # Include raylet integration test once it's ready. diff --git a/src/ray/test/run_object_manager_valgrind.sh b/src/ray/test/run_object_manager_valgrind.sh index f7e708e30401..80282b067f04 100644 --- a/src/ray/test/run_object_manager_valgrind.sh +++ b/src/ray/test/run_object_manager_valgrind.sh @@ -6,7 +6,7 @@ set -e set -x -bazel build -c opt "//:object_manager_stress_test" "//:object_manager_test" "@plasma//:plasma_store_server" +bazel build "//:object_manager_stress_test" "//:object_manager_test" "@plasma//:plasma_store_server" # Get the directory in which this script is executing. SCRIPT_DIR="`dirname \"$0\"`" @@ -30,9 +30,9 @@ VALGRIND_CMD="valgrind --track-origins=yes --leak-check=full --show-leak-kinds=a # Allow cleanup commands to fail. killall plasma_store || true -bazel run -c opt //:redis-cli -- -p 6379 shutdown || true +bazel run //:redis-cli -- -p 6379 shutdown || true sleep 1s -bazel run -c opt //:redis-server -- --loglevel warning ${LOAD_MODULE_ARGS} --port 6379 & +bazel run //:redis-server -- --loglevel warning ${LOAD_MODULE_ARGS} --port 6379 & sleep 1s # Run tests. Use timeout=10000ms for the Wait tests since tests run slower @@ -40,7 +40,7 @@ sleep 1s $VALGRIND_CMD ./bazel-bin/object_manager_test $STORE_EXEC 10000 sleep 1s $VALGRIND_CMD ./bazel-bin/object_manager_stress_test $STORE_EXEC -bazel run -c opt //:redis-cli -- -p 6379 shutdown +bazel run //:redis-cli -- -p 6379 shutdown sleep 1s # Include raylet integration test once it's ready. From 16d9dd609c64ae87402cb071c026a0509526367f Mon Sep 17 00:00:00 2001 From: Jovan Date: Fri, 15 Mar 2019 16:24:06 +0800 Subject: [PATCH 071/115] Fix function manager test WIP WIP WIP WIP --- java/runtime/src/main/resources/DemoApp.java | 27 ++++++++++ .../functionmanager/FunctionManagerTest.java | 53 ++++++++++++++----- 2 files changed, 66 insertions(+), 14 deletions(-) create mode 100644 java/runtime/src/main/resources/DemoApp.java diff --git a/java/runtime/src/main/resources/DemoApp.java b/java/runtime/src/main/resources/DemoApp.java new file mode 100644 index 000000000000..28bf5f5e7708 --- /dev/null +++ b/java/runtime/src/main/resources/DemoApp.java @@ -0,0 +1,27 @@ +package demo; + +import org.ray.api.Ray; +import org.ray.api.RayObject; +import org.ray.api.annotation.RayRemote; + +public class DemoApp { + + @RayRemote + public static String sayHello() { + String ret = "hello"; + return ret; + } + + public static void main(String []args) { + try { + Ray.init(); + RayObject hello = Ray.call(DemoApp::sayHello); + System.out.println(hello.get()); + } catch (Throwable t) { + t.printStackTrace(); + } finally { + Ray.shutdown(); + } + } + +} diff --git a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java index e0307635a9aa..0b7730bc4659 100644 --- a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java +++ b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java @@ -5,6 +5,7 @@ import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.Map; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.ray.api.annotation.RayRemote; @@ -15,6 +16,8 @@ import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import javax.tools.JavaCompiler; +import javax.tools.ToolProvider; /** * Tests for {@link FunctionManager} @@ -117,25 +120,47 @@ public void testLoadFunctionTableForClass() { //TODO(qwang): This is an integration test case, and we should move it to test folder in the future. @Test - public void testGetFunctionFromLocalResource() throws Exception{ - UniqueId driverId = UniqueId.fromHexString("0123456789012345678901234567890123456789"); - - //TODO(qwang): We should use a independent app demo instead of `tutorial`. - final String resourcePath = "/tmp/ray/test/resource"; - final String srcJarPath = System.getProperty("user.dir") + - "/../tutorial/target/ray-tutorial-0.1-SNAPSHOT.jar"; - final String destJarPath = resourcePath + "/" + driverId.toString() + - "/ray-tutorial-0.1-SNAPSHOT.jar"; - - File file = new File(resourcePath + "/" + driverId.toString()); + public void testGetFunctionFromLocalResource() throws Exception { + final String resourcePath = "/tmp/ray/java_test/resource"; + UniqueId driverId = UniqueId.randomId(); + final String driverResourcePath = resourcePath + "/" + driverId.toString(); + final String destDir = System.getProperty("user.dir") + "/demo"; + compileAndPack(System.getProperty("user.dir") + "/../runtime/src/main/resources/DemoApp.java", destDir); + + final String sourcePath = destDir + "/DemoApp.jar"; + final String destPath = driverResourcePath + "/DemoApp.jar"; + File file = new File(destPath); file.mkdirs(); - Files.copy(Paths.get(srcJarPath), Paths.get(destJarPath), StandardCopyOption.REPLACE_EXISTING); + Files.move(Paths.get(sourcePath), Paths.get(destPath), + StandardCopyOption.REPLACE_EXISTING); + FunctionDescriptor sayHelloDescriptor = new FunctionDescriptor( + "demo.DemoApp", "sayHello", "()Ljava/lang/String;"); final FunctionManager functionManager = new FunctionManager(resourcePath); - FunctionDescriptor sayHelloDescriptor = new FunctionDescriptor("org.ray.exercise.Exercise02", - "sayHello", "()Ljava/lang/String;"); + RayFunction func = functionManager.getFunction(driverId, sayHelloDescriptor); Assert.assertEquals(func.getFunctionDescriptor(), sayHelloDescriptor); } + private static void compileAndPack(String sourceFilePath, String destDir) throws Exception { + + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + int result = compiler.run(null, null, null, "-d", System.getProperty("user.dir"), sourceFilePath); + if (result != 0) { + throw new RuntimeException(String.format("Failed to compile file : %s", sourceFilePath)); + } + + String[] packJarCommand = new String[]{ + "jar", + "-cvf", + destDir + "/DemoApp.jar", + "demo/DemoApp.class" + }; + + Runtime.getRuntime().exec(packJarCommand); + + // Sleep for waiting for jar. + TimeUnit.SECONDS.sleep(1); + } + } From c4de040d6bce8658c77740a1e0a082cae9cfb729 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Fri, 15 Mar 2019 18:35:57 +0800 Subject: [PATCH 072/115] add import --- java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java b/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java index 90ed65d8761f..ff5a34f2026b 100644 --- a/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java +++ b/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java @@ -3,6 +3,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import java.io.File; +import java.io.InputStream; import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.file.Files; From fcdc8567acaf8f0a557876cf54cd74ef087d60fb Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Fri, 15 Mar 2019 18:45:54 +0800 Subject: [PATCH 073/115] fix --- java/runtime/src/main/resources/DemoApp.java | 20 +------ .../functionmanager/FunctionManagerTest.java | 58 ++++++++----------- 2 files changed, 25 insertions(+), 53 deletions(-) diff --git a/java/runtime/src/main/resources/DemoApp.java b/java/runtime/src/main/resources/DemoApp.java index 28bf5f5e7708..eb812ca538b1 100644 --- a/java/runtime/src/main/resources/DemoApp.java +++ b/java/runtime/src/main/resources/DemoApp.java @@ -1,27 +1,11 @@ package demo; -import org.ray.api.Ray; -import org.ray.api.RayObject; import org.ray.api.annotation.RayRemote; public class DemoApp { @RayRemote - public static String sayHello() { - String ret = "hello"; - return ret; + public static String hello() { + return "hello"; } - - public static void main(String []args) { - try { - Ray.init(); - RayObject hello = Ray.call(DemoApp::sayHello); - System.out.println(hello.get()); - } catch (Throwable t) { - t.printStackTrace(); - } finally { - Ray.shutdown(); - } - } - } diff --git a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java index 0b7730bc4659..ce10949c38c8 100644 --- a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java +++ b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java @@ -1,11 +1,12 @@ package org.ray.runtime.functionmanager; -import java.io.File; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.Map; -import java.util.concurrent.TimeUnit; +import javax.tools.JavaCompiler; +import javax.tools.ToolProvider; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.ray.api.annotation.RayRemote; @@ -16,8 +17,6 @@ import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import javax.tools.JavaCompiler; -import javax.tools.ToolProvider; /** * Tests for {@link FunctionManager} @@ -118,49 +117,38 @@ public void testLoadFunctionTableForClass() { ImmutablePair.of(barConstructorDescriptor.name, barConstructorDescriptor.typeDescriptor))); } - //TODO(qwang): This is an integration test case, and we should move it to test folder in the future. @Test public void testGetFunctionFromLocalResource() throws Exception { final String resourcePath = "/tmp/ray/java_test/resource"; UniqueId driverId = UniqueId.randomId(); final String driverResourcePath = resourcePath + "/" + driverId.toString(); - final String destDir = System.getProperty("user.dir") + "/demo"; - compileAndPack(System.getProperty("user.dir") + "/../runtime/src/main/resources/DemoApp.java", destDir); - - final String sourcePath = destDir + "/DemoApp.jar"; - final String destPath = driverResourcePath + "/DemoApp.jar"; - File file = new File(destPath); - file.mkdirs(); - Files.move(Paths.get(sourcePath), Paths.get(destPath), - StandardCopyOption.REPLACE_EXISTING); - - FunctionDescriptor sayHelloDescriptor = new FunctionDescriptor( - "demo.DemoApp", "sayHello", "()Ljava/lang/String;"); - final FunctionManager functionManager = new FunctionManager(resourcePath); - - RayFunction func = functionManager.getFunction(driverId, sayHelloDescriptor); - Assert.assertEquals(func.getFunctionDescriptor(), sayHelloDescriptor); - } - - private static void compileAndPack(String sourceFilePath, String destDir) throws Exception { + // Copy Demo.java to the driver resource path. + String javaFilePath = driverResourcePath + "/Demo.java"; + try (InputStream in = FunctionManagerTest.class.getResourceAsStream("Demo.java")) { + Files.copy(in, Paths.get(javaFilePath), StandardCopyOption.REPLACE_EXISTING); + } + // Compile the java file. JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - int result = compiler.run(null, null, null, "-d", System.getProperty("user.dir"), sourceFilePath); + int result = compiler.run(null, null, null, "-d", driverResourcePath, javaFilePath); if (result != 0) { - throw new RuntimeException(String.format("Failed to compile file : %s", sourceFilePath)); + throw new RuntimeException("Couldn't compile Demo.java."); } - + // Package the class file into a jar file. String[] packJarCommand = new String[]{ - "jar", - "-cvf", - destDir + "/DemoApp.jar", - "demo/DemoApp.class" + "jar", + "-cvf", + driverResourcePath + "/DemoApp.jar", + driverResourcePath + "/DemoApp.class" }; + Runtime.getRuntime().exec(packJarCommand).waitFor(); - Runtime.getRuntime().exec(packJarCommand); - - // Sleep for waiting for jar. - TimeUnit.SECONDS.sleep(1); + // Test loading the function. + FunctionDescriptor descriptor = new FunctionDescriptor( + "DemoApp", "hello", "()Ljava/lang/String;"); + final FunctionManager functionManager = new FunctionManager(resourcePath); + RayFunction func = functionManager.getFunction(driverId, descriptor); + Assert.assertEquals(func.getFunctionDescriptor(), descriptor); } } From 3a1c838c80fe624f63a63cee679250851d784b55 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Fri, 15 Mar 2019 18:46:35 +0800 Subject: [PATCH 074/115] fix --- java/runtime/src/main/resources/DemoApp.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/java/runtime/src/main/resources/DemoApp.java b/java/runtime/src/main/resources/DemoApp.java index eb812ca538b1..9f6d8b433a41 100644 --- a/java/runtime/src/main/resources/DemoApp.java +++ b/java/runtime/src/main/resources/DemoApp.java @@ -1,5 +1,3 @@ -package demo; - import org.ray.api.annotation.RayRemote; public class DemoApp { From a2bf5ceabe5928c148a09d96b782890bb4f77e3f Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Fri, 15 Mar 2019 19:05:15 +0800 Subject: [PATCH 075/115] fix --- java/runtime/src/main/resources/DemoApp.java | 9 --- .../functionmanager/FunctionManagerTest.java | 68 +++++++++++-------- 2 files changed, 40 insertions(+), 37 deletions(-) delete mode 100644 java/runtime/src/main/resources/DemoApp.java diff --git a/java/runtime/src/main/resources/DemoApp.java b/java/runtime/src/main/resources/DemoApp.java deleted file mode 100644 index 9f6d8b433a41..000000000000 --- a/java/runtime/src/main/resources/DemoApp.java +++ /dev/null @@ -1,9 +0,0 @@ -import org.ray.api.annotation.RayRemote; - -public class DemoApp { - - @RayRemote - public static String hello() { - return "hello"; - } -} diff --git a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java index ce10949c38c8..2be9f5784ce3 100644 --- a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java +++ b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java @@ -1,12 +1,12 @@ package org.ray.runtime.functionmanager; -import java.io.InputStream; +import java.io.File; import java.nio.file.Files; import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.util.Map; import javax.tools.JavaCompiler; import javax.tools.ToolProvider; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.ray.api.annotation.RayRemote; @@ -122,33 +122,45 @@ public void testGetFunctionFromLocalResource() throws Exception { final String resourcePath = "/tmp/ray/java_test/resource"; UniqueId driverId = UniqueId.randomId(); final String driverResourcePath = resourcePath + "/" + driverId.toString(); - - // Copy Demo.java to the driver resource path. - String javaFilePath = driverResourcePath + "/Demo.java"; - try (InputStream in = FunctionManagerTest.class.getResourceAsStream("Demo.java")) { - Files.copy(in, Paths.get(javaFilePath), StandardCopyOption.REPLACE_EXISTING); - } - // Compile the java file. - JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - int result = compiler.run(null, null, null, "-d", driverResourcePath, javaFilePath); - if (result != 0) { - throw new RuntimeException("Couldn't compile Demo.java."); + File driverResourceDir = new File(driverResourcePath); + driverResourceDir.mkdirs(); + + String demoJavaFile = ""; + demoJavaFile += "public class DemoApp {\n"; + demoJavaFile += " public static String hello() {\n"; + demoJavaFile += " return \"hello\";\n"; + demoJavaFile += " }\n"; + demoJavaFile += "}"; + + try { + // Write the demo java file to the driver resource path. + String javaFilePath = driverResourcePath + "/DemoApp.java"; + Files.write(Paths.get(javaFilePath), demoJavaFile.getBytes()); + + // Compile the java file. + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + int result = compiler.run(null, null, null, "-d", driverResourcePath, javaFilePath); + if (result != 0) { + throw new RuntimeException("Couldn't compile Demo.java."); + } + // Package the class file into a jar file. + String[] packJarCommand = new String[]{ + "jar", + "-cvf", + driverResourcePath + "/DemoApp.jar", + "DemoApp.class" + }; + new ProcessBuilder(packJarCommand).directory(driverResourceDir).start().waitFor(); + + // Test loading the function. + FunctionDescriptor descriptor = new FunctionDescriptor( + "DemoApp", "hello", "()Ljava/lang/String;"); + final FunctionManager functionManager = new FunctionManager(resourcePath); + RayFunction func = functionManager.getFunction(driverId, descriptor); + Assert.assertEquals(func.getFunctionDescriptor(), descriptor); + } finally { + FileUtils.deleteDirectory(driverResourceDir); } - // Package the class file into a jar file. - String[] packJarCommand = new String[]{ - "jar", - "-cvf", - driverResourcePath + "/DemoApp.jar", - driverResourcePath + "/DemoApp.class" - }; - Runtime.getRuntime().exec(packJarCommand).waitFor(); - - // Test loading the function. - FunctionDescriptor descriptor = new FunctionDescriptor( - "DemoApp", "hello", "()Ljava/lang/String;"); - final FunctionManager functionManager = new FunctionManager(resourcePath); - RayFunction func = functionManager.getFunction(driverId, descriptor); - Assert.assertEquals(func.getFunctionDescriptor(), descriptor); } } From 2b3d9228848976f74e540c99bba0dc5415479493 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Sat, 16 Mar 2019 02:04:06 +0800 Subject: [PATCH 076/115] move the java deps to runtime's resources --- .gitignore | 2 +- BUILD.bazel | 38 ++++++++++--------- CMakeLists.txt | 8 ++-- bazel/BUILD.plasma | 2 +- build.sh | 1 + java/BUILD.bazel | 5 ++- java/runtime/pom.xml | 3 ++ .../org/ray/runtime/RayNativeRuntime.java | 2 +- .../org/ray/runtime/runner/RunManager.java | 8 ++-- 9 files changed, 39 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index 290744290f05..a5c9754c27d6 100644 --- a/.gitignore +++ b/.gitignore @@ -149,7 +149,7 @@ java/**/.settings java/**/.classpath java/**/.project java/third_party/workspace.bzl -java/runtime/src/main/resources/native/ +java/runtime/native_dependencies/ # python virtual env venv diff --git a/BUILD.bazel b/BUILD.bazel index 66548cd665c2..b780cba3008b 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -395,7 +395,7 @@ cc_binary( "//conditions:default": ["external/bazel_tools/tools/jdk/include/linux"], }), linkshared = 1, - linkstatic = 0, + linkstatic = 1, deps = [ "//:raylet_lib", "@plasma//:plasma_client", @@ -416,13 +416,12 @@ filegroup( "@bazel_tools//src/conditions:darwin": [":libraylet_library_java.dylib"], "//conditions:default": [":libraylet_library_java.so"], }), - visibility = ["//visibility:public"], ) filegroup( name = "gcs_fbs_file", srcs = ["src/ray/gcs/format/gcs.fbs"], - visibility = ["//visibility:public"], + visibility = ["//java:__subpackages__"], ) flatbuffer_py_library( @@ -536,7 +535,7 @@ genrule( ) cc_binary( - name = "ray_redis_module.so", + name = "libray_redis_module.so", srcs = [ "src/ray/gcs/redis_module/ray_redis_module.cc", "src/ray/gcs/redis_module/redis_string.h", @@ -559,7 +558,7 @@ genrule( "//:python_node_manager_fbs", "//:redis-server", "//:redis-cli", - "//:ray_redis_module.so", + "//:libray_redis_module.so", "//:raylet", "//:raylet_monitor", "@plasma//:plasma_store_server", @@ -573,7 +572,7 @@ genrule( cp $(location //:redis-server) python/ray/core/src/ray/thirdparty/redis/src/ && cp $(location //:redis-cli) python/ray/core/src/ray/thirdparty/redis/src/ && mkdir -p python/ray/core/src/ray/gcs/redis_module/ && - cp $(locations //:ray_redis_module.so) python/ray/core/src/ray/gcs/redis_module/libray_redis_module.so && + cp $(locations //:libray_redis_module.so) python/ray/core/src/ray/gcs/redis_module/ && mkdir -p python/ray/core/src/ray/raylet/ && cp $(location //:raylet_monitor) python/ray/core/src/ray/raylet/ && mkdir -p python/ray/core/src/plasma && @@ -586,25 +585,30 @@ genrule( """, ) -genrule( - name = "ray_java_pkg", +filegroup( + name = "ray_java_deps_files", srcs = [ "//:redis-server", - "//:ray_redis_module.so", + "//:libray_redis_module.so", "//:raylet", "//:raylet_library_java", "@plasma//:plasma_store_server", "@plasma//:plasma_client_java", ], - outs = ["ray_java_pkg/java/runtime/src/main/resources/native"], + visibility = ["//java:__subpackages__"], +) + +genrule( + name = "ray_java_pkg", + srcs = [ + ":ray_java_deps_files", + ], + outs = ["ray_java_pkg/java/runtime/native_dependencies"], cmd = """ set -x && - mkdir -p $(@D)/native && - cp $(location //:redis-server) $(@D)/native/ && - cp $(location //:ray_redis_module.so) $(@D)/native/libray_redis_module.so && - cp $(location //:raylet) $(@D)/native/ && - cp $(location //:raylet_library_java) $(@D)/native/ && - cp $(location @plasma//:plasma_store_server) $(@D)/native/ && - cp $(location @plasma//:plasma_client_java) $(@D)/native/ + mkdir -p $(@D)/native_dependencies && + for f in $(locations //:ray_java_deps_files); do + cp $$f $(@D)/native_dependencies/; + done """, ) diff --git a/CMakeLists.txt b/CMakeLists.txt index b1ea4cc023b1..84236d4e9ce5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -154,7 +154,7 @@ if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES") # Copy java native dependencies. add_custom_command(TARGET copy_ray POST_BUILD - COMMAND mkdir -p ${CMAKE_SOURCE_DIR}/java/runtime/src/main/resources/native) + COMMAND mkdir -p ${CMAKE_SOURCE_DIR}/java/runtime/native_dependencies) set(java_native_dependencies "src/ray/thirdparty/redis/src/redis-server" "src/ray/gcs/redis_module/libray_redis_module.so" @@ -163,12 +163,12 @@ if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES") foreach(file ${java_native_dependencies}) add_custom_command(TARGET copy_ray POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/${file} - ${CMAKE_SOURCE_DIR}/java/runtime/src/main/resources/native) + ${CMAKE_SOURCE_DIR}/java/runtime/native_dependencies) endforeach() add_custom_command(TARGET copy_ray POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy ${ARROW_HOME}/bin/plasma_store_server - ${CMAKE_SOURCE_DIR}/java/runtime/src/main/resources/native) + ${CMAKE_SOURCE_DIR}/java/runtime/native_dependencies) add_custom_command(TARGET copy_ray POST_BUILD COMMAND $(CMAKE_COMMAND) -E copy ${ARROW_LIBRARY_DIR}/libplasma_java.* - ${CMAKE_SOURCE_DIR}/java/runtime/src/main/resources/native) + ${CMAKE_SOURCE_DIR}/java/runtime/native_dependencies) endif() diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index f221c8af17fd..db2b9d1afc11 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -102,7 +102,7 @@ cc_binary( "cpp/src", ], linkshared = 1, - linkstatic = 0, + linkstatic = 1, deps = [":plasma_client"], ) diff --git a/build.sh b/build.sh index 05ace6feb4b2..2db89b1865de 100755 --- a/build.sh +++ b/build.sh @@ -122,6 +122,7 @@ else --target=$ROOT_DIR/python/ray/pyarrow_files pyarrow==0.12.0.RAY \ --find-links https://s3-us-west-2.amazonaws.com/arrow-wheels/9357dc130789ee42f8181d8724bee1d5d1509060/index.html + # This is for mvn compile. if [ "$RAY_BUILD_JAVA" == "YES" ]; then bazel build //:ray_java_pkg --verbose_failures find $ROOT_DIR/bazel-genfiles/ray_java_pkg/ -exec chmod +w {} \; diff --git a/java/BUILD.bazel b/java/BUILD.bazel index d941822f8d16..04ea3c175586 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -68,8 +68,9 @@ java_library( ], resources = glob([ "runtime/src/main/resources/**", - ]), - data = ["//:raylet_library_java"], + ]) + [ + "//:ray_java_deps_files", + ], deps = [ ":org_ray_ray_api", "@plasma//:org_apache_arrow_arrow_plasma", diff --git a/java/runtime/pom.xml b/java/runtime/pom.xml index a87f14cc471a..4305b074313f 100644 --- a/java/runtime/pom.xml +++ b/java/runtime/pom.xml @@ -83,6 +83,9 @@ src/main/resources + + native_dependencies + diff --git a/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java b/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java index ff5a34f2026b..a0fbdf01f28d 100644 --- a/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java +++ b/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java @@ -88,7 +88,7 @@ public void start() throws Exception { // Copy the file from resources to a temp dir, and load the native library. File file = File.createTempFile(fileName, ""); file.deleteOnExit(); - InputStream in = RayNativeRuntime.class.getResourceAsStream("/native/" + fileName); + InputStream in = RayNativeRuntime.class.getResourceAsStream("/" + fileName); Preconditions.checkNotNull(in, "{} doesn't exist.", fileName); Files.copy(in, Paths.get(file.getAbsolutePath()), StandardCopyOption.REPLACE_EXISTING); System.load(file.getAbsolutePath()); diff --git a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java index 025cb82835a0..447da9d9e7f7 100644 --- a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java +++ b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java @@ -205,7 +205,7 @@ private void startRedisServer() { private String startRedisInstance(String ip, int port, String password, Integer shard) { List command = Lists.newArrayList( // The redis-server executable file. - getTempFile("/native/redis-server").getAbsolutePath(), + getTempFile("/redis-server").getAbsolutePath(), "--protected-mode", "no", "--port", @@ -214,7 +214,7 @@ private String startRedisInstance(String ip, int port, String password, Integer "warning", "--loadmodule", // The redis module file. - getTempFile("/native/libray_redis_module.so").getAbsolutePath() + getTempFile("/libray_redis_module.so").getAbsolutePath() ); if (!StringUtil.isNullOrEmpty(password)) { @@ -252,7 +252,7 @@ private void startRaylet() { // See `src/ray/raylet/main.cc` for the meaning of each parameter. List command = ImmutableList.of( // The raylet executable file. - getTempFile("/native/raylet").getAbsolutePath(), + getTempFile("/raylet").getAbsolutePath(), rayConfig.rayletSocketName, rayConfig.objectStoreSocketName, "0", // The object manager port. @@ -328,7 +328,7 @@ private String buildWorkerCommandRaylet() { private void startObjectStore() { List command = ImmutableList.of( // The plasma store executable file. - getTempFile("/native/plasma_store_server").getAbsolutePath(), + getTempFile("/plasma_store_server").getAbsolutePath(), "-s", rayConfig.objectStoreSocketName, "-m", From cc5d784f063eea34b800ce10c11a29ca7ea75336 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Wed, 6 Mar 2019 19:54:13 +0800 Subject: [PATCH 077/115] delete the variable RAY_USR_CMAKE --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 91bf94fe86cb..b2f419fcd44e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -114,7 +114,7 @@ matrix: # Build MacOS wheels. - os: osx osx_image: xcode7 - env: MAC_WHEELS=1 PYTHONWARNINGS=ignore RAY_USE_CMAKE=1 + env: MAC_WHEELS=1 PYTHONWARNINGS=ignore install: - ./ci/travis/install-dependencies.sh # This command should be kept in sync with ray/python/README-building-wheels.md. @@ -129,7 +129,6 @@ matrix: - PYTHON=3.5 - RAY_USE_NEW_GCS=on - PYTHONWARNINGS=ignore - - RAY_USE_CMAKE=1 install: From 36a20d6fd5784cb25b0dd6ac3007597249c31679 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Sat, 16 Mar 2019 02:38:47 +0800 Subject: [PATCH 078/115] modify .travis.yml as the master --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b2f419fcd44e..ef7e4719f8e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -103,7 +103,7 @@ matrix: # Build Linux wheels. - os: linux dist: trusty - env: LINUX_WHEELS=1 PYTHONWARNINGS=ignore RAY_USE_CMAKE=1 + env: LINUX_WHEELS=1 PYTHONWARNINGS=ignore install: - ./ci/suppress_output ./ci/travis/install-dependencies.sh # This command should be kept in sync with ray/python/README-building-wheels.md. From 575564da7281999b2a075684e06591fdf1bef6cb Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Sat, 16 Mar 2019 02:45:26 +0800 Subject: [PATCH 079/115] modify .travis.yml as ray-project/ray's master --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ef7e4719f8e3..608fbdd341ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -129,6 +129,7 @@ matrix: - PYTHON=3.5 - RAY_USE_NEW_GCS=on - PYTHONWARNINGS=ignore + - RAY_USE_CMAKE=1 install: From c0276b4b9343c2603451daa10a0e7e714766e06b Mon Sep 17 00:00:00 2001 From: Yuhong Guo Date: Mon, 18 Mar 2019 11:46:46 +0800 Subject: [PATCH 080/115] Add temporary commit for arrow to start to test --- WORKSPACE | 4 ++-- bazel/BUILD.plasma | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 0bd857b90d36..4bd4154a42e7 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -48,8 +48,8 @@ git_repository( new_git_repository( name = "plasma", build_file = "@//bazel:BUILD.plasma", - commit = "fa8967cb9ec228f6544b2eecc0f1bd78f29daa4d", - remote = "https://github.com/ray-project/arrow", + commit = "7fd7bd07b730ca01ccfa5ee965402224cd0d0c5e", + remote = "https://github.com/ant-tech-alliance/arrow", ) new_git_repository( diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index db2b9d1afc11..e0afd8580616 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -64,7 +64,6 @@ cc_library( "cpp/src/plasma/io.cc", "cpp/src/plasma/malloc.cc", "cpp/src/plasma/plasma.cc", - "cpp/src/plasma/plasma_allocator.cc", "cpp/src/plasma/protocol.cc", ], hdrs = [ @@ -72,11 +71,11 @@ cc_library( "cpp/src/plasma/common.h", "cpp/src/plasma/common_generated.h", "cpp/src/plasma/compat.h", + "cpp/src/plasma/external_store.h", "cpp/src/plasma/fling.h", "cpp/src/plasma/io.h", "cpp/src/plasma/malloc.h", "cpp/src/plasma/plasma.h", - "cpp/src/plasma/plasma_allocator.h", "cpp/src/plasma/plasma_generated.h", "cpp/src/plasma/protocol.h", ], @@ -148,11 +147,13 @@ cc_library( "cpp/src/plasma/events.cc", "cpp/src/plasma/eviction_policy.cc", "cpp/src/plasma/external_store.cc", + "cpp/src/plasma/plasma_allocator.cc", ], hdrs = [ "cpp/src/plasma/events.h", "cpp/src/plasma/eviction_policy.h", "cpp/src/plasma/external_store.h", + "cpp/src/plasma/plasma_allocator.h", "cpp/src/plasma/store.h", "cpp/src/plasma/thirdparty/ae/ae.h", "cpp/src/plasma/thirdparty/ae/ae_epoll.c", From 5364ebecd66a73f236b40e30bd98f75437275336 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Mon, 18 Mar 2019 20:43:45 +0800 Subject: [PATCH 081/115] modify java/BUILD --- BUILD.bazel | 65 +++----- bazel/ray.bzl | 31 ++++ build.sh | 11 +- java/BUILD.bazel | 150 +++++++----------- .../functionmanager/FunctionManagerTest.java | 31 ---- java/test.sh | 9 +- .../api/test/MultiLanguageClusterTest.java | 2 +- 7 files changed, 112 insertions(+), 187 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index b780cba3008b..1cc2b2d67f7b 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -15,6 +15,7 @@ cc_binary( ":ray_util", ":raylet_lib", ], + visibility = ["//java:__subpackages__"], ) cc_binary( @@ -416,6 +417,7 @@ filegroup( "@bazel_tools//src/conditions:darwin": [":libraylet_library_java.dylib"], "//conditions:default": [":libraylet_library_java.so"], }), + visibility = ["//java:__subpackages__"], ) filegroup( @@ -532,6 +534,7 @@ genrule( mv ./src/redis-cli $(location redis-cli) && chmod +x $(location redis-cli) """, + visibility = ["//java:__subpackages__"], ) cc_binary( @@ -547,6 +550,7 @@ cc_binary( deps = [ ":ray_common", ], + visibility = ["//java:__subpackages__"], ) genrule( @@ -563,52 +567,23 @@ genrule( "//:raylet_monitor", "@plasma//:plasma_store_server", ], - outs = ["ray_pkg"], - cmd = """ - set -x && - mkdir -p python/ray && - cp $(location python/ray/_raylet.so) python/ray && - mkdir -p python/ray/core/src/ray/thirdparty/redis/src/ && - cp $(location //:redis-server) python/ray/core/src/ray/thirdparty/redis/src/ && - cp $(location //:redis-cli) python/ray/core/src/ray/thirdparty/redis/src/ && - mkdir -p python/ray/core/src/ray/gcs/redis_module/ && - cp $(locations //:libray_redis_module.so) python/ray/core/src/ray/gcs/redis_module/ && - mkdir -p python/ray/core/src/ray/raylet/ && - cp $(location //:raylet_monitor) python/ray/core/src/ray/raylet/ && - mkdir -p python/ray/core/src/plasma && - cp $(location @plasma//:plasma_store_server) python/ray/core/src/plasma/ && - cp $(location //:raylet) python/ray/core/src/ray/raylet/ && - for f in $(locations //:python_gcs_fbs); do cp $$f python/ray/core/generated/; done && - mkdir -p python/ray/core/generated/ray/protocol/ && - for f in $(locations //:python_node_manager_fbs); do cp $$f python/ray/core/generated/ray/protocol/; done && - mv python $(location ray_pkg) - """, -) - -filegroup( - name = "ray_java_deps_files", - srcs = [ - "//:redis-server", - "//:libray_redis_module.so", - "//:raylet", - "//:raylet_library_java", - "@plasma//:plasma_store_server", - "@plasma//:plasma_client_java", - ], - visibility = ["//java:__subpackages__"], -) - -genrule( - name = "ray_java_pkg", - srcs = [ - ":ray_java_deps_files", - ], - outs = ["ray_java_pkg/java/runtime/native_dependencies"], + outs = ["ray_pkg.out"], cmd = """ set -x && - mkdir -p $(@D)/native_dependencies && - for f in $(locations //:ray_java_deps_files); do - cp $$f $(@D)/native_dependencies/; - done + WORK_DIR=$$(pwd) && + cp -f $(location python/ray/_raylet.so) $$WORK_DIR/python/ray && + mkdir -p $$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/ && + cp -f $(location //:redis-server) $$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/ && + cp -f $(location //:redis-cli) $$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/ && + mkdir -p $$WORK_DIR/python/ray/core/src/ray/gcs/redis_module/ && + cp -f $(locations //:libray_redis_module.so) $$WORK_DIR/python/ray/core/src/ray/gcs/redis_module/ && + cp -f $(location //:raylet_monitor) $$WORK_DIR/python/ray/core/src/ray/raylet/ && + cp -f $(location @plasma//:plasma_store_server) $$WORK_DIR/python/ray/core/src/plasma/ && + cp -f $(location //:raylet) $$WORK_DIR/python/ray/core/src/ray/raylet/ && + for f in $(locations //:python_gcs_fbs); do cp -f $$f $$WORK_DIR/python/ray/core/generated/; done && + mkdir -p $$WORK_DIR/python/ray/core/generated/ray/protocol/ && + for f in $(locations //:python_node_manager_fbs); do cp -f $$f $$WORK_DIR/python/ray/core/generated/ray/protocol/; done && + echo $$WORK_DIR > $@ """, + local = 1, ) diff --git a/bazel/ray.bzl b/bazel/ray.bzl index 85ae845ef1c0..71277b1a3ee1 100644 --- a/bazel/ray.bzl +++ b/bazel/ray.bzl @@ -1,4 +1,5 @@ load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_library_public") +load("@com_github_checkstyle_java//checkstyle:checkstyle.bzl", "checkstyle_test") def flatbuffer_py_library(name, srcs, outs, out_prefix, includes = [], include_paths = []): flatbuffer_library_public( @@ -21,3 +22,33 @@ def flatbuffer_java_library(name, srcs, outs, out_prefix, includes = [], include include_paths = include_paths, includes = includes, ) + +def define_java_module(name, additional_srcs = [], additional_resources = [], define_test_lib = False, test_deps = [], **kwargs): + native.java_library( + name = "org_ray_ray_" + name, + srcs = additional_srcs + native.glob([name + "/src/main/java/**/*.java"]), + resources = native.glob([name + "/src/main/resources/**"]) + additional_resources, + **kwargs + ) + checkstyle_test( + name = "org_ray_ray_" + name + "-checkstyle", + target = "//java:org_ray_ray_" + name, + config = "//java:checkstyle.xml", + suppressions = "//java:checkstyle-suppressions.xml", + size = "small", + tags = ["checkstyle"] + ) + if define_test_lib: + native.java_library( + name = "org_ray_ray_" + name + "_test", + srcs = native.glob([name + "/src/test/java/**/*.java"]), + deps = test_deps + ) + checkstyle_test( + name = "org_ray_ray_" + name + "_test-checkstyle", + target = "//java:org_ray_ray_" + name + "_test", + config = "//java:checkstyle.xml", + suppressions = "//java:checkstyle-suppressions.xml", + size = "small", + tags = ["checkstyle"] + ) diff --git a/build.sh b/build.sh index 2db89b1865de..f10be272af87 100755 --- a/build.sh +++ b/build.sh @@ -124,20 +124,11 @@ else # This is for mvn compile. if [ "$RAY_BUILD_JAVA" == "YES" ]; then - bazel build //:ray_java_pkg --verbose_failures - find $ROOT_DIR/bazel-genfiles/ray_java_pkg/ -exec chmod +w {} \; - cp -r $ROOT_DIR/bazel-genfiles/ray_java_pkg/java $ROOT_DIR/ || true + bazel build //java:copy_java_native_deps --verbose_failures fi if [ "$RAY_BUILD_PYTHON" == "YES" ]; then bazel build //:ray_pkg --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHON_EXECUTABLE - # Copy files and keep them writeable. This is a workaround, as Bazel - # marks all generated files non-writeable. If we would just copy them - # over without adding write permission, the copy would fail the next time. - # TODO(pcm): It would be great to have a solution here that does not - # require us to copy the files. - find $ROOT_DIR/bazel-genfiles/ray_pkg/ -exec chmod +w {} \; - cp -r $ROOT_DIR/bazel-genfiles/ray_pkg/ray $ROOT_DIR/python || true fi fi diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 04ea3c175586..c4c26f7cbc2e 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -1,5 +1,4 @@ -load("@com_github_checkstyle_java//checkstyle:checkstyle.bzl", "checkstyle_test") -load("//bazel:ray.bzl", "flatbuffer_java_library") +load("//bazel:ray.bzl", "flatbuffer_java_library", "define_java_module") exports_files([ "testng.xml", @@ -14,33 +13,24 @@ java_binary( ) java_import( - name = "org_ray_ray_java_root", + name = "all_modules", jars = [ "liborg_ray_ray_api.jar", "liborg_ray_ray_api-src.jar", "liborg_ray_ray_runtime.jar", "liborg_ray_ray_runtime-src.jar", - "liborg_ray_ray_runtime_test.jar", - "liborg_ray_ray_runtime_test-src.jar", - "liborg_ray_ray_test.jar", - "liborg_ray_ray_test-src.jar", - "org_ray_ray_tutorial_deploy.jar", - "org_ray_ray_tutorial_deploy-src.jar", - "all_tests_deploy.jar", - "all_tests_deploy-src.jar", + "liborg_ray_ray_tutorial.jar", + "liborg_ray_ray_tutorial-src.jar", ], deps = [ ":org_ray_ray_api", ":org_ray_ray_runtime", ":org_ray_ray_tutorial", - ":org_ray_ray_test", - ":org_ray_ray_runtime_test", ], ) -java_library( - name = "org_ray_ray_api", - srcs = glob(["api/src/main/java/**/*.java"]), +define_java_module( + name = "api", deps = [ "@org_slf4j_slf4j_log4j12//jar", "@log4j_log4j//jar", @@ -50,26 +40,22 @@ java_library( ], ) -checkstyle_test( - name = "org_ray_ray_api-checkstyle", - target = ":org_ray_ray_api", - config = "//java:checkstyle.xml", - suppressions = "//java:checkstyle-suppressions.xml", - size = "small", - tags = ["checkstyle"], -) - -java_library( - name = "org_ray_ray_runtime", - srcs = glob([ - "runtime/src/main/java/**/*.java" - ]) + [ +define_java_module( + name = "runtime", + additional_srcs = [ ":generate_java_gcs_fbs", ], - resources = glob([ - "runtime/src/main/resources/**", - ]) + [ - "//:ray_java_deps_files", + additional_resources = [ + ":java_native_deps", + ], + define_test_lib = True, + test_deps = [ + ":org_ray_ray_api", + ":org_ray_ray_runtime", + "@org_apache_commons_commons_lang3//jar", + "@com_beust_jcommander//jar", + "@org_testng_testng//jar", + "@commons_io_commons_io//jar", ], deps = [ ":org_ray_ray_api", @@ -93,19 +79,8 @@ java_library( ], ) -checkstyle_test( - name = "org_ray_ray_runtime-checkstyle", - target = ":org_ray_ray_runtime", - config = "//java:checkstyle.xml", - suppressions = "//java:checkstyle-suppressions.xml", - size = "small", - tags = ["checkstyle"], -) - -java_binary( - name = "org_ray_ray_tutorial", - srcs = glob(["tutorial/src/main/java/**/*.java"]), - resources = glob(["tutorial/src/main/resources/**"]), +define_java_module( + name = "tutorial", deps = [ ":org_ray_ray_api", ":org_ray_ray_runtime", @@ -113,18 +88,8 @@ java_binary( ], ) -checkstyle_test( - name = "org_ray_ray_tutorial-checkstyle", - target = ":org_ray_ray_tutorial", - config = "//java:checkstyle.xml", - suppressions = "//java:checkstyle-suppressions.xml", - size = "small", - tags = ["checkstyle"], -) - -java_library( - name = "org_ray_ray_test", - srcs = glob(["test/src/main/java/**/*.java"]), +define_java_module( + name = "test", deps = [ ":org_ray_ray_api", ":org_ray_ray_runtime", @@ -137,36 +102,6 @@ java_library( ], ) -checkstyle_test( - name = "org_ray_ray_test-checkstyle", - target = ":org_ray_ray_test", - config = "//java:checkstyle.xml", - suppressions = "//java:checkstyle-suppressions.xml", - size = "small", - tags = ["checkstyle"], -) - -java_library( - name = "org_ray_ray_runtime_test", - srcs = glob(["runtime/src/test/java/**/*.java"]), - deps = [ - ":org_ray_ray_api", - ":org_ray_ray_runtime", - "@org_apache_commons_commons_lang3//jar", - "@com_beust_jcommander//jar", - "@org_testng_testng//jar", - ], -) - -checkstyle_test( - name = "org_ray_ray_runtime_test-checkstyle", - target = ":org_ray_ray_runtime_test", - config = "//java:checkstyle.xml", - suppressions = "//java:checkstyle-suppressions.xml", - size = "small", - tags = ["checkstyle"], -) - java_test( name = "all_tests", main_class = "org.testng.TestNG", @@ -178,7 +113,7 @@ java_test( ], ) -flatbuffers_generate_files = [ +flatbuffers_generated_files = [ "ActorCheckpointData.java", "ActorCheckpointIdData.java", "ActorState.java", @@ -214,7 +149,7 @@ flatbuffers_generate_files = [ flatbuffer_java_library( name = "java_gcs_fbs", srcs = ["//:gcs_fbs_file"], - outs = flatbuffers_generate_files, + outs = flatbuffers_generated_files, out_prefix = "", ) @@ -222,7 +157,7 @@ genrule( name = "generate_java_gcs_fbs", srcs = [":java_gcs_fbs"], outs = [ - "runtime/src/main/java/org/ray/runtime/generated/" + file for file in flatbuffers_generate_files + "runtime/src/main/java/org/ray/runtime/generated/" + file for file in flatbuffers_generated_files ], cmd = """ for f in $(locations //java:java_gcs_fbs); do @@ -233,3 +168,34 @@ genrule( """, local = 1, ) + +filegroup( + name = "java_native_deps", + srcs = [ + "//:redis-server", + "//:libray_redis_module.so", + "//:raylet", + "//:raylet_library_java", + "@plasma//:plasma_store_server", + "@plasma//:plasma_client_java", + ], +) + +genrule( + name = "copy_java_native_deps", + srcs = [ + ":java_native_deps", + ], + outs = ["copy_java_native_deps.out"], + cmd = """ + set -x + WORK_DIR=$$(pwd) + mkdir -p $$WORK_DIR/java/runtime/native_dependencies/ + for f in $(locations //java:java_native_deps); do + chmod +w $$f + cp -f $$f $$WORK_DIR/java/runtime/native_dependencies + done + echo $$WORK_DIR > $@ + """, + local = 1, +) diff --git a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java index 7d42dc3e1598..72a6d82b7fb6 100644 --- a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java +++ b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java @@ -165,35 +165,4 @@ public void testGetFunctionFromLocalResource() throws Exception { } } - private static void compileAndPack(String sourceFilePath, String destDir) throws Exception { - JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - int result = compiler.run(null, null, null, sourceFilePath); - - if (result != 0) { - throw new RuntimeException(String.format("Failed to compile file : %s", sourceFilePath)); - } - - - String[] packJarCommand = new String[]{ - "jar", - "-cvf", - destDir + "/DemoApp.jar", - "org/ray/demo/DemoApp.class" - }; - - Runtime.getRuntime().exec(packJarCommand); - - // Move to destination directory. - final String sourceFileDir = (new File(sourceFilePath)).getParent(); - final String classFileSourcePath = sourceFileDir + "/DemoApp.class"; - final String classFileDestPath = destDir + "/DemoApp.class"; - File file = new File(classFileDestPath); - file.mkdirs(); - Files.move(Paths.get(classFileSourcePath), Paths.get(classFileDestPath), - StandardCopyOption.REPLACE_EXISTING); - - // Sleep for waiting for jar. - TimeUnit.SECONDS.sleep(1); - } - } diff --git a/java/test.sh b/java/test.sh index 3c69f60fee91..7e975524777c 100755 --- a/java/test.sh +++ b/java/test.sh @@ -13,17 +13,10 @@ sh $ROOT_DIR/generate_deps.sh echo "Compiling Java code." pushd $ROOT_DIR/.. # compile all the targets -bazel build //java:all --verbose_failures +bazel build //java:all_modules --verbose_failures # bazel checkstyle for java bazel test //java:all --test_tag_filters="checkstyle" -# The following are soft links -# TODO: remove this once cmake is removed -mkdir -p $ROOT_DIR/../build/java/ -ln -sf $ROOT_DIR/../bazel-bin/java/* $ROOT_DIR/../build/java/ -mkdir -p $ROOT_DIR/tutorial/target/ -ln -sf $ROOT_DIR/../bazel-bin/java/org_ray_ray_tutorial_deploy.jar $ROOT_DIR/tutorial/target/ray-tutorial-0.1-SNAPSHOT.jar - echo "Running tests under cluster mode." ENABLE_MULTI_LANGUAGE_TESTS=1 bazel test //java:all_tests --test_output="errors" || cluster_exit_code=$? diff --git a/java/test/src/main/java/org/ray/api/test/MultiLanguageClusterTest.java b/java/test/src/main/java/org/ray/api/test/MultiLanguageClusterTest.java index ad3033681a68..5542226c0974 100644 --- a/java/test/src/main/java/org/ray/api/test/MultiLanguageClusterTest.java +++ b/java/test/src/main/java/org/ray/api/test/MultiLanguageClusterTest.java @@ -67,7 +67,7 @@ public void setUp(Method method) { // Start ray cluster. String testDir = System.getProperty("user.dir"); String workerOptions = - " -classpath " + String.format("%s/../../build/java/*:%s/target/*", testDir, testDir); + " -classpath " + System.getProperty("java.class.path"); final List startCommand = ImmutableList.of( "ray", "start", From 90b94be75dea2f8e8cf3bfdc193e5cd015ddee36 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Mon, 18 Mar 2019 21:46:57 +0800 Subject: [PATCH 082/115] modify with raulchen's suggestion --- BUILD.bazel | 4 ++- build.sh | 4 +-- java/runtime/src/main/resources/DemoApp.java | 27 ------------------- .../functionmanager/FunctionManagerTest.java | 2 -- .../main/java/org/ray/api/test/BaseTest.java | 1 - 5 files changed, 5 insertions(+), 33 deletions(-) delete mode 100644 java/runtime/src/main/resources/DemoApp.java diff --git a/BUILD.bazel b/BUILD.bazel index 1cc2b2d67f7b..7e658c9d4d83 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -582,7 +582,9 @@ genrule( cp -f $(location //:raylet) $$WORK_DIR/python/ray/core/src/ray/raylet/ && for f in $(locations //:python_gcs_fbs); do cp -f $$f $$WORK_DIR/python/ray/core/generated/; done && mkdir -p $$WORK_DIR/python/ray/core/generated/ray/protocol/ && - for f in $(locations //:python_node_manager_fbs); do cp -f $$f $$WORK_DIR/python/ray/core/generated/ray/protocol/; done && + for f in $(locations //:python_node_manager_fbs); do + cp -f $$f $$WORK_DIR/python/ray/core/generated/ray/protocol/; + done && echo $$WORK_DIR > $@ """, local = 1, diff --git a/build.sh b/build.sh index f10be272af87..ae78c36106d0 100755 --- a/build.sh +++ b/build.sh @@ -122,9 +122,9 @@ else --target=$ROOT_DIR/python/ray/pyarrow_files pyarrow==0.12.0.RAY \ --find-links https://s3-us-west-2.amazonaws.com/arrow-wheels/9357dc130789ee42f8181d8724bee1d5d1509060/index.html - # This is for mvn compile. if [ "$RAY_BUILD_JAVA" == "YES" ]; then - bazel build //java:copy_java_native_deps --verbose_failures + bazel run //java:bazel_deps -- generate -r $ROOT_DIR -s java/third_party/workspace.bzl -d java/dependencies.yaml + bazel build //java:all --verbose_failures fi if [ "$RAY_BUILD_PYTHON" == "YES" ]; then diff --git a/java/runtime/src/main/resources/DemoApp.java b/java/runtime/src/main/resources/DemoApp.java deleted file mode 100644 index f9c493d5a337..000000000000 --- a/java/runtime/src/main/resources/DemoApp.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.ray.demo; - -import org.ray.api.Ray; -import org.ray.api.RayObject; -import org.ray.api.annotation.RayRemote; - -public class DemoApp { - - @RayRemote - public static String sayHello() { - String ret = "hello"; - return ret; - } - - public static void main(String []args) { - try { - Ray.init(); - RayObject hello = Ray.call(DemoApp::sayHello); - System.out.println(hello.get()); - } catch (Throwable t) { - t.printStackTrace(); - } finally { - Ray.shutdown(); - } - } - -} diff --git a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java index 72a6d82b7fb6..2be9f5784ce3 100644 --- a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java +++ b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java @@ -17,8 +17,6 @@ import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import javax.tools.JavaCompiler; -import javax.tools.ToolProvider; /** * Tests for {@link FunctionManager} diff --git a/java/test/src/main/java/org/ray/api/test/BaseTest.java b/java/test/src/main/java/org/ray/api/test/BaseTest.java index 259540722d83..366ee9a6737f 100644 --- a/java/test/src/main/java/org/ray/api/test/BaseTest.java +++ b/java/test/src/main/java/org/ray/api/test/BaseTest.java @@ -33,7 +33,6 @@ public void setUpBase(Method method) { @AfterMethod public void tearDownBase() { - Ray.shutdown(); for (File file : filesToDelete) { From 72fe9a5e79c9c1d6b86ab50cd048ef8325506972 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Mon, 18 Mar 2019 21:47:59 +0800 Subject: [PATCH 083/115] Update java/test.sh Co-Authored-By: alegithub111 --- java/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/test.sh b/java/test.sh index 7e975524777c..7bcb1e6fdcce 100755 --- a/java/test.sh +++ b/java/test.sh @@ -14,7 +14,7 @@ echo "Compiling Java code." pushd $ROOT_DIR/.. # compile all the targets bazel build //java:all_modules --verbose_failures -# bazel checkstyle for java +# Lint Java code with checkstyle. bazel test //java:all --test_tag_filters="checkstyle" echo "Running tests under cluster mode." From cb4fc92d274b461ea5cd7ca8cb3b9b3d7f5d1336 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Mon, 18 Mar 2019 21:48:21 +0800 Subject: [PATCH 084/115] Update java/test.sh Co-Authored-By: alegithub111 --- java/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/test.sh b/java/test.sh index 7bcb1e6fdcce..7aeefc21d7d6 100755 --- a/java/test.sh +++ b/java/test.sh @@ -12,7 +12,7 @@ sh $ROOT_DIR/generate_deps.sh echo "Compiling Java code." pushd $ROOT_DIR/.. -# compile all the targets +# Compile all the targets. bazel build //java:all_modules --verbose_failures # Lint Java code with checkstyle. bazel test //java:all --test_tag_filters="checkstyle" From e0fa777e8a824063465016687cdf1ecdf21d8337 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Tue, 19 Mar 2019 10:28:57 +0800 Subject: [PATCH 085/115] modify some errors --- BUILD.bazel | 4 ++-- WORKSPACE | 2 +- bazel/ray.bzl | 6 +++--- src/ray/test/run_gcs_tests.sh | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 7e658c9d4d83..41366578d2b9 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -11,11 +11,11 @@ cc_binary( name = "raylet", srcs = ["src/ray/raylet/main.cc"], copts = COPTS, + visibility = ["//java:__subpackages__"], deps = [ ":ray_util", ":raylet_lib", ], - visibility = ["//java:__subpackages__"], ) cc_binary( @@ -547,10 +547,10 @@ cc_binary( copts = COPTS, linkshared = 1, linkstatic = 1, + visibility = ["//java:__subpackages__"], deps = [ ":ray_common", ], - visibility = ["//java:__subpackages__"], ) genrule( diff --git a/WORKSPACE b/WORKSPACE index 4bd4154a42e7..bb067fbd28a0 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -7,8 +7,8 @@ java_repositories() git_repository( name = "com_github_checkstyle_java", - remote = "https://github.com/alegithub111/checkstyle_java.git", commit = "85f37871ca03b9d3fee63c69c8107f167e24e77b", + remote = "https://github.com/alegithub111/checkstyle_java.git", ) git_repository( diff --git a/bazel/ray.bzl b/bazel/ray.bzl index 71277b1a3ee1..f55d1d29009e 100644 --- a/bazel/ray.bzl +++ b/bazel/ray.bzl @@ -36,13 +36,13 @@ def define_java_module(name, additional_srcs = [], additional_resources = [], de config = "//java:checkstyle.xml", suppressions = "//java:checkstyle-suppressions.xml", size = "small", - tags = ["checkstyle"] + tags = ["checkstyle"], ) if define_test_lib: native.java_library( name = "org_ray_ray_" + name + "_test", srcs = native.glob([name + "/src/test/java/**/*.java"]), - deps = test_deps + deps = test_deps, ) checkstyle_test( name = "org_ray_ray_" + name + "_test-checkstyle", @@ -50,5 +50,5 @@ def define_java_module(name, additional_srcs = [], additional_resources = [], de config = "//java:checkstyle.xml", suppressions = "//java:checkstyle-suppressions.xml", size = "small", - tags = ["checkstyle"] + tags = ["checkstyle"], ) diff --git a/src/ray/test/run_gcs_tests.sh b/src/ray/test/run_gcs_tests.sh index 335d7fe0dbed..c4a2f6a504c5 100644 --- a/src/ray/test/run_gcs_tests.sh +++ b/src/ray/test/run_gcs_tests.sh @@ -6,7 +6,7 @@ set -e set -x -bazel build "//:gcs_client_test" "//:asio_test" "//:ray_redis_module.so" +bazel build "//:gcs_client_test" "//:asio_test" "//:libray_redis_module.so" # Start Redis. if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then From 2435f519a98aab1d342f7390438d848e129ddb07 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Tue, 19 Mar 2019 10:59:20 +0800 Subject: [PATCH 086/115] modify some errors --- src/ray/test/run_gcs_tests.sh | 6 +++--- src/ray/test/run_object_manager_tests.sh | 2 +- src/ray/test/run_object_manager_valgrind.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ray/test/run_gcs_tests.sh b/src/ray/test/run_gcs_tests.sh index c4a2f6a504c5..2aaf88da72c9 100644 --- a/src/ray/test/run_gcs_tests.sh +++ b/src/ray/test/run_gcs_tests.sh @@ -16,9 +16,9 @@ if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then --loadmodule ./src/ray/gcs/redis_module/libray_redis_module.so \ --port 6379 & else - ./bazel-genfiles/ray_pkg/ray/core/src/ray/thirdparty/redis/src/redis-server \ + ./bazel-genfiles/redis-server \ --loglevel warning \ - --loadmodule ./bazel-bin/ray_redis_module.so \ + --loadmodule ./bazel-bin/libray_redis_module.so \ --port 6379 & fi sleep 1s @@ -26,5 +26,5 @@ sleep 1s ./bazel-bin/gcs_client_test ./bazel-bin/asio_test -./bazel-genfiles/ray_pkg/ray/core/src/ray/thirdparty/redis/src/redis-cli -p 6379 shutdown +./bazel-genfiles/redis-cli -p 6379 shutdown sleep 1s diff --git a/src/ray/test/run_object_manager_tests.sh b/src/ray/test/run_object_manager_tests.sh index af844a11e93b..a739f6eeec0a 100644 --- a/src/ray/test/run_object_manager_tests.sh +++ b/src/ray/test/run_object_manager_tests.sh @@ -22,7 +22,7 @@ if [ ! -d "$RAY_ROOT/python" ]; then exit 1 fi -REDIS_MODULE="./bazel-genfiles/ray_pkg/ray/core/src/ray/gcs/redis_module/libray_redis_module.so" +REDIS_MODULE="./bazel-bin/libray_redis_module.so" LOAD_MODULE_ARGS="--loadmodule ${REDIS_MODULE}" STORE_EXEC="./bazel-bin/external/plasma/plasma_store_server" diff --git a/src/ray/test/run_object_manager_valgrind.sh b/src/ray/test/run_object_manager_valgrind.sh index 80282b067f04..1fcdb77f367b 100644 --- a/src/ray/test/run_object_manager_valgrind.sh +++ b/src/ray/test/run_object_manager_valgrind.sh @@ -22,7 +22,7 @@ if [ ! -d "$RAY_ROOT/python" ]; then exit 1 fi -REDIS_MODULE="./bazel-genfiles/ray_pkg/ray/core/src/ray/gcs/redis_module/libray_redis_module.so" +REDIS_MODULE="./bazel-bin/libray_redis_module.so" LOAD_MODULE_ARGS="--loadmodule ${REDIS_MODULE}" STORE_EXEC="./bazel-bin/external/plasma/plasma_store_server" From f58b6a699785813641d783b8ef7f66b7c7daec70 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Tue, 19 Mar 2019 12:25:13 +0800 Subject: [PATCH 087/115] Update BUILD.bazel --- java/BUILD.bazel | 2 ++ 1 file changed, 2 insertions(+) diff --git a/java/BUILD.bazel b/java/BUILD.bazel index c4c26f7cbc2e..a513fe44811a 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -181,6 +181,8 @@ filegroup( ], ) +# This rule copies Java native dependencies to 'java/runtime/native_dependencies', +# this is only needed by Maven (see 'java/runtime/pom.xml'). genrule( name = "copy_java_native_deps", srcs = [ From 40ceb4e2fab6bdf40630aac353143d8707c57ec3 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Tue, 19 Mar 2019 12:32:46 +0800 Subject: [PATCH 088/115] modify with raulchen's suggestions --- java/generate_deps.sh | 12 ------------ java/test.sh | 12 +++--------- 2 files changed, 3 insertions(+), 21 deletions(-) delete mode 100644 java/generate_deps.sh diff --git a/java/generate_deps.sh b/java/generate_deps.sh deleted file mode 100644 index 39ecc9f488ae..000000000000 --- a/java/generate_deps.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -x - -ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) - -cd $ROOT_DIR/.. -bazel run //java:bazel_deps -- generate -r `pwd` -s java/third_party/workspace.bzl -d java/dependencies.yaml - -set +x -set +e diff --git a/java/test.sh b/java/test.sh index 7aeefc21d7d6..d4132fde9c21 100755 --- a/java/test.sh +++ b/java/test.sh @@ -7,21 +7,15 @@ set -x ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -# run this file before compile the targets -sh $ROOT_DIR/generate_deps.sh - -echo "Compiling Java code." pushd $ROOT_DIR/.. -# Compile all the targets. -bazel build //java:all_modules --verbose_failures -# Lint Java code with checkstyle. +echo "Linting Java code with checkstyle." bazel test //java:all --test_tag_filters="checkstyle" echo "Running tests under cluster mode." ENABLE_MULTI_LANGUAGE_TESTS=1 bazel test //java:all_tests --test_output="errors" || cluster_exit_code=$? # exit_code == 2 means there are some tests skiped. -if [ $cluster_exit_code -eq 2 ] && [ $cluster_exit_code -eq 0 ] ; then +if [ $cluster_exit_code -ne 2 ] && [ $cluster_exit_code -ne 0 ] ; then exit $exit_code fi @@ -29,7 +23,7 @@ echo "Running tests under single-process mode." bazel test //java:all_tests --jvmopt="-Dray.run-mode=SINGLE_PROCESS" --test_output="errors" || single_exit_code=$? # exit_code == 2 means there are some tests skiped. -if [ $single_exit_code -eq 2 ] && [ $single_exit_code -eq 0 ] ; then +if [ $single_exit_code -ne 2 ] && [ $single_exit_code -ne 0 ] ; then exit $exit_code fi From d002e5bcff156caeb7867877e6bffc7a2c9465c0 Mon Sep 17 00:00:00 2001 From: Yuhong Guo Date: Wed, 20 Mar 2019 15:41:20 +0800 Subject: [PATCH 089/115] Use apache arrow commit in bazel --- WORKSPACE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 0ca068d2a912..8155cba45b47 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -49,8 +49,8 @@ new_git_repository( new_git_repository( name = "plasma", build_file = "@//bazel:BUILD.plasma", - commit = "7fd7bd07b730ca01ccfa5ee965402224cd0d0c5e", - remote = "https://github.com/ant-tech-alliance/arrow", + commit = "9c33e1aee98f96a82c6a0b6bb214df172a143fb1", + remote = "https://github.com/apache/arrow", ) new_git_repository( From c285710ddce61f0a0225fc50f9c924a733ad9cd7 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Wed, 20 Mar 2019 17:46:32 +0800 Subject: [PATCH 090/115] modify java/test.sh error --- java/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/test.sh b/java/test.sh index d4132fde9c21..0902a983eb15 100755 --- a/java/test.sh +++ b/java/test.sh @@ -16,7 +16,7 @@ ENABLE_MULTI_LANGUAGE_TESTS=1 bazel test //java:all_tests --test_output="errors" # exit_code == 2 means there are some tests skiped. if [ $cluster_exit_code -ne 2 ] && [ $cluster_exit_code -ne 0 ] ; then - exit $exit_code + exit $cluster_exit_code fi echo "Running tests under single-process mode." @@ -24,7 +24,7 @@ bazel test //java:all_tests --jvmopt="-Dray.run-mode=SINGLE_PROCESS" --test_outp # exit_code == 2 means there are some tests skiped. if [ $single_exit_code -ne 2 ] && [ $single_exit_code -ne 0 ] ; then - exit $exit_code + exit $single_exit_code fi popd From 326f5a2d77ea01295404836e568da86a8aa12931 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Wed, 20 Mar 2019 21:07:34 +0800 Subject: [PATCH 091/115] Auto generate deps --- .gitignore | 2 +- build.sh | 2 +- java/BUILD.bazel | 23 ----------- java/generate_bazel_deps.sh | 15 +++++++ java/repo.bzl | 14 ------- java/third_party/BUILD | 0 java/third_party/workspace.bzl | 73 ---------------------------------- 7 files changed, 17 insertions(+), 112 deletions(-) create mode 100755 java/generate_bazel_deps.sh delete mode 100644 java/third_party/BUILD delete mode 100644 java/third_party/workspace.bzl diff --git a/.gitignore b/.gitignore index a5c9754c27d6..a4895649ee49 100644 --- a/.gitignore +++ b/.gitignore @@ -148,7 +148,7 @@ java/**/lib java/**/.settings java/**/.classpath java/**/.project -java/third_party/workspace.bzl +java/third_party/ java/runtime/native_dependencies/ # python virtual env diff --git a/build.sh b/build.sh index ae78c36106d0..76670753a68b 100755 --- a/build.sh +++ b/build.sh @@ -123,7 +123,7 @@ else --find-links https://s3-us-west-2.amazonaws.com/arrow-wheels/9357dc130789ee42f8181d8724bee1d5d1509060/index.html if [ "$RAY_BUILD_JAVA" == "YES" ]; then - bazel run //java:bazel_deps -- generate -r $ROOT_DIR -s java/third_party/workspace.bzl -d java/dependencies.yaml + java/generate_bazel_deps.sh bazel build //java:all --verbose_failures fi diff --git a/java/BUILD.bazel b/java/BUILD.bazel index a513fe44811a..cd07062e318d 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -6,29 +6,6 @@ exports_files([ "checkstyle-suppressions.xml", ]) -java_binary( - name = "bazel_deps", - main_class = "com.github.johnynek.bazel_deps.ParseProject", - runtime_deps = ["@bazel_deps//jar"], -) - -java_import( - name = "all_modules", - jars = [ - "liborg_ray_ray_api.jar", - "liborg_ray_ray_api-src.jar", - "liborg_ray_ray_runtime.jar", - "liborg_ray_ray_runtime-src.jar", - "liborg_ray_ray_tutorial.jar", - "liborg_ray_ray_tutorial-src.jar", - ], - deps = [ - ":org_ray_ray_api", - ":org_ray_ray_runtime", - ":org_ray_ray_tutorial", - ], -) - define_java_module( name = "api", deps = [ diff --git a/java/generate_bazel_deps.sh b/java/generate_bazel_deps.sh new file mode 100755 index 000000000000..5ce2eac0b329 --- /dev/null +++ b/java/generate_bazel_deps.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Generate Java dependencies for bazel. + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +jar_file="/tmp/bazel_deps.jar" +if [[ -e $jar_file ]]; then + echo "Use existing jar file from $jar_file." +else + echo "Downloading jar file to $jar_file." + curl -L -o $jar_file "https://github.com/oferb/startupos-binaries/releases/download/0.1.01/bazel_deps.jar" +fi + +echo "Generating Java dependencies for bazel." +java -jar $jar_file generate -r $ROOT_DIR/.. -s java/third_party/workspace.bzl -d java/dependencies.yaml diff --git a/java/repo.bzl b/java/repo.bzl index a06c285f234b..9c83cfd73795 100644 --- a/java/repo.bzl +++ b/java/repo.bzl @@ -1,18 +1,4 @@ load("//java/third_party:workspace.bzl", "maven_dependencies") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_jar") - -def create_if_needed(rule, name, **kwargs): - if name not in native.existing_rules(): - rule(name = name, **kwargs) - -def bazel_deps(): - create_if_needed( - http_jar, - name = "bazel_deps", - sha256 = "98b05c2826f2248f70e7356dc6c78bc52395904bb932fbb409a5abf5416e4292", - urls = ["https://github.com/oferb/startupos-binaries/releases/download/0.1.01/bazel_deps.jar"], - ) def java_repositories(): maven_dependencies() - bazel_deps() diff --git a/java/third_party/BUILD b/java/third_party/BUILD deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/java/third_party/workspace.bzl b/java/third_party/workspace.bzl deleted file mode 100644 index 4aa7c40b298f..000000000000 --- a/java/third_party/workspace.bzl +++ /dev/null @@ -1,73 +0,0 @@ -# Do not edit. bazel-deps autogenerates this file from java/dependencies.yaml. -def _jar_artifact_impl(ctx): - jar_name = "%s.jar" % ctx.name - ctx.download( - output=ctx.path("jar/%s" % jar_name), - url=ctx.attr.urls, - sha256=ctx.attr.sha256, - executable=False - ) - src_name="%s-sources.jar" % ctx.name - srcjar_attr="" - has_sources = len(ctx.attr.src_urls) != 0 - if has_sources: - ctx.download( - output=ctx.path("jar/%s" % src_name), - url=ctx.attr.src_urls, - sha256=ctx.attr.src_sha256, - executable=False - ) - srcjar_attr ='\n srcjar = ":%s",' % src_name - - build_file_contents = """ -package(default_visibility = ['//visibility:public']) -java_import( - name = 'jar', - jars = ['{jar_name}'],{srcjar_attr} -) -filegroup( - name = 'file', - srcs = [ - '{jar_name}', - '{src_name}' - ], - visibility = ['//visibility:public'] -)\n""".format(jar_name = jar_name, src_name = src_name, srcjar_attr = srcjar_attr) - ctx.file(ctx.path("jar/BUILD"), build_file_contents, False) - return None - -jar_artifact = repository_rule( - attrs = { - "artifact": attr.string(mandatory = True), - "sha256": attr.string(mandatory = True), - "urls": attr.string_list(mandatory = True), - "src_sha256": attr.string(mandatory = False, default=""), - "src_urls": attr.string_list(mandatory = False, default=[]), - }, - implementation = _jar_artifact_impl -) - -def jar_artifact_callback(hash): - src_urls = [] - src_sha256 = "" - source=hash.get("source", None) - if source != None: - src_urls = [source["url"]] - src_sha256 = source["sha256"] - jar_artifact( - artifact = hash["artifact"], - name = hash["name"], - urls = [hash["url"]], - sha256 = hash["sha256"], - src_urls = src_urls, - src_sha256 = src_sha256 - ) - native.bind(name = hash["bind"], actual = hash["actual"]) - -def list_dependencies(): - return [ - ] - -def maven_dependencies(callback = jar_artifact_callback): - for hash in list_dependencies(): - callback(hash) From 5613347c70fc51dafe81f9c291003f86fc5b3a76 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Wed, 20 Mar 2019 22:06:36 +0800 Subject: [PATCH 092/115] add java/WORKSPACE --- WORKSPACE | 9 --------- java/WORKSPACE | 9 +++++++++ 2 files changed, 9 insertions(+), 9 deletions(-) create mode 100644 java/WORKSPACE diff --git a/WORKSPACE b/WORKSPACE index 8155cba45b47..5233ccc42362 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,15 +1,6 @@ workspace(name = "com_github_ray_project_ray") load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository") -load("//java:repo.bzl", "java_repositories") - -java_repositories() - -git_repository( - name = "com_github_checkstyle_java", - commit = "85f37871ca03b9d3fee63c69c8107f167e24e77b", - remote = "https://github.com/alegithub111/checkstyle_java.git", -) git_repository( name = "com_github_nelhage_rules_boost", diff --git a/java/WORKSPACE b/java/WORKSPACE new file mode 100644 index 000000000000..afed9b11483e --- /dev/null +++ b/java/WORKSPACE @@ -0,0 +1,9 @@ +load("//java/third_party:workspace.bzl", "maven_dependencies") + +maven_dependencies() + +git_repository( + name = "com_github_checkstyle_java", + commit = "85f37871ca03b9d3fee63c69c8107f167e24e77b", + remote = "https://github.com/alegithub111/checkstyle_java.git", +) From 723afbd0b1e4f897f71c7777f3e8ea4a494afcda Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Wed, 20 Mar 2019 22:37:46 +0800 Subject: [PATCH 093/115] Revert "add java/WORKSPACE" This reverts commit 5613347c70fc51dafe81f9c291003f86fc5b3a76. --- WORKSPACE | 9 +++++++++ java/WORKSPACE | 9 --------- 2 files changed, 9 insertions(+), 9 deletions(-) delete mode 100644 java/WORKSPACE diff --git a/WORKSPACE b/WORKSPACE index 5233ccc42362..8155cba45b47 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,6 +1,15 @@ workspace(name = "com_github_ray_project_ray") load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository") +load("//java:repo.bzl", "java_repositories") + +java_repositories() + +git_repository( + name = "com_github_checkstyle_java", + commit = "85f37871ca03b9d3fee63c69c8107f167e24e77b", + remote = "https://github.com/alegithub111/checkstyle_java.git", +) git_repository( name = "com_github_nelhage_rules_boost", diff --git a/java/WORKSPACE b/java/WORKSPACE deleted file mode 100644 index afed9b11483e..000000000000 --- a/java/WORKSPACE +++ /dev/null @@ -1,9 +0,0 @@ -load("//java/third_party:workspace.bzl", "maven_dependencies") - -maven_dependencies() - -git_repository( - name = "com_github_checkstyle_java", - commit = "85f37871ca03b9d3fee63c69c8107f167e24e77b", - remote = "https://github.com/alegithub111/checkstyle_java.git", -) From f9a134467217f5809c217d30efba777c2c26e56f Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Wed, 20 Mar 2019 22:42:06 +0800 Subject: [PATCH 094/115] Revert "Auto generate deps" This reverts commit 326f5a2d77ea01295404836e568da86a8aa12931. --- .gitignore | 2 +- build.sh | 2 +- java/BUILD.bazel | 23 +++++++++++ java/generate_bazel_deps.sh | 15 ------- java/repo.bzl | 14 +++++++ java/third_party/BUILD | 0 java/third_party/workspace.bzl | 73 ++++++++++++++++++++++++++++++++++ 7 files changed, 112 insertions(+), 17 deletions(-) delete mode 100755 java/generate_bazel_deps.sh create mode 100644 java/third_party/BUILD create mode 100644 java/third_party/workspace.bzl diff --git a/.gitignore b/.gitignore index a4895649ee49..a5c9754c27d6 100644 --- a/.gitignore +++ b/.gitignore @@ -148,7 +148,7 @@ java/**/lib java/**/.settings java/**/.classpath java/**/.project -java/third_party/ +java/third_party/workspace.bzl java/runtime/native_dependencies/ # python virtual env diff --git a/build.sh b/build.sh index 76670753a68b..ae78c36106d0 100755 --- a/build.sh +++ b/build.sh @@ -123,7 +123,7 @@ else --find-links https://s3-us-west-2.amazonaws.com/arrow-wheels/9357dc130789ee42f8181d8724bee1d5d1509060/index.html if [ "$RAY_BUILD_JAVA" == "YES" ]; then - java/generate_bazel_deps.sh + bazel run //java:bazel_deps -- generate -r $ROOT_DIR -s java/third_party/workspace.bzl -d java/dependencies.yaml bazel build //java:all --verbose_failures fi diff --git a/java/BUILD.bazel b/java/BUILD.bazel index cd07062e318d..a513fe44811a 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -6,6 +6,29 @@ exports_files([ "checkstyle-suppressions.xml", ]) +java_binary( + name = "bazel_deps", + main_class = "com.github.johnynek.bazel_deps.ParseProject", + runtime_deps = ["@bazel_deps//jar"], +) + +java_import( + name = "all_modules", + jars = [ + "liborg_ray_ray_api.jar", + "liborg_ray_ray_api-src.jar", + "liborg_ray_ray_runtime.jar", + "liborg_ray_ray_runtime-src.jar", + "liborg_ray_ray_tutorial.jar", + "liborg_ray_ray_tutorial-src.jar", + ], + deps = [ + ":org_ray_ray_api", + ":org_ray_ray_runtime", + ":org_ray_ray_tutorial", + ], +) + define_java_module( name = "api", deps = [ diff --git a/java/generate_bazel_deps.sh b/java/generate_bazel_deps.sh deleted file mode 100755 index 5ce2eac0b329..000000000000 --- a/java/generate_bazel_deps.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -# Generate Java dependencies for bazel. - -ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) - -jar_file="/tmp/bazel_deps.jar" -if [[ -e $jar_file ]]; then - echo "Use existing jar file from $jar_file." -else - echo "Downloading jar file to $jar_file." - curl -L -o $jar_file "https://github.com/oferb/startupos-binaries/releases/download/0.1.01/bazel_deps.jar" -fi - -echo "Generating Java dependencies for bazel." -java -jar $jar_file generate -r $ROOT_DIR/.. -s java/third_party/workspace.bzl -d java/dependencies.yaml diff --git a/java/repo.bzl b/java/repo.bzl index 9c83cfd73795..a06c285f234b 100644 --- a/java/repo.bzl +++ b/java/repo.bzl @@ -1,4 +1,18 @@ load("//java/third_party:workspace.bzl", "maven_dependencies") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_jar") + +def create_if_needed(rule, name, **kwargs): + if name not in native.existing_rules(): + rule(name = name, **kwargs) + +def bazel_deps(): + create_if_needed( + http_jar, + name = "bazel_deps", + sha256 = "98b05c2826f2248f70e7356dc6c78bc52395904bb932fbb409a5abf5416e4292", + urls = ["https://github.com/oferb/startupos-binaries/releases/download/0.1.01/bazel_deps.jar"], + ) def java_repositories(): maven_dependencies() + bazel_deps() diff --git a/java/third_party/BUILD b/java/third_party/BUILD new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/java/third_party/workspace.bzl b/java/third_party/workspace.bzl new file mode 100644 index 000000000000..4aa7c40b298f --- /dev/null +++ b/java/third_party/workspace.bzl @@ -0,0 +1,73 @@ +# Do not edit. bazel-deps autogenerates this file from java/dependencies.yaml. +def _jar_artifact_impl(ctx): + jar_name = "%s.jar" % ctx.name + ctx.download( + output=ctx.path("jar/%s" % jar_name), + url=ctx.attr.urls, + sha256=ctx.attr.sha256, + executable=False + ) + src_name="%s-sources.jar" % ctx.name + srcjar_attr="" + has_sources = len(ctx.attr.src_urls) != 0 + if has_sources: + ctx.download( + output=ctx.path("jar/%s" % src_name), + url=ctx.attr.src_urls, + sha256=ctx.attr.src_sha256, + executable=False + ) + srcjar_attr ='\n srcjar = ":%s",' % src_name + + build_file_contents = """ +package(default_visibility = ['//visibility:public']) +java_import( + name = 'jar', + jars = ['{jar_name}'],{srcjar_attr} +) +filegroup( + name = 'file', + srcs = [ + '{jar_name}', + '{src_name}' + ], + visibility = ['//visibility:public'] +)\n""".format(jar_name = jar_name, src_name = src_name, srcjar_attr = srcjar_attr) + ctx.file(ctx.path("jar/BUILD"), build_file_contents, False) + return None + +jar_artifact = repository_rule( + attrs = { + "artifact": attr.string(mandatory = True), + "sha256": attr.string(mandatory = True), + "urls": attr.string_list(mandatory = True), + "src_sha256": attr.string(mandatory = False, default=""), + "src_urls": attr.string_list(mandatory = False, default=[]), + }, + implementation = _jar_artifact_impl +) + +def jar_artifact_callback(hash): + src_urls = [] + src_sha256 = "" + source=hash.get("source", None) + if source != None: + src_urls = [source["url"]] + src_sha256 = source["sha256"] + jar_artifact( + artifact = hash["artifact"], + name = hash["name"], + urls = [hash["url"]], + sha256 = hash["sha256"], + src_urls = src_urls, + src_sha256 = src_sha256 + ) + native.bind(name = hash["bind"], actual = hash["actual"]) + +def list_dependencies(): + return [ + ] + +def maven_dependencies(callback = jar_artifact_callback): + for hash in list_dependencies(): + callback(hash) From 48030adda03bbc3aab9c2c702439f4235db7750b Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Thu, 21 Mar 2019 10:58:06 +0800 Subject: [PATCH 095/115] Restore java -jar way to execute java use case --- .gitignore | 1 - java/BUILD.bazel | 5 ++++- java/test.sh | 6 ++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index a5c9754c27d6..21d8edfc4f96 100644 --- a/.gitignore +++ b/.gitignore @@ -148,7 +148,6 @@ java/**/lib java/**/.settings java/**/.classpath java/**/.project -java/third_party/workspace.bzl java/runtime/native_dependencies/ # python virtual env diff --git a/java/BUILD.bazel b/java/BUILD.bazel index a513fe44811a..33ed3cfcfa2b 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -21,11 +21,14 @@ java_import( "liborg_ray_ray_runtime-src.jar", "liborg_ray_ray_tutorial.jar", "liborg_ray_ray_tutorial-src.jar", + "all_tests_deploy.jar", + "all_tests_deploy-src.jar", ], deps = [ ":org_ray_ray_api", ":org_ray_ray_runtime", ":org_ray_ray_tutorial", + ":all_tests", ], ) @@ -102,7 +105,7 @@ define_java_module( ], ) -java_test( +java_binary( name = "all_tests", main_class = "org.testng.TestNG", data = ["testng.xml"], diff --git a/java/test.sh b/java/test.sh index 0902a983eb15..09fe314f11e4 100755 --- a/java/test.sh +++ b/java/test.sh @@ -12,7 +12,8 @@ echo "Linting Java code with checkstyle." bazel test //java:all --test_tag_filters="checkstyle" echo "Running tests under cluster mode." -ENABLE_MULTI_LANGUAGE_TESTS=1 bazel test //java:all_tests --test_output="errors" || cluster_exit_code=$? +# bazel test //java:all_tests --action_env=ENABLE_MULTI_LANGUAGE_TESTS=1 --test_output="errors" || cluster_exit_code=$? +ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml|| cluster_exit_code=$? # exit_code == 2 means there are some tests skiped. if [ $cluster_exit_code -ne 2 ] && [ $cluster_exit_code -ne 0 ] ; then @@ -20,7 +21,8 @@ if [ $cluster_exit_code -ne 2 ] && [ $cluster_exit_code -ne 0 ] ; then fi echo "Running tests under single-process mode." -bazel test //java:all_tests --jvmopt="-Dray.run-mode=SINGLE_PROCESS" --test_output="errors" || single_exit_code=$? +# bazel test //java:all_tests --jvmopt="-Dray.run-mode=SINGLE_PROCESS" --test_output="errors" || single_exit_code=$? +java -jar -Dray.run-mode="SINGLE_PROCESS" $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml || single_exit_code=$? # exit_code == 2 means there are some tests skiped. if [ $single_exit_code -ne 2 ] && [ $single_exit_code -ne 0 ] ; then From 520c40a82627d073fdfbd709f2b8535397baf298 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Thu, 21 Mar 2019 11:53:44 +0800 Subject: [PATCH 096/115] modify java/third_party/workspace.bzl --- java/third_party/workspace.bzl | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/java/third_party/workspace.bzl b/java/third_party/workspace.bzl index 4aa7c40b298f..7c5c7d0d09f3 100644 --- a/java/third_party/workspace.bzl +++ b/java/third_party/workspace.bzl @@ -66,6 +66,52 @@ def jar_artifact_callback(hash): def list_dependencies(): return [ + {"artifact": "antlr:antlr:2.7.7", "lang": "java", "sha1": "83cd2cd674a217ade95a4bb83a8a14f351f48bd0", "sha256": "88fbda4b912596b9f56e8e12e580cc954bacfb51776ecfddd3e18fc1cf56dc4c", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/antlr/antlr/2.7.7/antlr-2.7.7.jar", "name": "antlr_antlr", "actual": "@antlr_antlr//jar", "bind": "jar/antlr/antlr"}, + {"artifact": "com.beust:jcommander:1.72", "lang": "java", "sha1": "6375e521c1e11d6563d4f25a07ce124ccf8cd171", "sha256": "e0de160b129b2414087e01fe845609cd55caec6820cfd4d0c90fabcc7bdb8c1e", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/beust/jcommander/1.72/jcommander-1.72.jar", "source": {"sha1": "7ef123d5dfb6f839b41265648ff1be34982d50f8", "sha256": "9b4fddc257d3572696333f48158802243141ce46579d92f0a4c0b2c14956b49d", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/beust/jcommander/1.72/jcommander-1.72-sources.jar"} , "name": "com_beust_jcommander", "actual": "@com_beust_jcommander//jar", "bind": "jar/com/beust/jcommander"}, + {"artifact": "com.fasterxml.jackson.core:jackson-core:2.5.3", "lang": "java", "sha1": "a8b8a6dfc8a17890e4c7ff8aed810763d265b68b", "sha256": "c9a900c30c702d371be2d3714938dfa6de616599c8eb39f8e3cf7aedd62dfd32", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/fasterxml/jackson/core/jackson-core/2.5.3/jackson-core-2.5.3.jar", "source": {"sha1": "8bfff00c810764fd4e0fc78959b45f8734dd0c3e", "sha256": "5fcf1bf0f37c24a136fe72f113ed9218560967d9b1483e8adf3bc49d0459a522", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/fasterxml/jackson/core/jackson-core/2.5.3/jackson-core-2.5.3-sources.jar"} , "name": "com_fasterxml_jackson_core_jackson_core", "actual": "@com_fasterxml_jackson_core_jackson_core//jar", "bind": "jar/com/fasterxml/jackson/core/jackson_core"}, + {"artifact": "com.github.davidmoten:flatbuffers-java:1.9.0.1", "lang": "java", "sha1": "0eaa1b27095bb5127879458bc2ee1b78e00d4a20", "sha256": "7432b0380f0ee3e1d018305fd812fdcde95222107dbaade6a7b49010d6de3020", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/github/davidmoten/flatbuffers-java/1.9.0.1/flatbuffers-java-1.9.0.1.jar", "source": {"sha1": "e5e796e7f1c2c84b5fa4fbff0ee9b40bde9c3870", "sha256": "e0e36c11c9639fda34e849ebcb828c3e40adf842c12cfe90dc8505d9ead69fc9", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/github/davidmoten/flatbuffers-java/1.9.0.1/flatbuffers-java-1.9.0.1-sources.jar"} , "name": "com_github_davidmoten_flatbuffers_java", "actual": "@com_github_davidmoten_flatbuffers_java//jar", "bind": "jar/com/github/davidmoten/flatbuffers_java"}, + {"artifact": "com.google.code.findbugs:jsr305:3.0.2", "lang": "java", "sha1": "25ea2e8b0c338a877313bd4672d3fe056ea78f0d", "sha256": "766ad2a0783f2687962c8ad74ceecc38a28b9f72a2d085ee438b7813e928d0c7", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar", "source": {"sha1": "b19b5927c2c25b6c70f093767041e641ae0b1b35", "sha256": "1c9e85e272d0708c6a591dc74828c71603053b48cc75ae83cce56912a2aa063b", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2-sources.jar"} , "name": "com_google_code_findbugs_jsr305", "actual": "@com_google_code_findbugs_jsr305//jar", "bind": "jar/com/google/code/findbugs/jsr305"}, + {"artifact": "com.google.errorprone:error_prone_annotations:2.2.0", "lang": "java", "sha1": "88e3c593e9b3586e1c6177f89267da6fc6986f0c", "sha256": "6ebd22ca1b9d8ec06d41de8d64e0596981d9607b42035f9ed374f9de271a481a", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/errorprone/error_prone_annotations/2.2.0/error_prone_annotations-2.2.0.jar", "source": {"sha1": "a8cd7823aa1dcd2fd6677c0c5988fdde9d1fb0a3", "sha256": "626adccd4894bee72c3f9a0384812240dcc1282fb37a87a3f6cb94924a089496", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/errorprone/error_prone_annotations/2.2.0/error_prone_annotations-2.2.0-sources.jar"} , "name": "com_google_errorprone_error_prone_annotations", "actual": "@com_google_errorprone_error_prone_annotations//jar", "bind": "jar/com/google/errorprone/error_prone_annotations"}, + {"artifact": "com.google.guava:failureaccess:1.0.1", "lang": "java", "sha1": "1dcf1de382a0bf95a3d8b0849546c88bac1292c9", "sha256": "a171ee4c734dd2da837e4b16be9df4661afab72a41adaf31eb84dfdaf936ca26", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/guava/failureaccess/1.0.1/failureaccess-1.0.1.jar", "source": {"sha1": "1d064e61aad6c51cc77f9b59dc2cccc78e792f5a", "sha256": "092346eebbb1657b51aa7485a246bf602bb464cc0b0e2e1c7e7201fadce1e98f", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/guava/failureaccess/1.0.1/failureaccess-1.0.1-sources.jar"} , "name": "com_google_guava_failureaccess", "actual": "@com_google_guava_failureaccess//jar", "bind": "jar/com/google/guava/failureaccess"}, + {"artifact": "com.google.guava:guava:27.0.1-jre", "lang": "java", "sha1": "bd41a290787b5301e63929676d792c507bbc00ae", "sha256": "e1c814fd04492a27c38e0317eabeaa1b3e950ec8010239e400fe90ad6c9107b4", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/guava/guava/27.0.1-jre/guava-27.0.1-jre.jar", "source": {"sha1": "cb5c1119df8d41a428013289b193eba3ccaf5f60", "sha256": "cba2e5680186062f42998b895a5e9a9ceccbaab94644ccc9f35bb73c2b2c7d8e", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/guava/guava/27.0.1-jre/guava-27.0.1-jre-sources.jar"} , "name": "com_google_guava_guava", "actual": "@com_google_guava_guava//jar", "bind": "jar/com/google/guava/guava"}, + {"artifact": "com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava", "lang": "java", "sha1": "b421526c5f297295adef1c886e5246c39d4ac629", "sha256": "b372a037d4230aa57fbeffdef30fd6123f9c0c2db85d0aced00c91b974f33f99", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar", "name": "com_google_guava_listenablefuture", "actual": "@com_google_guava_listenablefuture//jar", "bind": "jar/com/google/guava/listenablefuture"}, + {"artifact": "com.google.j2objc:j2objc-annotations:1.1", "lang": "java", "sha1": "ed28ded51a8b1c6b112568def5f4b455e6809019", "sha256": "2994a7eb78f2710bd3d3bfb639b2c94e219cedac0d4d084d516e78c16dddecf6", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar", "source": {"sha1": "1efdf5b737b02f9b72ebdec4f72c37ec411302ff", "sha256": "2cd9022a77151d0b574887635cdfcdf3b78155b602abc89d7f8e62aba55cfb4f", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1-sources.jar"} , "name": "com_google_j2objc_j2objc_annotations", "actual": "@com_google_j2objc_j2objc_annotations//jar", "bind": "jar/com/google/j2objc/j2objc_annotations"}, + {"artifact": "com.puppycrawl.tools:checkstyle:8.15", "lang": "java", "sha1": "8584d88c6aefcfb079adb8d102928b3eeb4de6ad", "sha256": "ac3602c4d50c3113b14614a6ac38ec03c63d9839e4316e057c4bb66d97183087", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/puppycrawl/tools/checkstyle/8.15/checkstyle-8.15.jar", "source": {"sha1": "1c1865dc6d20a1abc14ea7721fcbc78e2563d4ca", "sha256": "6745eda6450aa6e8609e56499fd56f515402a2ace3c7bd0345ec9b3f6eb8f6f6", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/puppycrawl/tools/checkstyle/8.15/checkstyle-8.15-sources.jar"} , "name": "com_puppycrawl_tools_checkstyle", "actual": "@com_puppycrawl_tools_checkstyle//jar", "bind": "jar/com/puppycrawl/tools/checkstyle"}, + {"artifact": "com.sun.xml.bind:jaxb-core:2.3.0", "lang": "java", "sha1": "d044c784e41d026778693fb44a8026c1fd9a7506", "sha256": "33061af8d2a07379d452f8f0fddcbbf518428dfd3b83d9e2d479c1948020795b", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/sun/xml/bind/jaxb-core/2.3.0/jaxb-core-2.3.0.jar", "source": {"sha1": "10bc9795ef8a09b28590efe9d0ecc06485820106", "sha256": "9a3650caf4af09105a9a4325cdabaac97b077f79bdeae9f0c3a94ce3123db01c", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/sun/xml/bind/jaxb-core/2.3.0/jaxb-core-2.3.0-sources.jar"} , "name": "com_sun_xml_bind_jaxb_core", "actual": "@com_sun_xml_bind_jaxb_core//jar", "bind": "jar/com/sun/xml/bind/jaxb_core"}, + {"artifact": "com.sun.xml.bind:jaxb-impl:2.3.0", "lang": "java", "sha1": "3a3c6a62719d967175b76b63925f1fb495f11437", "sha256": "edd691fc6ad7c7f3167e1a6833372367bedf6f4f2a4b8230df444a59bb3718b1", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/sun/xml/bind/jaxb-impl/2.3.0/jaxb-impl-2.3.0.jar", "source": {"sha1": "bb0a5a9f4e1116f2bc88fb7d38efd843704b65f8", "sha256": "8627a8a43b0ff84c61c698e3b94e3440e9b30439d0a7085a11677c75eda97ddc", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/sun/xml/bind/jaxb-impl/2.3.0/jaxb-impl-2.3.0-sources.jar"} , "name": "com_sun_xml_bind_jaxb_impl", "actual": "@com_sun_xml_bind_jaxb_impl//jar", "bind": "jar/com/sun/xml/bind/jaxb_impl"}, + {"artifact": "com.typesafe:config:1.3.2", "lang": "java", "sha1": "d6ac0ce079f114adce620f2360c92a70b2cb36dc", "sha256": "6563d1723f3300bf596f41e40bc03e54986108b5c45d0ac34ebc66d48c2e25a3", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/typesafe/config/1.3.2/config-1.3.2.jar", "source": {"sha1": "a22dda7d62b800b297c1c59de90d48992d8119b1", "sha256": "65995abd56d6aa99ee7f46e7cdaaaac2968554b16b26d38bf67e13706a12ca82", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/typesafe/config/1.3.2/config-1.3.2-sources.jar"} , "name": "com_typesafe_config", "actual": "@com_typesafe_config//jar", "bind": "jar/com/typesafe/config"}, + {"artifact": "commons-beanutils:commons-beanutils:1.9.3", "lang": "java", "sha1": "c845703de334ddc6b4b3cd26835458cb1cba1f3d", "sha256": "c058e39c7c64203d3a448f3adb588cb03d6378ed808485618f26e137f29dae73", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-beanutils/commons-beanutils/1.9.3/commons-beanutils-1.9.3.jar", "source": {"sha1": "9d70af7c5982bd09c33efa97ac98cee6363ba0f0", "sha256": "3315f25f3793c1bb2577b2d956f58f852c7386c73aff4dea450e419a80b00a41", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-beanutils/commons-beanutils/1.9.3/commons-beanutils-1.9.3-sources.jar"} , "name": "commons_beanutils_commons_beanutils", "actual": "@commons_beanutils_commons_beanutils//jar", "bind": "jar/commons_beanutils/commons_beanutils"}, + {"artifact": "commons-codec:commons-codec:1.4", "lang": "java", "sha1": "4216af16d38465bbab0f3dff8efa14204f7a399a", "sha256": "6aa4234c74f3a1035751a25822545867c8c3727125a642b6e049665d1863631b", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-codec/commons-codec/1.4/commons-codec-1.4.jar", "source": {"sha1": "5310da9f90e843883309e9e0bf5950faa79882a0", "sha256": "78e3cee598647e409ea8860c15c040d172fd5bc8e070cbad9ca1684c36e71753", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-codec/commons-codec/1.4/commons-codec-1.4-sources.jar"} , "name": "commons_codec_commons_codec", "actual": "@commons_codec_commons_codec//jar", "bind": "jar/commons_codec/commons_codec"}, + {"artifact": "commons-collections:commons-collections:3.2.2", "lang": "java", "sha1": "8ad72fe39fa8c91eaaf12aadb21e0c3661fe26d5", "sha256": "eeeae917917144a68a741d4c0dff66aa5c5c5fd85593ff217bced3fc8ca783b8", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar", "source": {"sha1": "78c50ebda5784937ca1615fc0e1d0cb35857d572", "sha256": "a5b5ee16a02edadf7fe637f250217c19878bc6134f15eb55635c48996f6fed1d", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2-sources.jar"} , "name": "commons_collections_commons_collections", "actual": "@commons_collections_commons_collections//jar", "bind": "jar/commons_collections/commons_collections"}, + {"artifact": "commons-io:commons-io:2.5", "lang": "java", "sha1": "2852e6e05fbb95076fc091f6d1780f1f8fe35e0f", "sha256": "a10418348d234968600ccb1d988efcbbd08716e1d96936ccc1880e7d22513474", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-io/commons-io/2.5/commons-io-2.5.jar", "source": {"sha1": "0caf033a4a7c37b4a8ff3ea084cba591539b0b69", "sha256": "3b69b518d9a844732e35509b79e499fca63a960ee4301b1c96dc32e87f3f60a1", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-io/commons-io/2.5/commons-io-2.5-sources.jar"} , "name": "commons_io_commons_io", "actual": "@commons_io_commons_io//jar", "bind": "jar/commons_io/commons_io"}, + {"artifact": "commons-logging:commons-logging:1.2", "lang": "java", "sha1": "4bfc12adfe4842bf07b657f0369c4cb522955686", "sha256": "daddea1ea0be0f56978ab3006b8ac92834afeefbd9b7e4e6316fca57df0fa636", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-logging/commons-logging/1.2/commons-logging-1.2.jar", "source": {"sha1": "ecf26c7507d67782a3bbd148d170b31dfad001aa", "sha256": "44347acfe5860461728e9cb33251e97345be36f8a0dfd5c5130c172559455f41", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-logging/commons-logging/1.2/commons-logging-1.2-sources.jar"} , "name": "commons_logging_commons_logging", "actual": "@commons_logging_commons_logging//jar", "bind": "jar/commons_logging/commons_logging"}, + {"artifact": "de.ruedigermoeller:fst:2.47", "lang": "java", "sha1": "281d390ebed24a3621d3053825affc13d0eead8b", "sha256": "8c871febc859cf72dbed1c6c1eebc5600c3d93c3e986feccf4623cc5086098cc", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/de/ruedigermoeller/fst/2.47/fst-2.47.jar", "source": {"sha1": "2e0bb6c02600b3c11c041685a29f6981f3c54c7a", "sha256": "d026fb853b6d5269ec06b9b430488c02c7ac1569fe0233ad785f8bfb7e09688b", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/de/ruedigermoeller/fst/2.47/fst-2.47-sources.jar"} , "name": "de_ruedigermoeller_fst", "actual": "@de_ruedigermoeller_fst//jar", "bind": "jar/de/ruedigermoeller/fst"}, + {"artifact": "info.picocli:picocli:3.8.0", "lang": "java", "sha1": "86d47318b178e9e1d8b600a7afd1b8bd36300d3c", "sha256": "2b615ea91a8487d630efb7d3b04cab334d30ca414575138ce94201d470450fc4", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/info/picocli/picocli/3.8.0/picocli-3.8.0.jar", "source": {"sha1": "6fbfd760f8ed0bc04ac7311c52878c25eed81176", "sha256": "a23d58cf6c90bbd5568dbd7bb519b7225377ec2330315da0c8c4e623fc3dbf05", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/info/picocli/picocli/3.8.0/picocli-3.8.0-sources.jar"} , "name": "info_picocli_picocli", "actual": "@info_picocli_picocli//jar", "bind": "jar/info/picocli/picocli"}, + {"artifact": "javax.xml.bind:jaxb-api:2.3.0", "lang": "java", "sha1": "99f802e0cb3e953ba3d6e698795c4aeb98d37c48", "sha256": "883007989d373d19f352ba9792b25dec21dc7d0e205a710a93a3815101bb3d03", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/javax/xml/bind/jaxb-api/2.3.0/jaxb-api-2.3.0.jar", "source": {"sha1": "431dc11c8e5d8b9f5149be5757ef8e170fc6824a", "sha256": "de09da7ffa14a4d6d35bde4452b3a88fafa1b8aa5aca0d5bad427649069e8b9f", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/javax/xml/bind/jaxb-api/2.3.0/jaxb-api-2.3.0-sources.jar"} , "name": "javax_xml_bind_jaxb_api", "actual": "@javax_xml_bind_jaxb_api//jar", "bind": "jar/javax/xml/bind/jaxb_api"}, + {"artifact": "junit:junit:4.10", "lang": "java", "sha1": "e4f1766ce7404a08f45d859fb9c226fc9e41a861", "sha256": "36a747ca1e0b86f6ea88055b8723bb87030d627766da6288bf077afdeeb0f75a", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/junit/junit/4.10/junit-4.10.jar", "source": {"sha1": "6c98d6766e72d5575f96c9479d1c1d3b865c6e25", "sha256": "e3a4cb7ac3343265f5663b68857078ae68787450afc6e72dc6826962a1bf5212", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/junit/junit/4.10/junit-4.10-sources.jar"} , "name": "junit_junit", "actual": "@junit_junit//jar", "bind": "jar/junit/junit"}, + {"artifact": "log4j:log4j:1.2.17", "lang": "java", "sha1": "5af35056b4d257e4b64b9e8069c0746e8b08629f", "sha256": "1d31696445697720527091754369082a6651bd49781b6005deb94e56753406f9", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/log4j/log4j/1.2.17/log4j-1.2.17.jar", "source": {"sha1": "677abe279b68c5e7490d6d50c6951376238d7d3e", "sha256": "4d9ba787af1692aa88417c2a47a37a98125d645b91ab556252dbee0f45225493", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/log4j/log4j/1.2.17/log4j-1.2.17-sources.jar"} , "name": "log4j_log4j", "actual": "@log4j_log4j//jar", "bind": "jar/log4j/log4j"}, + {"artifact": "net.java.dev.jna:jna:4.1.0", "lang": "java", "sha1": "1c12d070e602efd8021891cdd7fd18bc129372d4", "sha256": "1aa37e9ea6baa0ee152d89509f758f0847eac66ec179b955cafe0919e540a92e", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/net/java/dev/jna/jna/4.1.0/jna-4.1.0.jar", "source": {"sha1": "28a5d592b494fe54c120c64689087e779a7bf55b", "sha256": "a7d37342b0447865934bc74c19be90565e3b10a0c0251f1652647e9437533414", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/net/java/dev/jna/jna/4.1.0/jna-4.1.0-sources.jar"} , "name": "net_java_dev_jna_jna", "actual": "@net_java_dev_jna_jna//jar", "bind": "jar/net/java/dev/jna/jna"}, + {"artifact": "net.lingala.zip4j:zip4j:1.3.2", "lang": "java", "sha1": "4ba84e98ee017b74cb52f45962f929a221f3074c", "sha256": "c67098d430c574311432728ebd4c7c45672f9ccf5c64702eb6afb8816c22ad08", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/net/lingala/zip4j/zip4j/1.3.2/zip4j-1.3.2.jar", "source": {"sha1": "b2a62723195cf14964e7937f60da7c4daa0f61be", "sha256": "a55009fecaf47d8c4b01a30823de8d8ede3b4df6cf388e348687e0d2a9b0a143", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/net/lingala/zip4j/zip4j/1.3.2/zip4j-1.3.2-sources.jar"} , "name": "net_lingala_zip4j_zip4j", "actual": "@net_lingala_zip4j_zip4j//jar", "bind": "jar/net/lingala/zip4j/zip4j"}, + {"artifact": "net.sf.saxon:Saxon-HE:9.9.0-2", "lang": "java", "sha1": "c3437344acc4d2f1b5d1a638166e6ce52b300595", "sha256": "34661e9c6dd69caff14ec31e3ec47a9e7dbbc08be9da2c6326daf4e0038ed97c", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/net/sf/saxon/Saxon-HE/9.9.0-2/Saxon-HE-9.9.0-2.jar", "source": {"sha1": "294ea53dfb38f001372deee809c375e8736f16f4", "sha256": "b229a09b43c67690eeba0344840cd3a053bb1946ebcd668b9764ef454ed7d1cf", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/net/sf/saxon/Saxon-HE/9.9.0-2/Saxon-HE-9.9.0-2-sources.jar"} , "name": "net_sf_saxon_Saxon_HE", "actual": "@net_sf_saxon_Saxon_HE//jar", "bind": "jar/net/sf/saxon/Saxon_HE"}, + {"artifact": "org.antlr:antlr4-runtime:4.7.1", "lang": "java", "sha1": "946f8aa9daa917dd81a8b818111bec7e288f821a", "sha256": "43516d19beae35909e04d06af6c0c58c17bc94e0070c85e8dc9929ca640dc91d", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/antlr/antlr4-runtime/4.7.1/antlr4-runtime-4.7.1.jar", "source": {"sha1": "1e68e18aa14f3229b95820d354a594846134af38", "sha256": "a33d52d0d64e68c60d5e3ae2c1098fe7200d57cff59032c19930fd9d487fc7d4", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/antlr/antlr4-runtime/4.7.1/antlr4-runtime-4.7.1-sources.jar"} , "name": "org_antlr_antlr4_runtime", "actual": "@org_antlr_antlr4_runtime//jar", "bind": "jar/org/antlr/antlr4_runtime"}, + {"artifact": "org.apache.ant:ant-launcher:1.7.0", "lang": "java", "sha1": "e7e30789211e074aa70ef3eaea59bd5b22a7fa7a", "sha256": "72b3d03e0d7d86a56513ec38dd4cd6abe3da6620189be222ab255352cb6eba4a", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/apache/ant/ant-launcher/1.7.0/ant-launcher-1.7.0.jar", "name": "org_apache_ant_ant_launcher", "actual": "@org_apache_ant_ant_launcher//jar", "bind": "jar/org/apache/ant/ant_launcher"}, + {"artifact": "org.apache.ant:ant:1.7.0", "lang": "java", "sha1": "9746af1a485e50cf18dcb232489032a847067066", "sha256": "92f72307e7440f1e352c916f2438d2bbab3ffd2cf730c71316117ad04abadea8", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/apache/ant/ant/1.7.0/ant-1.7.0.jar", "source": {"sha1": "58ec0e378f1d33cb72a92ee3a9442909789090db", "sha256": "81e4de3e9e9d254b010809d679f902e7703eb4056097641861c07687e3f1fecf", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/apache/ant/ant/1.7.0/ant-1.7.0-sources.jar"} , "name": "org_apache_ant_ant", "actual": "@org_apache_ant_ant//jar", "bind": "jar/org/apache/ant/ant"}, + {"artifact": "org.apache.commons:commons-lang3:3.4", "lang": "java", "sha1": "5fe28b9518e58819180a43a850fbc0dd24b7c050", "sha256": "734c8356420cc8e30c795d64fd1fcd5d44ea9d90342a2cc3262c5158fbc6d98b", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/apache/commons/commons-lang3/3.4/commons-lang3-3.4.jar", "source": {"sha1": "b49dafc9cfef24c356827f322e773e7c26725dd2", "sha256": "4709f16a9e0f8fd83ae155083d63044e23045aac8f6f0183a2db09f492491b12", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/apache/commons/commons-lang3/3.4/commons-lang3-3.4-sources.jar"} , "name": "org_apache_commons_commons_lang3", "actual": "@org_apache_commons_commons_lang3//jar", "bind": "jar/org/apache/commons/commons_lang3"}, + {"artifact": "org.apache.commons:commons-pool2:2.3", "lang": "java", "sha1": "62a559a025fd890c30364296ece14643ba9c8c5b", "sha256": "3e794e75be78f63cc31b6676697e45f07085903a1039169459132513d85c62ae", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/apache/commons/commons-pool2/2.3/commons-pool2-2.3.jar", "source": {"sha1": "9e03327e766c3d16ed34a617818ab082192e83a7", "sha256": "3b75f7ce4fa929928b4a6e48d23779ce6065e0965c1fb934b5a6c5a54c8e84ea", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/apache/commons/commons-pool2/2.3/commons-pool2-2.3-sources.jar"} , "name": "org_apache_commons_commons_pool2", "actual": "@org_apache_commons_commons_pool2//jar", "bind": "jar/org/apache/commons/commons_pool2"}, + {"artifact": "org.beanshell:bsh:2.0b4", "lang": "java", "sha1": "a05f0a0feefa8d8467ac80e16e7de071489f0d9c", "sha256": "91395c07885839a8c6986d5b7c577cd9bacf01bf129c89141f35e8ea858427b6", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/beanshell/bsh/2.0b4/bsh-2.0b4.jar", "name": "org_beanshell_bsh", "actual": "@org_beanshell_bsh//jar", "bind": "jar/org/beanshell/bsh"}, + {"artifact": "org.checkerframework:checker-qual:2.5.2", "lang": "java", "sha1": "cea74543d5904a30861a61b4643a5f2bb372efc4", "sha256": "64b02691c8b9d4e7700f8ee2e742dce7ea2c6e81e662b7522c9ee3bf568c040a", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar", "source": {"sha1": "ebb8ebccd42218434674f3e1d9022c13df1c19f8", "sha256": "821c5c63a6f156a3bb498c5bbb613580d9d8f4134131a5627d330fc4018669d2", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2-sources.jar"} , "name": "org_checkerframework_checker_qual", "actual": "@org_checkerframework_checker_qual//jar", "bind": "jar/org/checkerframework/checker_qual"}, + {"artifact": "org.codehaus.mojo:animal-sniffer-annotations:1.17", "lang": "java", "sha1": "f97ce6decaea32b36101e37979f8b647f00681fb", "sha256": "92654f493ecfec52082e76354f0ebf87648dc3d5cec2e3c3cdb947c016747a53", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar", "source": {"sha1": "8fb5b5ad9c9723951b9fccaba5bb657fa6064868", "sha256": "2571474a676f775a8cdd15fb9b1da20c4c121ed7f42a5d93fca0e7b6e2015b40", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17-sources.jar"} , "name": "org_codehaus_mojo_animal_sniffer_annotations", "actual": "@org_codehaus_mojo_animal_sniffer_annotations//jar", "bind": "jar/org/codehaus/mojo/animal_sniffer_annotations"}, + {"artifact": "org.hamcrest:hamcrest-core:1.1", "lang": "java", "sha1": "860340562250678d1a344907ac75754e259cdb14", "sha256": "0361d1493ff0d94f861359efea91200720635072134792efb7217f6e09d5cffb", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/hamcrest/hamcrest-core/1.1/hamcrest-core-1.1.jar", "source": {"sha1": "2ccf1154d1a8936042a8a742dc3e611d02ac7213", "sha256": "bcc20dca5b6f9bd6a6f7611b4d5734a75f5e4ede272ed3e1b156669d44e493d6", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/hamcrest/hamcrest-core/1.1/hamcrest-core-1.1-sources.jar"} , "name": "org_hamcrest_hamcrest_core", "actual": "@org_hamcrest_hamcrest_core//jar", "bind": "jar/org/hamcrest/hamcrest_core"}, + {"artifact": "org.ini4j:ini4j:0.5.2", "lang": "java", "sha1": "16561cb11c221b5928119e10d7636c95ee5c960d", "sha256": "631656eb38639b0ae41161f706ff7fbe04313b5b8f42892da5ec656390031fc6", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/ini4j/ini4j/0.5.2/ini4j-0.5.2.jar", "source": {"sha1": "17a383f897ae0ec8fca2f3effc2a2b8dbb336fc2", "sha256": "10f11429b7bfc298ed776e6d1ed8a9365f73a1df90737cd178169cd5b7849ba7", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/ini4j/ini4j/0.5.2/ini4j-0.5.2-sources.jar"} , "name": "org_ini4j_ini4j", "actual": "@org_ini4j_ini4j//jar", "bind": "jar/org/ini4j/ini4j"}, + {"artifact": "org.javassist:javassist:3.19.0-GA", "lang": "java", "sha1": "50120f69224dd8684b445a6f3a5b08fe9b5c60f6", "sha256": "d19c1ef43ccd9cb1b39466bb2f1c8e45c2b6752f1e13a3dfb60096543d1791fa", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/javassist/javassist/3.19.0-GA/javassist-3.19.0-GA.jar", "source": {"sha1": "37e6508aeea3eef21df5b7ee3ea3c708618c1b7b", "sha256": "26690d98e9b005eecebf0eba2b7331b8f7cb9f3d3d9e41975a41574cfe83a656", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/javassist/javassist/3.19.0-GA/javassist-3.19.0-GA-sources.jar"} , "name": "org_javassist_javassist", "actual": "@org_javassist_javassist//jar", "bind": "jar/org/javassist/javassist"}, + {"artifact": "org.mockito:mockito-all:1.10.19", "lang": "java", "sha1": "539df70269cc254a58cccc5d8e43286b4a73bf30", "sha256": "d1a7a7ef14b3db5c0fc3e0a63a81b374b510afe85add9f7984b97911f4c70605", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar", "source": {"sha1": "8269667b73d9616600359a9b0ba1b1c7d0cf7a97", "sha256": "6c8680afe9cfdb6e0fe40aca1e27ded721252bd53107390f166f2077b9e5b16e", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/mockito/mockito-all/1.10.19/mockito-all-1.10.19-sources.jar"} , "name": "org_mockito_mockito_all", "actual": "@org_mockito_mockito_all//jar", "bind": "jar/org/mockito/mockito_all"}, + {"artifact": "org.objenesis:objenesis:2.4", "lang": "java", "sha1": "2916b6c96b50c5b3ec4452ed99401db745aabb27", "sha256": "090416e160a7e398f79e0149a7b05ef328b1843898a34e08c1cc170128d326d0", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/objenesis/objenesis/2.4/objenesis-2.4.jar", "source": {"sha1": "19b316d8da7da3ae2539ee4ee2cf73a1e127fa73", "sha256": "f5d6ea639bb9d1e7260447ad9a7bb9e14e16a7c98912db3c401c8468a9382871", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/objenesis/objenesis/2.4/objenesis-2.4-sources.jar"} , "name": "org_objenesis_objenesis", "actual": "@org_objenesis_objenesis//jar", "bind": "jar/org/objenesis/objenesis"}, + {"artifact": "org.ow2.asm:asm:6.0", "lang": "java", "sha1": "bc6fa6b19424bb9592fe43bbc20178f92d403105", "sha256": "dd8971c74a4e697899a8e95caae4ea8760ea6c486dc6b97b1795e75760420461", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/ow2/asm/asm/6.0/asm-6.0.jar", "source": {"sha1": "066e3ec9f4ff0d70713868456142040111cc201c", "sha256": "0687933fe7b7989ec5110946b5792a01784011fe7cbfe555f7e416da7984a1ca", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/ow2/asm/asm/6.0/asm-6.0-sources.jar"} , "name": "org_ow2_asm_asm", "actual": "@org_ow2_asm_asm//jar", "bind": "jar/org/ow2/asm/asm"}, + {"artifact": "org.slf4j:slf4j-api:1.7.25", "lang": "java", "sha1": "da76ca59f6a57ee3102f8f9bd9cee742973efa8a", "sha256": "18c4a0095d5c1da6b817592e767bb23d29dd2f560ad74df75ff3961dbde25b79", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar", "source": {"sha1": "962153db4a9ea71b79d047dfd1b2a0d80d8f4739", "sha256": "c4bc93180a4f0aceec3b057a2514abe04a79f06c174bbed910a2afb227b79366", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25-sources.jar"} , "name": "org_slf4j_slf4j_api", "actual": "@org_slf4j_slf4j_api//jar", "bind": "jar/org/slf4j/slf4j_api"}, + {"artifact": "org.slf4j:slf4j-log4j12:1.7.25", "lang": "java", "sha1": "110cefe2df103412849d72ef7a67e4e91e4266b4", "sha256": "ddb343954deb6f046f862606c534178730c02ed23d0b7f6ca1012c1e3fa74273", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/slf4j/slf4j-log4j12/1.7.25/slf4j-log4j12-1.7.25.jar", "source": {"sha1": "d6f907c4254a49f40fa46005b65bc97261ad9e46", "sha256": "3c57ac91d961ab2b68b5246275ddc8d6fc700eadc8deb113b2d353176112e0c8", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/slf4j/slf4j-log4j12/1.7.25/slf4j-log4j12-1.7.25-sources.jar"} , "name": "org_slf4j_slf4j_log4j12", "actual": "@org_slf4j_slf4j_log4j12//jar", "bind": "jar/org/slf4j/slf4j_log4j12"}, + {"artifact": "org.testng:testng:6.9.9", "lang": "java", "sha1": "1bf509349476d6a48978cc2b04af9caa907781ab", "sha256": "e89d524ca6dba6ea6c9619c2537f4669524eb9306bb6fd9b7a52240cb3b4b690", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/testng/testng/6.9.9/testng-6.9.9.jar", "source": {"sha1": "045c07e00b4cea80ac8715deb749a54d784f7cb6", "sha256": "304face92c85dcf3418be273713499cadc8a10c2735852496f69a0e9faf55b57", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/testng/testng/6.9.9/testng-6.9.9-sources.jar"} , "name": "org_testng_testng", "actual": "@org_testng_testng//jar", "bind": "jar/org/testng/testng"}, + {"artifact": "org.yaml:snakeyaml:1.15", "lang": "java", "sha1": "3b132bea69e8ee099f416044970997bde80f4ea6", "sha256": "79ea8aac6590f49ee8390c2f17ed9343079e85b44158a097b301dfee42af86ec", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/yaml/snakeyaml/1.15/snakeyaml-1.15.jar", "source": {"sha1": "7dc55271cad1cf73bb495e372164269e75d9345a", "sha256": "cd92585a477bf81e3c8f2bcbfff182dd67999fdc8fd880c3a1531c0626c7a4ee", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/yaml/snakeyaml/1.15/snakeyaml-1.15-sources.jar"} , "name": "org_yaml_snakeyaml", "actual": "@org_yaml_snakeyaml//jar", "bind": "jar/org/yaml/snakeyaml"}, + {"artifact": "redis.clients:jedis:2.8.0", "lang": "java", "sha1": "a9486e200b2195f31ce9b8779a70a5e2450a73ba", "sha256": "e8194b42cbdbe990f6cc1ec7a91a6b553d901abaf155dd6cc599139eb3f5d68d", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/redis/clients/jedis/2.8.0/jedis-2.8.0.jar", "source": {"sha1": "356324b58a8d6dfaeb6ddd6289f3a972551c915b", "sha256": "c71f6d176af72617a18bb685f4a658fa0ea6f35f872d9d073b29792e842541ca", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/redis/clients/jedis/2.8.0/jedis-2.8.0-sources.jar"} , "name": "redis_clients_jedis", "actual": "@redis_clients_jedis//jar", "bind": "jar/redis/clients/jedis"}, ] def maven_dependencies(callback = jar_artifact_callback): From 27150f36d13112345790ed674838fc1a1ea08d79 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Thu, 21 Mar 2019 11:57:55 +0800 Subject: [PATCH 097/115] Update test.sh --- java/test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/java/test.sh b/java/test.sh index 09fe314f11e4..716d05970009 100755 --- a/java/test.sh +++ b/java/test.sh @@ -12,6 +12,8 @@ echo "Linting Java code with checkstyle." bazel test //java:all --test_tag_filters="checkstyle" echo "Running tests under cluster mode." +# TODO(hchen): Ideally, we should use the following bazel command to run Java tests. However, if there're skipped tests, +# TestNG will exit with code 2. And bazel treats it as test failure. # bazel test //java:all_tests --action_env=ENABLE_MULTI_LANGUAGE_TESTS=1 --test_output="errors" || cluster_exit_code=$? ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml|| cluster_exit_code=$? From 3b900e5dcdd4a6d4b6d1c18dd3ba92d72375904a Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Thu, 21 Mar 2019 13:55:31 +0800 Subject: [PATCH 098/115] update checkstyle_java github's address --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 8155cba45b47..79adb90f109d 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -8,7 +8,7 @@ java_repositories() git_repository( name = "com_github_checkstyle_java", commit = "85f37871ca03b9d3fee63c69c8107f167e24e77b", - remote = "https://github.com/alegithub111/checkstyle_java.git", + remote = "https://github.com/ruifangChen/checkstyle_java", ) git_repository( From 347b147a877d1c3f65c2136c405e82e3bbfe2260 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Thu, 21 Mar 2019 14:56:16 +0800 Subject: [PATCH 099/115] add deps to org_ray_ray_test --- java/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 33ed3cfcfa2b..2bc515244869 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -102,6 +102,7 @@ define_java_module( "@org_testng_testng//jar", "@commons_collections_commons_collections//jar", "@com_google_guava_guava//jar", + "@commons_io_commons_io//jar", ], ) From f0e36562d096bfb40831d9d3be1ff97fb5f55e67 Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Thu, 21 Mar 2019 16:27:32 +0800 Subject: [PATCH 100/115] add 'cat hs_err_*' to .travis.yml --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 242e21e8c76d..f428e21fb6f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -223,3 +223,5 @@ deploy: on: repo: ray-project/ray condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 + +cat hs_err_* From 606a876aa571b003e394495c2d5956ba07690ec9 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Thu, 21 Mar 2019 18:42:57 +0800 Subject: [PATCH 101/115] Revert "add 'cat hs_err_*' to .travis.yml" This reverts commit f0e36562d096bfb40831d9d3be1ff97fb5f55e67. --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f428e21fb6f5..242e21e8c76d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -223,5 +223,3 @@ deploy: on: repo: ray-project/ray condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 - -cat hs_err_* From abccca94257f3c8d8d843a809454236662313755 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Thu, 21 Mar 2019 18:45:39 +0800 Subject: [PATCH 102/115] Fix FunctionManagerTest --- .../functionmanager/FunctionManager.java | 29 +++--- .../java/org/ray/runtime/util/JarLoader.java | 92 ------------------- .../functionmanager/FunctionManagerTest.java | 48 ++++------ 3 files changed, 37 insertions(+), 132 deletions(-) delete mode 100644 java/runtime/src/main/java/org/ray/runtime/util/JarLoader.java diff --git a/java/runtime/src/main/java/org/ray/runtime/functionmanager/FunctionManager.java b/java/runtime/src/main/java/org/ray/runtime/functionmanager/FunctionManager.java index 7c267d0eae0d..8ae245f8357e 100644 --- a/java/runtime/src/main/java/org/ray/runtime/functionmanager/FunctionManager.java +++ b/java/runtime/src/main/java/org/ray/runtime/functionmanager/FunctionManager.java @@ -1,9 +1,14 @@ package org.ray.runtime.functionmanager; +import com.google.common.base.Strings; +import java.io.File; import java.lang.invoke.SerializedLambda; import java.lang.reflect.Constructor; import java.lang.reflect.Executable; import java.lang.reflect.Method; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -15,7 +20,6 @@ import org.objectweb.asm.Type; import org.ray.api.function.RayFunc; import org.ray.api.id.UniqueId; -import org.ray.runtime.util.JarLoader; import org.ray.runtime.util.LambdaUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,8 +53,8 @@ public class FunctionManager { /** * Construct a FunctionManager with the specified driver resource path. * - * @param driverResourcePath The specified driver resource that - * can store the driver's resources. + * @param driverResourcePath The specified driver resource that can store the driver's + * resources. */ public FunctionManager(String driverResourcePath) { this.driverResourcePath = driverResourcePath; @@ -71,7 +75,7 @@ public RayFunction getFunction(UniqueId driverId, RayFunc func) { final String methodName = serializedLambda.getImplMethodName(); final String typeDescriptor = serializedLambda.getImplMethodSignature(); functionDescriptor = new JavaFunctionDescriptor(className, methodName, typeDescriptor); - RAY_FUNC_CACHE.get().put(func.getClass(),functionDescriptor); + RAY_FUNC_CACHE.get().put(func.getClass(), functionDescriptor); } return getFunction(driverId, functionDescriptor); } @@ -86,15 +90,18 @@ public RayFunction getFunction(UniqueId driverId, RayFunc func) { public RayFunction getFunction(UniqueId driverId, JavaFunctionDescriptor functionDescriptor) { DriverFunctionTable driverFunctionTable = driverFunctionTables.get(driverId); if (driverFunctionTable == null) { - String resourcePath = driverResourcePath + "/" + driverId.toString() + "/"; ClassLoader classLoader; - - if (driverResourcePath != null && !driverResourcePath.isEmpty()) { - classLoader = JarLoader.loadJars(resourcePath, false); - LOGGER.info("Succeeded to load driver({}) resource. Resource path is {}", - driverId, resourcePath); - } else { + if (Strings.isNullOrEmpty(driverResourcePath)) { classLoader = getClass().getClassLoader(); + } else { + File resourceDir = new File(driverResourcePath + "/" + driverId.toString() + "/"); + try { + classLoader = new URLClassLoader(new URL[]{resourceDir.toURI().toURL()}); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + LOGGER.debug("Resource loaded for driver {} from path {}.", driverId, + resourceDir.getAbsolutePath()); } driverFunctionTable = new DriverFunctionTable(classLoader); diff --git a/java/runtime/src/main/java/org/ray/runtime/util/JarLoader.java b/java/runtime/src/main/java/org/ray/runtime/util/JarLoader.java deleted file mode 100644 index c6ab5650c038..000000000000 --- a/java/runtime/src/main/java/org/ray/runtime/util/JarLoader.java +++ /dev/null @@ -1,92 +0,0 @@ -package org.ray.runtime.util; - -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Enumeration; -import java.util.List; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.commons.io.filefilter.DirectoryFileFilter; -import org.apache.commons.io.filefilter.RegexFileFilter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * load and unload jars from a dir. - */ -public class JarLoader { - - private static final Logger LOGGER = LoggerFactory.getLogger(JarLoader.class); - - public static URLClassLoader loadJars(String dir, boolean explicitLoad) { - // get all jars - Collection jars = FileUtils.listFiles( - new File(dir), - new RegexFileFilter(".*\\.jar"), - DirectoryFileFilter.DIRECTORY - ); - return loadJar(jars, explicitLoad); - } - - public static void unloadJars(ClassLoader loader) { - // now do nothing, if no ref to the loader and loader's class. - // they would be gc. - } - - private static URLClassLoader loadJar(Collection appJars, boolean explicitLoad) { - List jars = new ArrayList<>(); - List urls = new ArrayList<>(); - - for (File appJar : appJars) { - try { - LOGGER.info("succeeded to load jar {}.", appJar.getAbsolutePath()); - JarFile jar = new JarFile(appJar.getAbsolutePath()); - jars.add(jar); - urls.add(appJar.toURI().toURL()); - } catch (IOException e) { - throw new RuntimeException( - "invalid app jar path: " + appJar.getAbsolutePath() + ", load failed with exception", - e); - } - } - - URLClassLoader cl = URLClassLoader.newInstance(urls.toArray(new URL[urls.size()])); - - if (!explicitLoad) { - return cl; - } - for (JarFile jar : jars) { - try { - Enumeration e = jar.entries(); - while (e.hasMoreElements()) { - JarEntry je = e.nextElement(); - if (je.isDirectory() || !je.getName().endsWith(".class")) { - continue; - } - - String className = classNameOfJarEntry(je); - className = className.replace('/', '.'); - try { - Class.forName(className, true, cl); - } catch (ClassNotFoundException e1) { - e1.printStackTrace(); - } - } - } finally { - IOUtils.closeQuietly(jar); - } - } - return cl; - } - - private static String classNameOfJarEntry(JarEntry je) { - return je.getName().substring(0, je.getName().length() - ".class".length()); - } - -} diff --git a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java index 5fdd4a739b07..6641abc6c4d7 100644 --- a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java +++ b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java @@ -119,11 +119,13 @@ public void testLoadFunctionTableForClass() { @Test public void testGetFunctionFromLocalResource() throws Exception { - final String resourcePath = "/tmp/ray/java_test/resource"; UniqueId driverId = UniqueId.randomId(); + final String resourcePath = FileUtils.getTempDirectoryPath() + "/ray_test_resources"; final String driverResourcePath = resourcePath + "/" + driverId.toString(); File driverResourceDir = new File(driverResourcePath); + FileUtils.deleteQuietly(driverResourceDir); driverResourceDir.mkdirs(); + driverResourceDir.deleteOnExit(); String demoJavaFile = ""; demoJavaFile += "public class DemoApp {\n"; @@ -132,35 +134,23 @@ public void testGetFunctionFromLocalResource() throws Exception { demoJavaFile += " }\n"; demoJavaFile += "}"; - try { - // Write the demo java file to the driver resource path. - String javaFilePath = driverResourcePath + "/DemoApp.java"; - Files.write(Paths.get(javaFilePath), demoJavaFile.getBytes()); - - // Compile the java file. - JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - int result = compiler.run(null, null, null, "-d", driverResourcePath, javaFilePath); - if (result != 0) { - throw new RuntimeException("Couldn't compile Demo.java."); - } - // Package the class file into a jar file. - String[] packJarCommand = new String[]{ - "jar", - "-cvf", - driverResourcePath + "/DemoApp.jar", - "DemoApp.class" - }; - new ProcessBuilder(packJarCommand).directory(driverResourceDir).start().waitFor(); - - // Test loading the function. - JavaFunctionDescriptor descriptor = new JavaFunctionDescriptor( - "DemoApp", "hello", "()Ljava/lang/String;"); - final FunctionManager functionManager = new FunctionManager(resourcePath); - RayFunction func = functionManager.getFunction(driverId, descriptor); - Assert.assertEquals(func.getFunctionDescriptor(), descriptor); - } finally { - FileUtils.deleteDirectory(driverResourceDir); + // Write the demo java file to the driver resource path. + String javaFilePath = driverResourcePath + "/DemoApp.java"; + Files.write(Paths.get(javaFilePath), demoJavaFile.getBytes()); + + // Compile the java file. + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + int result = compiler.run(null, null, null, "-d", driverResourcePath, javaFilePath); + if (result != 0) { + throw new RuntimeException("Couldn't compile Demo.java."); } + + // Test loading the function. + JavaFunctionDescriptor descriptor = new JavaFunctionDescriptor( + "DemoApp", "hello", "()Ljava/lang/String;"); + final FunctionManager functionManager = new FunctionManager(resourcePath); + RayFunction func = functionManager.getFunction(driverId, descriptor); + Assert.assertEquals(func.getFunctionDescriptor(), descriptor); } } From b504a22e081acc8a15c55acc26e4ad727051c14e Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Thu, 21 Mar 2019 18:56:21 +0800 Subject: [PATCH 103/115] fix generate_java_gcs_fbs --- java/BUILD.bazel | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 2bc515244869..529a16a6169b 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -45,9 +45,6 @@ define_java_module( define_java_module( name = "runtime", - additional_srcs = [ - ":generate_java_gcs_fbs", - ], additional_resources = [ ":java_native_deps", ], @@ -61,6 +58,7 @@ define_java_module( "@commons_io_commons_io//jar", ], deps = [ + ":copy_java_native_deps", ":org_ray_ray_api", "@plasma//:org_apache_arrow_arrow_plasma", "@org_slf4j_slf4j_api//jar", @@ -161,14 +159,19 @@ genrule( name = "generate_java_gcs_fbs", srcs = [":java_gcs_fbs"], outs = [ - "runtime/src/main/java/org/ray/runtime/generated/" + file for file in flatbuffers_generated_files + "generate_java_gcs_fbs.out" ], cmd = """ + WORK_DIR=$$(pwd) + TARGET_DIR=$$WORK_DIR/java/runtime/src/main/java/org/ray/runtime/generated + rm -rf $$TARGET_DIR + mkdir -p $$TARGET_DIR for f in $(locations //java:java_gcs_fbs); do chmod +w $$f - cp -f $$f $(@D)/runtime/src/main/java/org/ray/runtime/generated + cp $$f $$TARGET_DIR done - python $$(pwd)/java/modify_generated_java_flatbuffers_files.py $(@D)/.. + python $$WORK_DIR/java/modify_generated_java_flatbuffers_files.py $$WORK_DIR + echo $$(date) > $@ """, local = 1, ) From cd0f583dcab462ca575642f9bc2d37744496db50 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Thu, 21 Mar 2019 19:18:03 +0800 Subject: [PATCH 104/115] Supress output --- .travis.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 242e21e8c76d..c904a58a9e36 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,13 +29,12 @@ matrix: install: - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - if [ $RAY_CI_JAVA_AFFECTED != "1" ]; then exit; fi - - ./ci/travis/install-bazel.sh - - ./ci/travis/install-dependencies.sh + - ./ci/suppress_output ./ci/travis/install-bazel.sh + - ./ci/suppress_output ./ci/travis/install-dependencies.sh - export PATH="$HOME/miniconda/bin:$PATH" - - ./ci/travis/install-ray.sh + - ./ci/suppress_output ./ci/travis/install-ray.sh script: - if [ $RAY_CI_JAVA_AFFECTED != "1" ]; then exit; fi - - ./java/test.sh - os: linux From 6477aca2fd66dcb2f6315af6eb759583bc01c4e0 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Thu, 21 Mar 2019 22:11:56 +0800 Subject: [PATCH 105/115] Revert "fix generate_java_gcs_fbs" This reverts commit b504a22e081acc8a15c55acc26e4ad727051c14e. --- java/BUILD.bazel | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 529a16a6169b..2bc515244869 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -45,6 +45,9 @@ define_java_module( define_java_module( name = "runtime", + additional_srcs = [ + ":generate_java_gcs_fbs", + ], additional_resources = [ ":java_native_deps", ], @@ -58,7 +61,6 @@ define_java_module( "@commons_io_commons_io//jar", ], deps = [ - ":copy_java_native_deps", ":org_ray_ray_api", "@plasma//:org_apache_arrow_arrow_plasma", "@org_slf4j_slf4j_api//jar", @@ -159,19 +161,14 @@ genrule( name = "generate_java_gcs_fbs", srcs = [":java_gcs_fbs"], outs = [ - "generate_java_gcs_fbs.out" + "runtime/src/main/java/org/ray/runtime/generated/" + file for file in flatbuffers_generated_files ], cmd = """ - WORK_DIR=$$(pwd) - TARGET_DIR=$$WORK_DIR/java/runtime/src/main/java/org/ray/runtime/generated - rm -rf $$TARGET_DIR - mkdir -p $$TARGET_DIR for f in $(locations //java:java_gcs_fbs); do chmod +w $$f - cp $$f $$TARGET_DIR + cp -f $$f $(@D)/runtime/src/main/java/org/ray/runtime/generated done - python $$WORK_DIR/java/modify_generated_java_flatbuffers_files.py $$WORK_DIR - echo $$(date) > $@ + python $$(pwd)/java/modify_generated_java_flatbuffers_files.py $(@D)/.. """, local = 1, ) From 60073ae8bc466ae2e3dec11f24eaa3dcce3127e7 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Thu, 21 Mar 2019 22:31:32 +0800 Subject: [PATCH 106/115] fix maven install --- java/BUILD.bazel | 28 +++++++++++++++++++++------- java/test.sh | 5 +++++ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 2bc515244869..16f29d0af0ae 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -185,23 +185,37 @@ filegroup( ], ) -# This rule copies Java native dependencies to 'java/runtime/native_dependencies', -# this is only needed by Maven (see 'java/runtime/pom.xml'). +# Generates the depedencies needed by maven. genrule( - name = "copy_java_native_deps", + name = "gen_maven_deps", srcs = [ ":java_native_deps", + ":generate_java_gcs_fbs", + "@plasma//:org_apache_arrow_arrow_plasma", ], - outs = ["copy_java_native_deps.out"], + outs = ["gen_maven_deps.out"], cmd = """ set -x WORK_DIR=$$(pwd) - mkdir -p $$WORK_DIR/java/runtime/native_dependencies/ + # Copy native dependecies. + NATIVE_DEPS_DIR=$$WORK_DIR/java/runtime/native_dependencies/ + rm -rf $$NATIVE_DEPS_DIR + mkdir -p $$NATIVE_DEPS_DIR for f in $(locations //java:java_native_deps); do chmod +w $$f - cp -f $$f $$WORK_DIR/java/runtime/native_dependencies + cp $$f $$NATIVE_DEPS_DIR + done + # Copy flatbuffers-generated files + GENERATED_DIR=$$WORK_DIR/java/runtime/src/main/java/org/ray/runtime/generated + rm -rf $$GENERATED_DIR + mkdir -p $$GENERATED_DIR + for f in $(locations //java:generate_java_gcs_fbs); do + cp $$f $$GENERATED_DIR done - echo $$WORK_DIR > $@ + # Install plasma jar to local maven repo. + mvn install:install-file -Dfile=$(locations @plasma//:org_apache_arrow_arrow_plasma) -Dpackaging=jar \ + -DgroupId=org.apache.arrow -DartifactId=arrow-plasma -Dversion=0.13.0-SNAPSHOT + echo $$(date) > $@ """, local = 1, ) diff --git a/java/test.sh b/java/test.sh index 716d05970009..b8b8aae9a441 100755 --- a/java/test.sh +++ b/java/test.sh @@ -32,3 +32,8 @@ if [ $single_exit_code -ne 2 ] && [ $single_exit_code -ne 0 ] ; then fi popd + +pushd $ROOT_DIR +echo "Testing maven install." +mvn clean install -Dmaven.test.skip +popd From 9d1fed25dda4f7516727a1f2a9a3ed6ed7e50e6c Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Thu, 21 Mar 2019 23:18:43 +0800 Subject: [PATCH 107/115] pass in PATH --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index ae78c36106d0..11c86813dc46 100755 --- a/build.sh +++ b/build.sh @@ -124,7 +124,7 @@ else if [ "$RAY_BUILD_JAVA" == "YES" ]; then bazel run //java:bazel_deps -- generate -r $ROOT_DIR -s java/third_party/workspace.bzl -d java/dependencies.yaml - bazel build //java:all --verbose_failures + bazel build //java:all --verbose_failures --action_ENV=PATH fi if [ "$RAY_BUILD_PYTHON" == "YES" ]; then From c71acfecc03b88f7b24abbedcfbe2ddd67be69e2 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Thu, 21 Mar 2019 23:21:14 +0800 Subject: [PATCH 108/115] debug --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c904a58a9e36..3603abcf0a72 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,7 @@ matrix: - PYTHON=3.5 PYTHONWARNINGS=ignore - RAY_INSTALL_JAVA=1 install: + - which mvn - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - if [ $RAY_CI_JAVA_AFFECTED != "1" ]; then exit; fi - ./ci/suppress_output ./ci/travis/install-bazel.sh From 330301bdcf0e0bb2b325326e57a0f978049bd497 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Thu, 21 Mar 2019 23:28:54 +0800 Subject: [PATCH 109/115] fix --- .travis.yml | 1 - build.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3603abcf0a72..c904a58a9e36 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,6 @@ matrix: - PYTHON=3.5 PYTHONWARNINGS=ignore - RAY_INSTALL_JAVA=1 install: - - which mvn - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - if [ $RAY_CI_JAVA_AFFECTED != "1" ]; then exit; fi - ./ci/suppress_output ./ci/travis/install-bazel.sh diff --git a/build.sh b/build.sh index 11c86813dc46..08a404288b63 100755 --- a/build.sh +++ b/build.sh @@ -124,7 +124,7 @@ else if [ "$RAY_BUILD_JAVA" == "YES" ]; then bazel run //java:bazel_deps -- generate -r $ROOT_DIR -s java/third_party/workspace.bzl -d java/dependencies.yaml - bazel build //java:all --verbose_failures --action_ENV=PATH + bazel build //java:all --verbose_failures --action_env=PATH fi if [ "$RAY_BUILD_PYTHON" == "YES" ]; then From 28c916920718f18bae3105662e7578d34ab8fd68 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Fri, 22 Mar 2019 00:12:54 +0800 Subject: [PATCH 110/115] fix --- java/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/test.sh b/java/test.sh index b8b8aae9a441..2f8dd0164666 100755 --- a/java/test.sh +++ b/java/test.sh @@ -9,7 +9,7 @@ ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) pushd $ROOT_DIR/.. echo "Linting Java code with checkstyle." -bazel test //java:all --test_tag_filters="checkstyle" +bazel test //java:all --test_tag_filters="checkstyle" --action_env=PATH echo "Running tests under cluster mode." # TODO(hchen): Ideally, we should use the following bazel command to run Java tests. However, if there're skipped tests, From ba2b535df6257b81251b39028aa4331e97b5f60b Mon Sep 17 00:00:00 2001 From: "ruifang.crf" Date: Fri, 22 Mar 2019 01:27:43 +0800 Subject: [PATCH 111/115] modify problem: mvn command not found --- build.sh | 2 +- java/test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 08a404288b63..da7fc4aabb29 100755 --- a/build.sh +++ b/build.sh @@ -124,7 +124,7 @@ else if [ "$RAY_BUILD_JAVA" == "YES" ]; then bazel run //java:bazel_deps -- generate -r $ROOT_DIR -s java/third_party/workspace.bzl -d java/dependencies.yaml - bazel build //java:all --verbose_failures --action_env=PATH + bazel build //java:all --verbose_failures --action_env PATH=$PATH fi if [ "$RAY_BUILD_PYTHON" == "YES" ]; then diff --git a/java/test.sh b/java/test.sh index 2f8dd0164666..af46be073bfc 100755 --- a/java/test.sh +++ b/java/test.sh @@ -9,7 +9,7 @@ ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) pushd $ROOT_DIR/.. echo "Linting Java code with checkstyle." -bazel test //java:all --test_tag_filters="checkstyle" --action_env=PATH +bazel test //java:all --test_tag_filters="checkstyle" --action_env PATH=$PATH echo "Running tests under cluster mode." # TODO(hchen): Ideally, we should use the following bazel command to run Java tests. However, if there're skipped tests, From 794354379d676bff079d07adbf11d80f46f44e88 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Fri, 22 Mar 2019 08:53:39 +0800 Subject: [PATCH 112/115] Revert "modify problem: mvn command not found" This reverts commit ba2b535df6257b81251b39028aa4331e97b5f60b. --- build.sh | 2 +- java/test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index da7fc4aabb29..08a404288b63 100755 --- a/build.sh +++ b/build.sh @@ -124,7 +124,7 @@ else if [ "$RAY_BUILD_JAVA" == "YES" ]; then bazel run //java:bazel_deps -- generate -r $ROOT_DIR -s java/third_party/workspace.bzl -d java/dependencies.yaml - bazel build //java:all --verbose_failures --action_env PATH=$PATH + bazel build //java:all --verbose_failures --action_env=PATH fi if [ "$RAY_BUILD_PYTHON" == "YES" ]; then diff --git a/java/test.sh b/java/test.sh index af46be073bfc..2f8dd0164666 100755 --- a/java/test.sh +++ b/java/test.sh @@ -9,7 +9,7 @@ ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) pushd $ROOT_DIR/.. echo "Linting Java code with checkstyle." -bazel test //java:all --test_tag_filters="checkstyle" --action_env PATH=$PATH +bazel test //java:all --test_tag_filters="checkstyle" --action_env=PATH echo "Running tests under cluster mode." # TODO(hchen): Ideally, we should use the following bazel command to run Java tests. However, if there're skipped tests, From 101fbd17f725a6849e8fa8310bc18997af444cb7 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Fri, 22 Mar 2019 08:55:37 +0800 Subject: [PATCH 113/115] skip test --- .../org/ray/runtime/functionmanager/FunctionManagerTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java index 6641abc6c4d7..47e2c864a453 100644 --- a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java +++ b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java @@ -117,7 +117,8 @@ public void testLoadFunctionTableForClass() { ImmutablePair.of(barConstructorDescriptor.name, barConstructorDescriptor.typeDescriptor))); } - @Test + // TODO(hchen): Figure out why this test crashes in Travis. + @Test(enabled = false) public void testGetFunctionFromLocalResource() throws Exception { UniqueId driverId = UniqueId.randomId(); final String resourcePath = FileUtils.getTempDirectoryPath() + "/ray_test_resources"; From 8c756d0bbd203c3733a0b06d0b340d7957d2e99e Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Fri, 22 Mar 2019 12:05:34 +0800 Subject: [PATCH 114/115] Revert "skip test" This reverts commit 101fbd17f725a6849e8fa8310bc18997af444cb7. --- .../org/ray/runtime/functionmanager/FunctionManagerTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java index 47e2c864a453..6641abc6c4d7 100644 --- a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java +++ b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java @@ -117,8 +117,7 @@ public void testLoadFunctionTableForClass() { ImmutablePair.of(barConstructorDescriptor.name, barConstructorDescriptor.typeDescriptor))); } - // TODO(hchen): Figure out why this test crashes in Travis. - @Test(enabled = false) + @Test public void testGetFunctionFromLocalResource() throws Exception { UniqueId driverId = UniqueId.randomId(); final String resourcePath = FileUtils.getTempDirectoryPath() + "/ray_test_resources"; From 67b628f97f6d2ed91fbe3d011fe2e27d870394e7 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Fri, 22 Mar 2019 12:13:41 +0800 Subject: [PATCH 115/115] Fix loading native libraries --- .../org/ray/runtime/RayNativeRuntime.java | 56 ++++++++++--------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java b/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java index a0fbdf01f28d..fcbe9123184f 100644 --- a/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java +++ b/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java @@ -3,6 +3,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import java.io.File; +import java.io.IOException; import java.io.InputStream; import java.lang.reflect.Field; import java.nio.ByteBuffer; @@ -30,7 +31,7 @@ import org.slf4j.LoggerFactory; /** - * native runtime for local box and cluster run. + * Native runtime for cluster mode. */ public final class RayNativeRuntime extends AbstractRayRuntime { @@ -46,57 +47,62 @@ public final class RayNativeRuntime extends AbstractRayRuntime { private List redisClients; private RunManager manager = null; + static { + try { + LOGGER.debug("Loading native libraries."); + // Load native libraries. + String[] libraries = new String[]{"raylet_library_java", "plasma_java"}; + for (String library : libraries) { + String fileName = System.mapLibraryName(library); + // Copy the file from resources to a temp dir, and load the native library. + File file = File.createTempFile(fileName, ""); + file.deleteOnExit(); + InputStream in = RayNativeRuntime.class.getResourceAsStream("/" + fileName); + Preconditions.checkNotNull(in, "{} doesn't exist.", fileName); + Files.copy(in, Paths.get(file.getAbsolutePath()), StandardCopyOption.REPLACE_EXISTING); + System.load(file.getAbsolutePath()); + } + LOGGER.debug("Native libraries loaded."); + } catch (IOException e) { + throw new RuntimeException("Couldn't load native libraries.", e); + } + } + public RayNativeRuntime(RayConfig rayConfig) { super(rayConfig); } private void resetLibraryPath() { + if (rayConfig.libraryPath.isEmpty()) { + return; + } + String path = System.getProperty("java.library.path"); if (Strings.isNullOrEmpty(path)) { path = ""; } else { path += ":"; } - path += String.join(":", rayConfig.libraryPath); // This is a hack to reset library path at runtime, // see https://stackoverflow.com/questions/15409223/. System.setProperty("java.library.path", path); - //set sys_paths to null so that java.library.path will be re-evalueted next time it is needed + // Set sys_paths to null so that java.library.path will be re-evaluated next time it is needed. final Field sysPathsField; try { sysPathsField = ClassLoader.class.getDeclaredField("sys_paths"); sysPathsField.setAccessible(true); sysPathsField.set(null, null); } catch (NoSuchFieldException | IllegalAccessException e) { - e.printStackTrace(); LOGGER.error("Failed to set library path.", e); } } @Override - public void start() throws Exception { - try { - // Reset library path at runtime. - resetLibraryPath(); - - // Load native libraries. - String[] libraries = new String[]{"raylet_library_java", "plasma_java"}; - for (String library : libraries) { - String fileName = System.mapLibraryName(library); - // Copy the file from resources to a temp dir, and load the native library. - File file = File.createTempFile(fileName, ""); - file.deleteOnExit(); - InputStream in = RayNativeRuntime.class.getResourceAsStream("/" + fileName); - Preconditions.checkNotNull(in, "{} doesn't exist.", fileName); - Files.copy(in, Paths.get(file.getAbsolutePath()), StandardCopyOption.REPLACE_EXISTING); - System.load(file.getAbsolutePath()); - } - } catch (Exception e) { - LOGGER.error("Failed to load native libraries.", e); - throw e; - } + public void start() { + // Reset library path at runtime. + resetLibraryPath(); if (rayConfig.getRedisAddress() == null) { manager = new RunManager(rayConfig);