diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 000000000000..55bcd0242bd9 --- /dev/null +++ b/.bazelrc @@ -0,0 +1,2 @@ +# build config +build --compilation_mode=opt diff --git a/.travis.yml b/.travis.yml index 4504f99dfcdc..c904a58a9e36 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,18 +25,16 @@ matrix: env: - JDK='Oracle JDK 8' - PYTHON=3.5 PYTHONWARNINGS=ignore - - RAY_USE_CMAKE=1 - RAY_INSTALL_JAVA=1 install: - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - if [ $RAY_CI_JAVA_AFFECTED != "1" ]; then exit; fi - - - ./ci/travis/install-dependencies.sh + - ./ci/suppress_output ./ci/travis/install-bazel.sh + - ./ci/suppress_output ./ci/travis/install-dependencies.sh - export PATH="$HOME/miniconda/bin:$PATH" - - ./ci/travis/install-ray.sh + - ./ci/suppress_output ./ci/travis/install-ray.sh script: - if [ $RAY_CI_JAVA_AFFECTED != "1" ]; then exit; fi - - ./java/test.sh - os: linux @@ -167,7 +165,7 @@ install: - ./ci/suppress_output bash src/ray/test/run_gcs_tests.sh # Raylet tests. - ./ci/suppress_output bash src/ray/test/run_object_manager_tests.sh - - ./ci/suppress_output bazel test --build_tests_only --test_lang_filters=cc ... -c opt + - ./ci/suppress_output bazel test --build_tests_only --test_lang_filters=cc //:all script: - export PATH="$HOME/miniconda/bin:$PATH" diff --git a/BUILD.bazel b/BUILD.bazel index b597939896b9..41366578d2b9 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -11,6 +11,7 @@ cc_binary( name = "raylet", srcs = ["src/ray/raylet/main.cc"], copts = COPTS, + visibility = ["//java:__subpackages__"], deps = [ ":ray_util", ":raylet_lib", @@ -100,6 +101,7 @@ cc_test( copts = COPTS, deps = [ ":ray_util", + "@boost//:asio", "@com_google_googletest//:gtest_main", ], ) @@ -372,7 +374,7 @@ pyx_library( ) cc_binary( - name = "raylet_library_java.so", + name = "libraylet_library_java.so", srcs = [ "src/ray/raylet/lib/java/org_ray_runtime_raylet_RayletClientImpl.h", "src/ray/raylet/lib/java/org_ray_runtime_raylet_RayletClientImpl.cc", @@ -394,7 +396,7 @@ cc_binary( "//conditions:default": ["external/bazel_tools/tools/jdk/include/linux"], }), linkshared = 1, - linkstatic = 0, + linkstatic = 1, deps = [ "//:raylet_lib", "@plasma//:plasma_client", @@ -403,8 +405,8 @@ cc_binary( genrule( name = "raylet-jni-darwin-compat", - srcs = [":raylet_library_java.so"], - outs = ["raylet_library_java.dylib"], + srcs = [":libraylet_library_java.so"], + outs = ["libraylet_library_java.dylib"], cmd = "cp $< $@", output_to_bindir = 1, ) @@ -412,16 +414,22 @@ genrule( filegroup( name = "raylet_library_java", srcs = select({ - "@bazel_tools//src/conditions:darwin": [":raylet_library_java.dylib"], - "//conditions:default": [":raylet_library_java.so"], + "@bazel_tools//src/conditions:darwin": [":libraylet_library_java.dylib"], + "//conditions:default": [":libraylet_library_java.so"], }), - visibility = ["//visibility:public"], + visibility = ["//java:__subpackages__"], +) + +filegroup( + name = "gcs_fbs_file", + srcs = ["src/ray/gcs/format/gcs.fbs"], + visibility = ["//java:__subpackages__"], ) flatbuffer_py_library( name = "python_gcs_fbs", srcs = [ - "src/ray/gcs/format/gcs.fbs", + ":gcs_fbs_file", ], outs = [ "ActorCheckpointIdData.py", @@ -526,10 +534,11 @@ genrule( mv ./src/redis-cli $(location redis-cli) && chmod +x $(location redis-cli) """, + visibility = ["//java:__subpackages__"], ) cc_binary( - name = "ray_redis_module.so", + name = "libray_redis_module.so", srcs = [ "src/ray/gcs/redis_module/ray_redis_module.cc", "src/ray/gcs/redis_module/redis_string.h", @@ -538,6 +547,7 @@ cc_binary( copts = COPTS, linkshared = 1, linkstatic = 1, + visibility = ["//java:__subpackages__"], deps = [ ":ray_common", ], @@ -552,29 +562,30 @@ genrule( "//:python_node_manager_fbs", "//:redis-server", "//:redis-cli", - "//:ray_redis_module.so", + "//:libray_redis_module.so", "//:raylet", "//:raylet_monitor", "@plasma//:plasma_store_server", ], - outs = ["ray_pkg"], + outs = ["ray_pkg.out"], cmd = """ set -x && - mkdir -p python/ray && - cp $(location python/ray/_raylet.so) python/ray && - mkdir -p python/ray/core/src/ray/thirdparty/redis/src/ && - cp $(location //:redis-server) python/ray/core/src/ray/thirdparty/redis/src/ && - cp $(location //:redis-cli) python/ray/core/src/ray/thirdparty/redis/src/ && - mkdir -p python/ray/core/src/ray/gcs/redis_module/ && - cp $(locations //:ray_redis_module.so) python/ray/core/src/ray/gcs/redis_module/libray_redis_module.so && - mkdir -p python/ray/core/src/ray/raylet/ && - cp $(location //:raylet_monitor) python/ray/core/src/ray/raylet/ && - mkdir -p python/ray/core/src/plasma && - cp $(location @plasma//:plasma_store_server) python/ray/core/src/plasma/ && - cp $(location //:raylet) python/ray/core/src/ray/raylet/ && - for f in $(locations //:python_gcs_fbs); do cp $$f python/ray/core/generated/; done && - mkdir -p python/ray/core/generated/ray/protocol/ && - for f in $(locations //:python_node_manager_fbs); do cp $$f python/ray/core/generated/ray/protocol/; done && - mv python $(location ray_pkg) - """, + WORK_DIR=$$(pwd) && + cp -f $(location python/ray/_raylet.so) $$WORK_DIR/python/ray && + mkdir -p $$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/ && + cp -f $(location //:redis-server) $$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/ && + cp -f $(location //:redis-cli) $$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/ && + mkdir -p $$WORK_DIR/python/ray/core/src/ray/gcs/redis_module/ && + cp -f $(locations //:libray_redis_module.so) $$WORK_DIR/python/ray/core/src/ray/gcs/redis_module/ && + cp -f $(location //:raylet_monitor) $$WORK_DIR/python/ray/core/src/ray/raylet/ && + cp -f $(location @plasma//:plasma_store_server) $$WORK_DIR/python/ray/core/src/plasma/ && + cp -f $(location //:raylet) $$WORK_DIR/python/ray/core/src/ray/raylet/ && + for f in $(locations //:python_gcs_fbs); do cp -f $$f $$WORK_DIR/python/ray/core/generated/; done && + mkdir -p $$WORK_DIR/python/ray/core/generated/ray/protocol/ && + for f in $(locations //:python_node_manager_fbs); do + cp -f $$f $$WORK_DIR/python/ray/core/generated/ray/protocol/; + done && + echo $$WORK_DIR > $@ + """, + local = 1, ) diff --git a/WORKSPACE b/WORKSPACE index 32b2e7b49dbc..79adb90f109d 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,4 +1,15 @@ +workspace(name = "com_github_ray_project_ray") + load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository") +load("//java:repo.bzl", "java_repositories") + +java_repositories() + +git_repository( + name = "com_github_checkstyle_java", + commit = "85f37871ca03b9d3fee63c69c8107f167e24e77b", + remote = "https://github.com/ruifangChen/checkstyle_java", +) git_repository( name = "com_github_nelhage_rules_boost", @@ -38,8 +49,8 @@ new_git_repository( new_git_repository( name = "plasma", build_file = "@//bazel:BUILD.plasma", - commit = "6a27c660ea700febf6fd73b2e851ab96e9315134", - remote = "https://github.com/ray-project/arrow", + commit = "9c33e1aee98f96a82c6a0b6bb214df172a143fb1", + remote = "https://github.com/apache/arrow", ) new_git_repository( diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index 33353caaf6f7..e0afd8580616 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -1,5 +1,26 @@ load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library") +java_library( + name = "org_apache_arrow_arrow_plasma", + srcs = glob(["java/plasma/src/main/java/**/*.java"]), + data = [":plasma_client_java"], + visibility = ["//visibility:public"], + deps = [ + "@org_slf4j_slf4j_api//jar", + ], +) + +java_binary( + name = "org_apache_arrow_arrow_plasma_test", + srcs = ["java/plasma/src/test/java/org/apache/arrow/plasma/PlasmaClientTest.java"], + main_class = "org.apache.arrow.plasma.PlasmaClientTest", + visibility = ["//visibility:public"], + deps = [ + ":org_apache_arrow_arrow_plasma", + "@junit_junit//jar", + ], +) + cc_library( name = "arrow", srcs = [ @@ -8,6 +29,7 @@ cc_library( "cpp/src/arrow/status.cc", "cpp/src/arrow/util/io-util.cc", "cpp/src/arrow/util/logging.cc", + "cpp/src/arrow/util/memory.cc", "cpp/src/arrow/util/thread-pool.cc", ], hdrs = [ @@ -19,6 +41,7 @@ cc_library( "cpp/src/arrow/util/io-util.h", "cpp/src/arrow/util/logging.h", "cpp/src/arrow/util/macros.h", + "cpp/src/arrow/util/memory.h", "cpp/src/arrow/util/string_builder.h", "cpp/src/arrow/util/string_view.h", "cpp/src/arrow/util/thread-pool.h", @@ -48,6 +71,7 @@ cc_library( "cpp/src/plasma/common.h", "cpp/src/plasma/common_generated.h", "cpp/src/plasma/compat.h", + "cpp/src/plasma/external_store.h", "cpp/src/plasma/fling.h", "cpp/src/plasma/io.h", "cpp/src/plasma/malloc.h", @@ -65,7 +89,7 @@ cc_library( ) cc_binary( - name = "plasma_client_java.so", + name = "libplasma_java.so", srcs = [ "cpp/src/plasma/lib/java/org_apache_arrow_plasma_PlasmaClientJNI.cc", "cpp/src/plasma/lib/java/org_apache_arrow_plasma_PlasmaClientJNI.h", @@ -77,7 +101,7 @@ cc_binary( "cpp/src", ], linkshared = 1, - linkstatic = 0, + linkstatic = 1, deps = [":plasma_client"], ) @@ -101,8 +125,8 @@ genrule( genrule( name = "plasma-jni-darwin-compat", - srcs = [":plasma_client_java.so"], - outs = ["plasma_client_java.dylib"], + srcs = [":libplasma_java.so"], + outs = ["libplasma_java.dylib"], cmd = "cp $< $@", output_to_bindir = 1, ) @@ -110,8 +134,8 @@ genrule( filegroup( name = "plasma_client_java", srcs = select({ - "@bazel_tools//src/conditions:darwin": [":plasma_client_java.dylib"], - "//conditions:default": [":plasma_client_java.so"], + "@bazel_tools//src/conditions:darwin": [":libplasma_java.dylib"], + "//conditions:default": [":libplasma_java.so"], }), visibility = ["//visibility:public"], ) @@ -122,10 +146,14 @@ cc_library( "cpp/src/plasma/dlmalloc.cc", "cpp/src/plasma/events.cc", "cpp/src/plasma/eviction_policy.cc", + "cpp/src/plasma/external_store.cc", + "cpp/src/plasma/plasma_allocator.cc", ], hdrs = [ "cpp/src/plasma/events.h", "cpp/src/plasma/eviction_policy.h", + "cpp/src/plasma/external_store.h", + "cpp/src/plasma/plasma_allocator.h", "cpp/src/plasma/store.h", "cpp/src/plasma/thirdparty/ae/ae.h", "cpp/src/plasma/thirdparty/ae/ae_epoll.c", diff --git a/bazel/ray.bzl b/bazel/ray.bzl index 4a1e4f1063e4..f55d1d29009e 100644 --- a/bazel/ray.bzl +++ b/bazel/ray.bzl @@ -1,4 +1,5 @@ load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_library_public") +load("@com_github_checkstyle_java//checkstyle:checkstyle.bzl", "checkstyle_test") def flatbuffer_py_library(name, srcs, outs, out_prefix, includes = [], include_paths = []): flatbuffer_library_public( @@ -10,3 +11,44 @@ def flatbuffer_py_library(name, srcs, outs, out_prefix, includes = [], include_p include_paths = include_paths, includes = includes, ) + +def flatbuffer_java_library(name, srcs, outs, out_prefix, includes = [], include_paths = []): + flatbuffer_library_public( + name = name, + srcs = srcs, + outs = outs, + language_flag = "-j", + out_prefix = out_prefix, + include_paths = include_paths, + includes = includes, + ) + +def define_java_module(name, additional_srcs = [], additional_resources = [], define_test_lib = False, test_deps = [], **kwargs): + native.java_library( + name = "org_ray_ray_" + name, + srcs = additional_srcs + native.glob([name + "/src/main/java/**/*.java"]), + resources = native.glob([name + "/src/main/resources/**"]) + additional_resources, + **kwargs + ) + checkstyle_test( + name = "org_ray_ray_" + name + "-checkstyle", + target = "//java:org_ray_ray_" + name, + config = "//java:checkstyle.xml", + suppressions = "//java:checkstyle-suppressions.xml", + size = "small", + tags = ["checkstyle"], + ) + if define_test_lib: + native.java_library( + name = "org_ray_ray_" + name + "_test", + srcs = native.glob([name + "/src/test/java/**/*.java"]), + deps = test_deps, + ) + checkstyle_test( + name = "org_ray_ray_" + name + "_test-checkstyle", + target = "//java:org_ray_ray_" + name + "_test", + config = "//java:checkstyle.xml", + suppressions = "//java:checkstyle-suppressions.xml", + size = "small", + tags = ["checkstyle"], + ) diff --git a/build.sh b/build.sh index 283f16757e49..08a404288b63 100755 --- a/build.sh +++ b/build.sh @@ -121,14 +121,15 @@ else $PYTHON_EXECUTABLE -m pip install \ --target=$ROOT_DIR/python/ray/pyarrow_files pyarrow==0.12.0.RAY \ --find-links https://s3-us-west-2.amazonaws.com/arrow-wheels/9357dc130789ee42f8181d8724bee1d5d1509060/index.html - bazel build //:ray_pkg -c opt --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHON_EXECUTABLE - # Copy files and keep them writeable. This is a workaround, as Bazel - # marks all generated files non-writeable. If we would just copy them - # over without adding write permission, the copy would fail the next time. - # TODO(pcm): It would be great to have a solution here that does not - # require us to copy the files. - find $ROOT_DIR/bazel-genfiles/ray_pkg/ -exec chmod +w {} \; - cp -r $ROOT_DIR/bazel-genfiles/ray_pkg/ray $ROOT_DIR/python || true + + if [ "$RAY_BUILD_JAVA" == "YES" ]; then + bazel run //java:bazel_deps -- generate -r $ROOT_DIR -s java/third_party/workspace.bzl -d java/dependencies.yaml + bazel build //java:all --verbose_failures --action_env=PATH + fi + + if [ "$RAY_BUILD_PYTHON" == "YES" ]; then + bazel build //:ray_pkg --verbose_failures --action_env=PYTHON_BIN_PATH=$PYTHON_EXECUTABLE + fi fi popd diff --git a/java/BUILD.bazel b/java/BUILD.bazel new file mode 100644 index 000000000000..16f29d0af0ae --- /dev/null +++ b/java/BUILD.bazel @@ -0,0 +1,221 @@ +load("//bazel:ray.bzl", "flatbuffer_java_library", "define_java_module") + +exports_files([ + "testng.xml", + "checkstyle.xml", + "checkstyle-suppressions.xml", +]) + +java_binary( + name = "bazel_deps", + main_class = "com.github.johnynek.bazel_deps.ParseProject", + runtime_deps = ["@bazel_deps//jar"], +) + +java_import( + name = "all_modules", + jars = [ + "liborg_ray_ray_api.jar", + "liborg_ray_ray_api-src.jar", + "liborg_ray_ray_runtime.jar", + "liborg_ray_ray_runtime-src.jar", + "liborg_ray_ray_tutorial.jar", + "liborg_ray_ray_tutorial-src.jar", + "all_tests_deploy.jar", + "all_tests_deploy-src.jar", + ], + deps = [ + ":org_ray_ray_api", + ":org_ray_ray_runtime", + ":org_ray_ray_tutorial", + ":all_tests", + ], +) + +define_java_module( + name = "api", + deps = [ + "@org_slf4j_slf4j_log4j12//jar", + "@log4j_log4j//jar", + "@javax_xml_bind_jaxb_api//jar", + "@com_sun_xml_bind_jaxb_core//jar", + "@com_sun_xml_bind_jaxb_impl//jar", + ], +) + +define_java_module( + name = "runtime", + additional_srcs = [ + ":generate_java_gcs_fbs", + ], + additional_resources = [ + ":java_native_deps", + ], + define_test_lib = True, + test_deps = [ + ":org_ray_ray_api", + ":org_ray_ray_runtime", + "@org_apache_commons_commons_lang3//jar", + "@com_beust_jcommander//jar", + "@org_testng_testng//jar", + "@commons_io_commons_io//jar", + ], + deps = [ + ":org_ray_ray_api", + "@plasma//:org_apache_arrow_arrow_plasma", + "@org_slf4j_slf4j_api//jar", + "@com_fasterxml_jackson_core_jackson_core//jar", + "@org_apache_commons_commons_pool2//jar", + "@org_javassist_javassist//jar", + "@org_objenesis_objenesis//jar", + "@com_typesafe_config//jar", + "@org_apache_commons_commons_lang3//jar", + "@de_ruedigermoeller_fst//jar", + "@com_github_davidmoten_flatbuffers_java//jar", + "@redis_clients_jedis//jar", + "@commons_io_commons_io//jar", + "@com_google_guava_guava//jar", + "@net_lingala_zip4j_zip4j//jar", + "@org_slf4j_slf4j_log4j12//jar", + "@org_ini4j_ini4j//jar", + "@org_ow2_asm_asm//jar", + ], +) + +define_java_module( + name = "tutorial", + deps = [ + ":org_ray_ray_api", + ":org_ray_ray_runtime", + "@com_google_guava_guava//jar", + ], +) + +define_java_module( + name = "test", + deps = [ + ":org_ray_ray_api", + ":org_ray_ray_runtime", + "@org_apache_commons_commons_lang3//jar", + "@org_slf4j_slf4j_api//jar", + "@plasma//:org_apache_arrow_arrow_plasma", + "@org_testng_testng//jar", + "@commons_collections_commons_collections//jar", + "@com_google_guava_guava//jar", + "@commons_io_commons_io//jar", + ], +) + +java_binary( + name = "all_tests", + main_class = "org.testng.TestNG", + data = ["testng.xml"], + args = ["java/testng.xml"], + runtime_deps = [ + ":org_ray_ray_test", + ":org_ray_ray_runtime_test", + ], +) + +flatbuffers_generated_files = [ + "ActorCheckpointData.java", + "ActorCheckpointIdData.java", + "ActorState.java", + "ActorTableData.java", + "Arg.java", + "ClassTableData.java", + "ClientTableData.java", + "ConfigTableData.java", + "CustomSerializerData.java", + "DriverTableData.java", + "ErrorTableData.java", + "ErrorType.java", + "FunctionTableData.java", + "GcsTableEntry.java", + "HeartbeatBatchTableData.java", + "HeartbeatTableData.java", + "Language.java", + "ObjectTableData.java", + "ProfileEvent.java", + "ProfileTableData.java", + "RayResource.java", + "ResourcePair.java", + "SchedulingState.java", + "TablePrefix.java", + "TablePubsub.java", + "TaskInfo.java", + "TaskLeaseData.java", + "TaskReconstructionData.java", + "TaskTableData.java", + "TaskTableTestAndUpdate.java", +] + +flatbuffer_java_library( + name = "java_gcs_fbs", + srcs = ["//:gcs_fbs_file"], + outs = flatbuffers_generated_files, + out_prefix = "", +) + +genrule( + name = "generate_java_gcs_fbs", + srcs = [":java_gcs_fbs"], + outs = [ + "runtime/src/main/java/org/ray/runtime/generated/" + file for file in flatbuffers_generated_files + ], + cmd = """ + for f in $(locations //java:java_gcs_fbs); do + chmod +w $$f + cp -f $$f $(@D)/runtime/src/main/java/org/ray/runtime/generated + done + python $$(pwd)/java/modify_generated_java_flatbuffers_files.py $(@D)/.. + """, + local = 1, +) + +filegroup( + name = "java_native_deps", + srcs = [ + "//:redis-server", + "//:libray_redis_module.so", + "//:raylet", + "//:raylet_library_java", + "@plasma//:plasma_store_server", + "@plasma//:plasma_client_java", + ], +) + +# Generates the depedencies needed by maven. +genrule( + name = "gen_maven_deps", + srcs = [ + ":java_native_deps", + ":generate_java_gcs_fbs", + "@plasma//:org_apache_arrow_arrow_plasma", + ], + outs = ["gen_maven_deps.out"], + cmd = """ + set -x + WORK_DIR=$$(pwd) + # Copy native dependecies. + NATIVE_DEPS_DIR=$$WORK_DIR/java/runtime/native_dependencies/ + rm -rf $$NATIVE_DEPS_DIR + mkdir -p $$NATIVE_DEPS_DIR + for f in $(locations //java:java_native_deps); do + chmod +w $$f + cp $$f $$NATIVE_DEPS_DIR + done + # Copy flatbuffers-generated files + GENERATED_DIR=$$WORK_DIR/java/runtime/src/main/java/org/ray/runtime/generated + rm -rf $$GENERATED_DIR + mkdir -p $$GENERATED_DIR + for f in $(locations //java:generate_java_gcs_fbs); do + cp $$f $$GENERATED_DIR + done + # Install plasma jar to local maven repo. + mvn install:install-file -Dfile=$(locations @plasma//:org_apache_arrow_arrow_plasma) -Dpackaging=jar \ + -DgroupId=org.apache.arrow -DartifactId=arrow-plasma -Dversion=0.13.0-SNAPSHOT + echo $$(date) > $@ + """, + local = 1, +) diff --git a/java/build.sh b/java/build.sh deleted file mode 100755 index 60b361ae8eb9..000000000000 --- a/java/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -mvn clean install -Dmaven.test.skip diff --git a/java/dependencies.yaml b/java/dependencies.yaml new file mode 100644 index 000000000000..0d4cdd68fae8 --- /dev/null +++ b/java/dependencies.yaml @@ -0,0 +1,116 @@ +options: + languages: [ "java" ] + licenses: [ "permissive" ] + namePrefix: "" + resolverType: "coursier" + resolvers: + - id: "mavencentral" + type: "default" + url: https://repo.maven.apache.org/maven2/ + thirdPartyDirectory: "java/third_party/" + transitivity: runtime_deps + versionConflictPolicy: highest + +dependencies: + com.puppycrawl.tools: + checkstyle: + lang: java + version: "8.15" + + de.ruedigermoeller: + fst: + lang: java + version: "2.47" + + org.ini4j: + ini4j: + lang: java + version: "0.5.2" + + org.ow2.asm: + asm: + lang: java + version: "6.0" + + com.github.davidmoten: + flatbuffers-java: + lang: java + version: "1.9.0.1" + + com.beust: + jcommander: + lang: java + version: "1.72" + + redis.clients: + jedis: + lang: java + version: "2.8.0" + + commons-io: + commons-io: + lang: java + version: "2.5" + + org.apache.commons: + commons-lang3: + lang: java + version: "3.4" + + commons-codec: + commons-codec: + lang: java + version: "1.4" + + net.lingala.zip4j: + zip4j: + lang: java + version: "1.3.2" + + com.google.guava: + guava: + lang: java + version: "27.0.1-jre" + + commons-collections: + commons-collections: + lang: java + version: "3.2.2" + + org.slf4j: + slf4j-log4j12: + lang: java + version: "1.7.25" + + com.typesafe: + config: + lang: java + version: "1.3.2" + + net.java.dev.jna: + jna: + lang: java + version: "4.1.0" + + org.mockito: + mockito-all: + lang: java + version: "1.10.19" + + org.testng: + testng: + lang: java + version: "6.9.9" + + javax.xml.bind: + jaxb-api: + lang: java + version: "2.3.0" + + com.sun.xml.bind: + jaxb-core: + lang: java + version: "2.3.0" + jaxb-impl: + lang: java + version: "2.3.0" diff --git a/java/repo.bzl b/java/repo.bzl new file mode 100644 index 000000000000..a06c285f234b --- /dev/null +++ b/java/repo.bzl @@ -0,0 +1,18 @@ +load("//java/third_party:workspace.bzl", "maven_dependencies") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_jar") + +def create_if_needed(rule, name, **kwargs): + if name not in native.existing_rules(): + rule(name = name, **kwargs) + +def bazel_deps(): + create_if_needed( + http_jar, + name = "bazel_deps", + sha256 = "98b05c2826f2248f70e7356dc6c78bc52395904bb932fbb409a5abf5416e4292", + urls = ["https://github.com/oferb/startupos-binaries/releases/download/0.1.01/bazel_deps.jar"], + ) + +def java_repositories(): + maven_dependencies() + bazel_deps() diff --git a/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java b/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java index a0fbdf01f28d..fcbe9123184f 100644 --- a/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java +++ b/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java @@ -3,6 +3,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import java.io.File; +import java.io.IOException; import java.io.InputStream; import java.lang.reflect.Field; import java.nio.ByteBuffer; @@ -30,7 +31,7 @@ import org.slf4j.LoggerFactory; /** - * native runtime for local box and cluster run. + * Native runtime for cluster mode. */ public final class RayNativeRuntime extends AbstractRayRuntime { @@ -46,57 +47,62 @@ public final class RayNativeRuntime extends AbstractRayRuntime { private List redisClients; private RunManager manager = null; + static { + try { + LOGGER.debug("Loading native libraries."); + // Load native libraries. + String[] libraries = new String[]{"raylet_library_java", "plasma_java"}; + for (String library : libraries) { + String fileName = System.mapLibraryName(library); + // Copy the file from resources to a temp dir, and load the native library. + File file = File.createTempFile(fileName, ""); + file.deleteOnExit(); + InputStream in = RayNativeRuntime.class.getResourceAsStream("/" + fileName); + Preconditions.checkNotNull(in, "{} doesn't exist.", fileName); + Files.copy(in, Paths.get(file.getAbsolutePath()), StandardCopyOption.REPLACE_EXISTING); + System.load(file.getAbsolutePath()); + } + LOGGER.debug("Native libraries loaded."); + } catch (IOException e) { + throw new RuntimeException("Couldn't load native libraries.", e); + } + } + public RayNativeRuntime(RayConfig rayConfig) { super(rayConfig); } private void resetLibraryPath() { + if (rayConfig.libraryPath.isEmpty()) { + return; + } + String path = System.getProperty("java.library.path"); if (Strings.isNullOrEmpty(path)) { path = ""; } else { path += ":"; } - path += String.join(":", rayConfig.libraryPath); // This is a hack to reset library path at runtime, // see https://stackoverflow.com/questions/15409223/. System.setProperty("java.library.path", path); - //set sys_paths to null so that java.library.path will be re-evalueted next time it is needed + // Set sys_paths to null so that java.library.path will be re-evaluated next time it is needed. final Field sysPathsField; try { sysPathsField = ClassLoader.class.getDeclaredField("sys_paths"); sysPathsField.setAccessible(true); sysPathsField.set(null, null); } catch (NoSuchFieldException | IllegalAccessException e) { - e.printStackTrace(); LOGGER.error("Failed to set library path.", e); } } @Override - public void start() throws Exception { - try { - // Reset library path at runtime. - resetLibraryPath(); - - // Load native libraries. - String[] libraries = new String[]{"raylet_library_java", "plasma_java"}; - for (String library : libraries) { - String fileName = System.mapLibraryName(library); - // Copy the file from resources to a temp dir, and load the native library. - File file = File.createTempFile(fileName, ""); - file.deleteOnExit(); - InputStream in = RayNativeRuntime.class.getResourceAsStream("/" + fileName); - Preconditions.checkNotNull(in, "{} doesn't exist.", fileName); - Files.copy(in, Paths.get(file.getAbsolutePath()), StandardCopyOption.REPLACE_EXISTING); - System.load(file.getAbsolutePath()); - } - } catch (Exception e) { - LOGGER.error("Failed to load native libraries.", e); - throw e; - } + public void start() { + // Reset library path at runtime. + resetLibraryPath(); if (rayConfig.getRedisAddress() == null) { manager = new RunManager(rayConfig); diff --git a/java/runtime/src/main/java/org/ray/runtime/functionmanager/FunctionManager.java b/java/runtime/src/main/java/org/ray/runtime/functionmanager/FunctionManager.java index 7c267d0eae0d..8ae245f8357e 100644 --- a/java/runtime/src/main/java/org/ray/runtime/functionmanager/FunctionManager.java +++ b/java/runtime/src/main/java/org/ray/runtime/functionmanager/FunctionManager.java @@ -1,9 +1,14 @@ package org.ray.runtime.functionmanager; +import com.google.common.base.Strings; +import java.io.File; import java.lang.invoke.SerializedLambda; import java.lang.reflect.Constructor; import java.lang.reflect.Executable; import java.lang.reflect.Method; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -15,7 +20,6 @@ import org.objectweb.asm.Type; import org.ray.api.function.RayFunc; import org.ray.api.id.UniqueId; -import org.ray.runtime.util.JarLoader; import org.ray.runtime.util.LambdaUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,8 +53,8 @@ public class FunctionManager { /** * Construct a FunctionManager with the specified driver resource path. * - * @param driverResourcePath The specified driver resource that - * can store the driver's resources. + * @param driverResourcePath The specified driver resource that can store the driver's + * resources. */ public FunctionManager(String driverResourcePath) { this.driverResourcePath = driverResourcePath; @@ -71,7 +75,7 @@ public RayFunction getFunction(UniqueId driverId, RayFunc func) { final String methodName = serializedLambda.getImplMethodName(); final String typeDescriptor = serializedLambda.getImplMethodSignature(); functionDescriptor = new JavaFunctionDescriptor(className, methodName, typeDescriptor); - RAY_FUNC_CACHE.get().put(func.getClass(),functionDescriptor); + RAY_FUNC_CACHE.get().put(func.getClass(), functionDescriptor); } return getFunction(driverId, functionDescriptor); } @@ -86,15 +90,18 @@ public RayFunction getFunction(UniqueId driverId, RayFunc func) { public RayFunction getFunction(UniqueId driverId, JavaFunctionDescriptor functionDescriptor) { DriverFunctionTable driverFunctionTable = driverFunctionTables.get(driverId); if (driverFunctionTable == null) { - String resourcePath = driverResourcePath + "/" + driverId.toString() + "/"; ClassLoader classLoader; - - if (driverResourcePath != null && !driverResourcePath.isEmpty()) { - classLoader = JarLoader.loadJars(resourcePath, false); - LOGGER.info("Succeeded to load driver({}) resource. Resource path is {}", - driverId, resourcePath); - } else { + if (Strings.isNullOrEmpty(driverResourcePath)) { classLoader = getClass().getClassLoader(); + } else { + File resourceDir = new File(driverResourcePath + "/" + driverId.toString() + "/"); + try { + classLoader = new URLClassLoader(new URL[]{resourceDir.toURI().toURL()}); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + LOGGER.debug("Resource loaded for driver {} from path {}.", driverId, + resourceDir.getAbsolutePath()); } driverFunctionTable = new DriverFunctionTable(classLoader); diff --git a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java index bec4eea591e8..447da9d9e7f7 100644 --- a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java +++ b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java @@ -1,6 +1,7 @@ package org.ray.runtime.runner; import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import java.io.File; @@ -94,6 +95,7 @@ private File getTempFile(String fileName) { file = File.createTempFile(fileName, ""); file.deleteOnExit(); try (InputStream in = RunManager.class.getResourceAsStream(fileName)) { + Preconditions.checkNotNull(in, "{} doesn't exist.", fileName); Files.copy(in, Paths.get(file.getCanonicalPath()), StandardCopyOption.REPLACE_EXISTING); } file.setExecutable(true); diff --git a/java/runtime/src/main/java/org/ray/runtime/util/JarLoader.java b/java/runtime/src/main/java/org/ray/runtime/util/JarLoader.java deleted file mode 100644 index c6ab5650c038..000000000000 --- a/java/runtime/src/main/java/org/ray/runtime/util/JarLoader.java +++ /dev/null @@ -1,92 +0,0 @@ -package org.ray.runtime.util; - -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Enumeration; -import java.util.List; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.commons.io.filefilter.DirectoryFileFilter; -import org.apache.commons.io.filefilter.RegexFileFilter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * load and unload jars from a dir. - */ -public class JarLoader { - - private static final Logger LOGGER = LoggerFactory.getLogger(JarLoader.class); - - public static URLClassLoader loadJars(String dir, boolean explicitLoad) { - // get all jars - Collection jars = FileUtils.listFiles( - new File(dir), - new RegexFileFilter(".*\\.jar"), - DirectoryFileFilter.DIRECTORY - ); - return loadJar(jars, explicitLoad); - } - - public static void unloadJars(ClassLoader loader) { - // now do nothing, if no ref to the loader and loader's class. - // they would be gc. - } - - private static URLClassLoader loadJar(Collection appJars, boolean explicitLoad) { - List jars = new ArrayList<>(); - List urls = new ArrayList<>(); - - for (File appJar : appJars) { - try { - LOGGER.info("succeeded to load jar {}.", appJar.getAbsolutePath()); - JarFile jar = new JarFile(appJar.getAbsolutePath()); - jars.add(jar); - urls.add(appJar.toURI().toURL()); - } catch (IOException e) { - throw new RuntimeException( - "invalid app jar path: " + appJar.getAbsolutePath() + ", load failed with exception", - e); - } - } - - URLClassLoader cl = URLClassLoader.newInstance(urls.toArray(new URL[urls.size()])); - - if (!explicitLoad) { - return cl; - } - for (JarFile jar : jars) { - try { - Enumeration e = jar.entries(); - while (e.hasMoreElements()) { - JarEntry je = e.nextElement(); - if (je.isDirectory() || !je.getName().endsWith(".class")) { - continue; - } - - String className = classNameOfJarEntry(je); - className = className.replace('/', '.'); - try { - Class.forName(className, true, cl); - } catch (ClassNotFoundException e1) { - e1.printStackTrace(); - } - } - } finally { - IOUtils.closeQuietly(jar); - } - } - return cl; - } - - private static String classNameOfJarEntry(JarEntry je) { - return je.getName().substring(0, je.getName().length() - ".class".length()); - } - -} diff --git a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java index 7bc1864d38eb..6641abc6c4d7 100644 --- a/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java +++ b/java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java @@ -3,8 +3,10 @@ import java.io.File; import java.nio.file.Files; import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.util.Map; +import javax.tools.JavaCompiler; +import javax.tools.ToolProvider; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.ray.api.annotation.RayRemote; @@ -115,27 +117,40 @@ public void testLoadFunctionTableForClass() { ImmutablePair.of(barConstructorDescriptor.name, barConstructorDescriptor.typeDescriptor))); } - //TODO(qwang): This is an integration test case, and we should move it to test folder in the future. @Test - public void testGetFunctionFromLocalResource() throws Exception{ - UniqueId driverId = UniqueId.fromHexString("0123456789012345678901234567890123456789"); - - //TODO(qwang): We should use a independent app demo instead of `tutorial`. - final String resourcePath = "/tmp/ray/test/resource"; - final String srcJarPath = System.getProperty("user.dir") + - "/../tutorial/target/ray-tutorial-0.1-SNAPSHOT.jar"; - final String destJarPath = resourcePath + "/" + driverId.toString() + - "/ray-tutorial-0.1-SNAPSHOT.jar"; - - File file = new File(resourcePath + "/" + driverId.toString()); - file.mkdirs(); - Files.copy(Paths.get(srcJarPath), Paths.get(destJarPath), StandardCopyOption.REPLACE_EXISTING); + public void testGetFunctionFromLocalResource() throws Exception { + UniqueId driverId = UniqueId.randomId(); + final String resourcePath = FileUtils.getTempDirectoryPath() + "/ray_test_resources"; + final String driverResourcePath = resourcePath + "/" + driverId.toString(); + File driverResourceDir = new File(driverResourcePath); + FileUtils.deleteQuietly(driverResourceDir); + driverResourceDir.mkdirs(); + driverResourceDir.deleteOnExit(); + + String demoJavaFile = ""; + demoJavaFile += "public class DemoApp {\n"; + demoJavaFile += " public static String hello() {\n"; + demoJavaFile += " return \"hello\";\n"; + demoJavaFile += " }\n"; + demoJavaFile += "}"; + + // Write the demo java file to the driver resource path. + String javaFilePath = driverResourcePath + "/DemoApp.java"; + Files.write(Paths.get(javaFilePath), demoJavaFile.getBytes()); + + // Compile the java file. + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + int result = compiler.run(null, null, null, "-d", driverResourcePath, javaFilePath); + if (result != 0) { + throw new RuntimeException("Couldn't compile Demo.java."); + } + // Test loading the function. + JavaFunctionDescriptor descriptor = new JavaFunctionDescriptor( + "DemoApp", "hello", "()Ljava/lang/String;"); final FunctionManager functionManager = new FunctionManager(resourcePath); - JavaFunctionDescriptor sayHelloDescriptor = new JavaFunctionDescriptor("org.ray.exercise.Exercise02", - "sayHello", "()Ljava/lang/String;"); - RayFunction func = functionManager.getFunction(driverId, sayHelloDescriptor); - Assert.assertEquals(func.getFunctionDescriptor(), sayHelloDescriptor); + RayFunction func = functionManager.getFunction(driverId, descriptor); + Assert.assertEquals(func.getFunctionDescriptor(), descriptor); } } diff --git a/java/test.sh b/java/test.sh index b3e889371bcd..2f8dd0164666 100755 --- a/java/test.sh +++ b/java/test.sh @@ -7,20 +7,33 @@ set -x ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -pushd $ROOT_DIR/../java -echo "Compiling Java code." -mvn clean install -Dmaven.test.skip - -echo "Checking code format." -mvn checkstyle:check +pushd $ROOT_DIR/.. +echo "Linting Java code with checkstyle." +bazel test //java:all --test_tag_filters="checkstyle" --action_env=PATH echo "Running tests under cluster mode." -ENABLE_MULTI_LANGUAGE_TESTS=1 mvn test +# TODO(hchen): Ideally, we should use the following bazel command to run Java tests. However, if there're skipped tests, +# TestNG will exit with code 2. And bazel treats it as test failure. +# bazel test //java:all_tests --action_env=ENABLE_MULTI_LANGUAGE_TESTS=1 --test_output="errors" || cluster_exit_code=$? +ENABLE_MULTI_LANGUAGE_TESTS=1 java -jar $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml|| cluster_exit_code=$? + +# exit_code == 2 means there are some tests skiped. +if [ $cluster_exit_code -ne 2 ] && [ $cluster_exit_code -ne 0 ] ; then + exit $cluster_exit_code +fi echo "Running tests under single-process mode." -mvn test -Dray.run-mode=SINGLE_PROCESS +# bazel test //java:all_tests --jvmopt="-Dray.run-mode=SINGLE_PROCESS" --test_output="errors" || single_exit_code=$? +java -jar -Dray.run-mode="SINGLE_PROCESS" $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar $ROOT_DIR/testng.xml || single_exit_code=$? -set +x -set +e +# exit_code == 2 means there are some tests skiped. +if [ $single_exit_code -ne 2 ] && [ $single_exit_code -ne 0 ] ; then + exit $single_exit_code +fi popd + +pushd $ROOT_DIR +echo "Testing maven install." +mvn clean install -Dmaven.test.skip +popd diff --git a/java/test/src/main/java/org/ray/api/test/ClientExceptionTest.java b/java/test/src/main/java/org/ray/api/test/ClientExceptionTest.java index e9f53dddd794..b588822712c5 100644 --- a/java/test/src/main/java/org/ray/api/test/ClientExceptionTest.java +++ b/java/test/src/main/java/org/ray/api/test/ClientExceptionTest.java @@ -37,12 +37,12 @@ public void testWaitAndCrash() { Ray.wait(ImmutableList.of(notExisting), 1, 2000); Assert.fail("Should not reach here"); } catch (RayException e) { - LOGGER.debug(String.format("Expected runtime exception: {}", e)); + LOGGER.debug("Expected runtime exception: {}", e); } try { thread.join(); } catch (Exception e) { - LOGGER.error(String.format("Excpetion caught: {}", e)); + LOGGER.error("Excpetion caught: {}", e); } } } diff --git a/java/testng.xml b/java/testng.xml new file mode 100644 index 000000000000..81503edf4b85 --- /dev/null +++ b/java/testng.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/java/third_party/BUILD b/java/third_party/BUILD new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/java/third_party/workspace.bzl b/java/third_party/workspace.bzl new file mode 100644 index 000000000000..7c5c7d0d09f3 --- /dev/null +++ b/java/third_party/workspace.bzl @@ -0,0 +1,119 @@ +# Do not edit. bazel-deps autogenerates this file from java/dependencies.yaml. +def _jar_artifact_impl(ctx): + jar_name = "%s.jar" % ctx.name + ctx.download( + output=ctx.path("jar/%s" % jar_name), + url=ctx.attr.urls, + sha256=ctx.attr.sha256, + executable=False + ) + src_name="%s-sources.jar" % ctx.name + srcjar_attr="" + has_sources = len(ctx.attr.src_urls) != 0 + if has_sources: + ctx.download( + output=ctx.path("jar/%s" % src_name), + url=ctx.attr.src_urls, + sha256=ctx.attr.src_sha256, + executable=False + ) + srcjar_attr ='\n srcjar = ":%s",' % src_name + + build_file_contents = """ +package(default_visibility = ['//visibility:public']) +java_import( + name = 'jar', + jars = ['{jar_name}'],{srcjar_attr} +) +filegroup( + name = 'file', + srcs = [ + '{jar_name}', + '{src_name}' + ], + visibility = ['//visibility:public'] +)\n""".format(jar_name = jar_name, src_name = src_name, srcjar_attr = srcjar_attr) + ctx.file(ctx.path("jar/BUILD"), build_file_contents, False) + return None + +jar_artifact = repository_rule( + attrs = { + "artifact": attr.string(mandatory = True), + "sha256": attr.string(mandatory = True), + "urls": attr.string_list(mandatory = True), + "src_sha256": attr.string(mandatory = False, default=""), + "src_urls": attr.string_list(mandatory = False, default=[]), + }, + implementation = _jar_artifact_impl +) + +def jar_artifact_callback(hash): + src_urls = [] + src_sha256 = "" + source=hash.get("source", None) + if source != None: + src_urls = [source["url"]] + src_sha256 = source["sha256"] + jar_artifact( + artifact = hash["artifact"], + name = hash["name"], + urls = [hash["url"]], + sha256 = hash["sha256"], + src_urls = src_urls, + src_sha256 = src_sha256 + ) + native.bind(name = hash["bind"], actual = hash["actual"]) + +def list_dependencies(): + return [ + {"artifact": "antlr:antlr:2.7.7", "lang": "java", "sha1": "83cd2cd674a217ade95a4bb83a8a14f351f48bd0", "sha256": "88fbda4b912596b9f56e8e12e580cc954bacfb51776ecfddd3e18fc1cf56dc4c", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/antlr/antlr/2.7.7/antlr-2.7.7.jar", "name": "antlr_antlr", "actual": "@antlr_antlr//jar", "bind": "jar/antlr/antlr"}, + {"artifact": "com.beust:jcommander:1.72", "lang": "java", "sha1": "6375e521c1e11d6563d4f25a07ce124ccf8cd171", "sha256": "e0de160b129b2414087e01fe845609cd55caec6820cfd4d0c90fabcc7bdb8c1e", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/beust/jcommander/1.72/jcommander-1.72.jar", "source": {"sha1": "7ef123d5dfb6f839b41265648ff1be34982d50f8", "sha256": "9b4fddc257d3572696333f48158802243141ce46579d92f0a4c0b2c14956b49d", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/beust/jcommander/1.72/jcommander-1.72-sources.jar"} , "name": "com_beust_jcommander", "actual": "@com_beust_jcommander//jar", "bind": "jar/com/beust/jcommander"}, + {"artifact": "com.fasterxml.jackson.core:jackson-core:2.5.3", "lang": "java", "sha1": "a8b8a6dfc8a17890e4c7ff8aed810763d265b68b", "sha256": "c9a900c30c702d371be2d3714938dfa6de616599c8eb39f8e3cf7aedd62dfd32", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/fasterxml/jackson/core/jackson-core/2.5.3/jackson-core-2.5.3.jar", "source": {"sha1": "8bfff00c810764fd4e0fc78959b45f8734dd0c3e", "sha256": "5fcf1bf0f37c24a136fe72f113ed9218560967d9b1483e8adf3bc49d0459a522", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/fasterxml/jackson/core/jackson-core/2.5.3/jackson-core-2.5.3-sources.jar"} , "name": "com_fasterxml_jackson_core_jackson_core", "actual": "@com_fasterxml_jackson_core_jackson_core//jar", "bind": "jar/com/fasterxml/jackson/core/jackson_core"}, + {"artifact": "com.github.davidmoten:flatbuffers-java:1.9.0.1", "lang": "java", "sha1": "0eaa1b27095bb5127879458bc2ee1b78e00d4a20", "sha256": "7432b0380f0ee3e1d018305fd812fdcde95222107dbaade6a7b49010d6de3020", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/github/davidmoten/flatbuffers-java/1.9.0.1/flatbuffers-java-1.9.0.1.jar", "source": {"sha1": "e5e796e7f1c2c84b5fa4fbff0ee9b40bde9c3870", "sha256": "e0e36c11c9639fda34e849ebcb828c3e40adf842c12cfe90dc8505d9ead69fc9", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/github/davidmoten/flatbuffers-java/1.9.0.1/flatbuffers-java-1.9.0.1-sources.jar"} , "name": "com_github_davidmoten_flatbuffers_java", "actual": "@com_github_davidmoten_flatbuffers_java//jar", "bind": "jar/com/github/davidmoten/flatbuffers_java"}, + {"artifact": "com.google.code.findbugs:jsr305:3.0.2", "lang": "java", "sha1": "25ea2e8b0c338a877313bd4672d3fe056ea78f0d", "sha256": "766ad2a0783f2687962c8ad74ceecc38a28b9f72a2d085ee438b7813e928d0c7", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar", "source": {"sha1": "b19b5927c2c25b6c70f093767041e641ae0b1b35", "sha256": "1c9e85e272d0708c6a591dc74828c71603053b48cc75ae83cce56912a2aa063b", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2-sources.jar"} , "name": "com_google_code_findbugs_jsr305", "actual": "@com_google_code_findbugs_jsr305//jar", "bind": "jar/com/google/code/findbugs/jsr305"}, + {"artifact": "com.google.errorprone:error_prone_annotations:2.2.0", "lang": "java", "sha1": "88e3c593e9b3586e1c6177f89267da6fc6986f0c", "sha256": "6ebd22ca1b9d8ec06d41de8d64e0596981d9607b42035f9ed374f9de271a481a", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/errorprone/error_prone_annotations/2.2.0/error_prone_annotations-2.2.0.jar", "source": {"sha1": "a8cd7823aa1dcd2fd6677c0c5988fdde9d1fb0a3", "sha256": "626adccd4894bee72c3f9a0384812240dcc1282fb37a87a3f6cb94924a089496", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/errorprone/error_prone_annotations/2.2.0/error_prone_annotations-2.2.0-sources.jar"} , "name": "com_google_errorprone_error_prone_annotations", "actual": "@com_google_errorprone_error_prone_annotations//jar", "bind": "jar/com/google/errorprone/error_prone_annotations"}, + {"artifact": "com.google.guava:failureaccess:1.0.1", "lang": "java", "sha1": "1dcf1de382a0bf95a3d8b0849546c88bac1292c9", "sha256": "a171ee4c734dd2da837e4b16be9df4661afab72a41adaf31eb84dfdaf936ca26", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/guava/failureaccess/1.0.1/failureaccess-1.0.1.jar", "source": {"sha1": "1d064e61aad6c51cc77f9b59dc2cccc78e792f5a", "sha256": "092346eebbb1657b51aa7485a246bf602bb464cc0b0e2e1c7e7201fadce1e98f", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/guava/failureaccess/1.0.1/failureaccess-1.0.1-sources.jar"} , "name": "com_google_guava_failureaccess", "actual": "@com_google_guava_failureaccess//jar", "bind": "jar/com/google/guava/failureaccess"}, + {"artifact": "com.google.guava:guava:27.0.1-jre", "lang": "java", "sha1": "bd41a290787b5301e63929676d792c507bbc00ae", "sha256": "e1c814fd04492a27c38e0317eabeaa1b3e950ec8010239e400fe90ad6c9107b4", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/guava/guava/27.0.1-jre/guava-27.0.1-jre.jar", "source": {"sha1": "cb5c1119df8d41a428013289b193eba3ccaf5f60", "sha256": "cba2e5680186062f42998b895a5e9a9ceccbaab94644ccc9f35bb73c2b2c7d8e", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/guava/guava/27.0.1-jre/guava-27.0.1-jre-sources.jar"} , "name": "com_google_guava_guava", "actual": "@com_google_guava_guava//jar", "bind": "jar/com/google/guava/guava"}, + {"artifact": "com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava", "lang": "java", "sha1": "b421526c5f297295adef1c886e5246c39d4ac629", "sha256": "b372a037d4230aa57fbeffdef30fd6123f9c0c2db85d0aced00c91b974f33f99", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar", "name": "com_google_guava_listenablefuture", "actual": "@com_google_guava_listenablefuture//jar", "bind": "jar/com/google/guava/listenablefuture"}, + {"artifact": "com.google.j2objc:j2objc-annotations:1.1", "lang": "java", "sha1": "ed28ded51a8b1c6b112568def5f4b455e6809019", "sha256": "2994a7eb78f2710bd3d3bfb639b2c94e219cedac0d4d084d516e78c16dddecf6", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar", "source": {"sha1": "1efdf5b737b02f9b72ebdec4f72c37ec411302ff", "sha256": "2cd9022a77151d0b574887635cdfcdf3b78155b602abc89d7f8e62aba55cfb4f", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1-sources.jar"} , "name": "com_google_j2objc_j2objc_annotations", "actual": "@com_google_j2objc_j2objc_annotations//jar", "bind": "jar/com/google/j2objc/j2objc_annotations"}, + {"artifact": "com.puppycrawl.tools:checkstyle:8.15", "lang": "java", "sha1": "8584d88c6aefcfb079adb8d102928b3eeb4de6ad", "sha256": "ac3602c4d50c3113b14614a6ac38ec03c63d9839e4316e057c4bb66d97183087", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/puppycrawl/tools/checkstyle/8.15/checkstyle-8.15.jar", "source": {"sha1": "1c1865dc6d20a1abc14ea7721fcbc78e2563d4ca", "sha256": "6745eda6450aa6e8609e56499fd56f515402a2ace3c7bd0345ec9b3f6eb8f6f6", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/puppycrawl/tools/checkstyle/8.15/checkstyle-8.15-sources.jar"} , "name": "com_puppycrawl_tools_checkstyle", "actual": "@com_puppycrawl_tools_checkstyle//jar", "bind": "jar/com/puppycrawl/tools/checkstyle"}, + {"artifact": "com.sun.xml.bind:jaxb-core:2.3.0", "lang": "java", "sha1": "d044c784e41d026778693fb44a8026c1fd9a7506", "sha256": "33061af8d2a07379d452f8f0fddcbbf518428dfd3b83d9e2d479c1948020795b", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/sun/xml/bind/jaxb-core/2.3.0/jaxb-core-2.3.0.jar", "source": {"sha1": "10bc9795ef8a09b28590efe9d0ecc06485820106", "sha256": "9a3650caf4af09105a9a4325cdabaac97b077f79bdeae9f0c3a94ce3123db01c", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/sun/xml/bind/jaxb-core/2.3.0/jaxb-core-2.3.0-sources.jar"} , "name": "com_sun_xml_bind_jaxb_core", "actual": "@com_sun_xml_bind_jaxb_core//jar", "bind": "jar/com/sun/xml/bind/jaxb_core"}, + {"artifact": "com.sun.xml.bind:jaxb-impl:2.3.0", "lang": "java", "sha1": "3a3c6a62719d967175b76b63925f1fb495f11437", "sha256": "edd691fc6ad7c7f3167e1a6833372367bedf6f4f2a4b8230df444a59bb3718b1", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/sun/xml/bind/jaxb-impl/2.3.0/jaxb-impl-2.3.0.jar", "source": {"sha1": "bb0a5a9f4e1116f2bc88fb7d38efd843704b65f8", "sha256": "8627a8a43b0ff84c61c698e3b94e3440e9b30439d0a7085a11677c75eda97ddc", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/sun/xml/bind/jaxb-impl/2.3.0/jaxb-impl-2.3.0-sources.jar"} , "name": "com_sun_xml_bind_jaxb_impl", "actual": "@com_sun_xml_bind_jaxb_impl//jar", "bind": "jar/com/sun/xml/bind/jaxb_impl"}, + {"artifact": "com.typesafe:config:1.3.2", "lang": "java", "sha1": "d6ac0ce079f114adce620f2360c92a70b2cb36dc", "sha256": "6563d1723f3300bf596f41e40bc03e54986108b5c45d0ac34ebc66d48c2e25a3", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/typesafe/config/1.3.2/config-1.3.2.jar", "source": {"sha1": "a22dda7d62b800b297c1c59de90d48992d8119b1", "sha256": "65995abd56d6aa99ee7f46e7cdaaaac2968554b16b26d38bf67e13706a12ca82", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/com/typesafe/config/1.3.2/config-1.3.2-sources.jar"} , "name": "com_typesafe_config", "actual": "@com_typesafe_config//jar", "bind": "jar/com/typesafe/config"}, + {"artifact": "commons-beanutils:commons-beanutils:1.9.3", "lang": "java", "sha1": "c845703de334ddc6b4b3cd26835458cb1cba1f3d", "sha256": "c058e39c7c64203d3a448f3adb588cb03d6378ed808485618f26e137f29dae73", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-beanutils/commons-beanutils/1.9.3/commons-beanutils-1.9.3.jar", "source": {"sha1": "9d70af7c5982bd09c33efa97ac98cee6363ba0f0", "sha256": "3315f25f3793c1bb2577b2d956f58f852c7386c73aff4dea450e419a80b00a41", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-beanutils/commons-beanutils/1.9.3/commons-beanutils-1.9.3-sources.jar"} , "name": "commons_beanutils_commons_beanutils", "actual": "@commons_beanutils_commons_beanutils//jar", "bind": "jar/commons_beanutils/commons_beanutils"}, + {"artifact": "commons-codec:commons-codec:1.4", "lang": "java", "sha1": "4216af16d38465bbab0f3dff8efa14204f7a399a", "sha256": "6aa4234c74f3a1035751a25822545867c8c3727125a642b6e049665d1863631b", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-codec/commons-codec/1.4/commons-codec-1.4.jar", "source": {"sha1": "5310da9f90e843883309e9e0bf5950faa79882a0", "sha256": "78e3cee598647e409ea8860c15c040d172fd5bc8e070cbad9ca1684c36e71753", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-codec/commons-codec/1.4/commons-codec-1.4-sources.jar"} , "name": "commons_codec_commons_codec", "actual": "@commons_codec_commons_codec//jar", "bind": "jar/commons_codec/commons_codec"}, + {"artifact": "commons-collections:commons-collections:3.2.2", "lang": "java", "sha1": "8ad72fe39fa8c91eaaf12aadb21e0c3661fe26d5", "sha256": "eeeae917917144a68a741d4c0dff66aa5c5c5fd85593ff217bced3fc8ca783b8", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar", "source": {"sha1": "78c50ebda5784937ca1615fc0e1d0cb35857d572", "sha256": "a5b5ee16a02edadf7fe637f250217c19878bc6134f15eb55635c48996f6fed1d", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2-sources.jar"} , "name": "commons_collections_commons_collections", "actual": "@commons_collections_commons_collections//jar", "bind": "jar/commons_collections/commons_collections"}, + {"artifact": "commons-io:commons-io:2.5", "lang": "java", "sha1": "2852e6e05fbb95076fc091f6d1780f1f8fe35e0f", "sha256": "a10418348d234968600ccb1d988efcbbd08716e1d96936ccc1880e7d22513474", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-io/commons-io/2.5/commons-io-2.5.jar", "source": {"sha1": "0caf033a4a7c37b4a8ff3ea084cba591539b0b69", "sha256": "3b69b518d9a844732e35509b79e499fca63a960ee4301b1c96dc32e87f3f60a1", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-io/commons-io/2.5/commons-io-2.5-sources.jar"} , "name": "commons_io_commons_io", "actual": "@commons_io_commons_io//jar", "bind": "jar/commons_io/commons_io"}, + {"artifact": "commons-logging:commons-logging:1.2", "lang": "java", "sha1": "4bfc12adfe4842bf07b657f0369c4cb522955686", "sha256": "daddea1ea0be0f56978ab3006b8ac92834afeefbd9b7e4e6316fca57df0fa636", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-logging/commons-logging/1.2/commons-logging-1.2.jar", "source": {"sha1": "ecf26c7507d67782a3bbd148d170b31dfad001aa", "sha256": "44347acfe5860461728e9cb33251e97345be36f8a0dfd5c5130c172559455f41", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/commons-logging/commons-logging/1.2/commons-logging-1.2-sources.jar"} , "name": "commons_logging_commons_logging", "actual": "@commons_logging_commons_logging//jar", "bind": "jar/commons_logging/commons_logging"}, + {"artifact": "de.ruedigermoeller:fst:2.47", "lang": "java", "sha1": "281d390ebed24a3621d3053825affc13d0eead8b", "sha256": "8c871febc859cf72dbed1c6c1eebc5600c3d93c3e986feccf4623cc5086098cc", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/de/ruedigermoeller/fst/2.47/fst-2.47.jar", "source": {"sha1": "2e0bb6c02600b3c11c041685a29f6981f3c54c7a", "sha256": "d026fb853b6d5269ec06b9b430488c02c7ac1569fe0233ad785f8bfb7e09688b", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/de/ruedigermoeller/fst/2.47/fst-2.47-sources.jar"} , "name": "de_ruedigermoeller_fst", "actual": "@de_ruedigermoeller_fst//jar", "bind": "jar/de/ruedigermoeller/fst"}, + {"artifact": "info.picocli:picocli:3.8.0", "lang": "java", "sha1": "86d47318b178e9e1d8b600a7afd1b8bd36300d3c", "sha256": "2b615ea91a8487d630efb7d3b04cab334d30ca414575138ce94201d470450fc4", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/info/picocli/picocli/3.8.0/picocli-3.8.0.jar", "source": {"sha1": "6fbfd760f8ed0bc04ac7311c52878c25eed81176", "sha256": "a23d58cf6c90bbd5568dbd7bb519b7225377ec2330315da0c8c4e623fc3dbf05", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/info/picocli/picocli/3.8.0/picocli-3.8.0-sources.jar"} , "name": "info_picocli_picocli", "actual": "@info_picocli_picocli//jar", "bind": "jar/info/picocli/picocli"}, + {"artifact": "javax.xml.bind:jaxb-api:2.3.0", "lang": "java", "sha1": "99f802e0cb3e953ba3d6e698795c4aeb98d37c48", "sha256": "883007989d373d19f352ba9792b25dec21dc7d0e205a710a93a3815101bb3d03", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/javax/xml/bind/jaxb-api/2.3.0/jaxb-api-2.3.0.jar", "source": {"sha1": "431dc11c8e5d8b9f5149be5757ef8e170fc6824a", "sha256": "de09da7ffa14a4d6d35bde4452b3a88fafa1b8aa5aca0d5bad427649069e8b9f", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/javax/xml/bind/jaxb-api/2.3.0/jaxb-api-2.3.0-sources.jar"} , "name": "javax_xml_bind_jaxb_api", "actual": "@javax_xml_bind_jaxb_api//jar", "bind": "jar/javax/xml/bind/jaxb_api"}, + {"artifact": "junit:junit:4.10", "lang": "java", "sha1": "e4f1766ce7404a08f45d859fb9c226fc9e41a861", "sha256": "36a747ca1e0b86f6ea88055b8723bb87030d627766da6288bf077afdeeb0f75a", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/junit/junit/4.10/junit-4.10.jar", "source": {"sha1": "6c98d6766e72d5575f96c9479d1c1d3b865c6e25", "sha256": "e3a4cb7ac3343265f5663b68857078ae68787450afc6e72dc6826962a1bf5212", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/junit/junit/4.10/junit-4.10-sources.jar"} , "name": "junit_junit", "actual": "@junit_junit//jar", "bind": "jar/junit/junit"}, + {"artifact": "log4j:log4j:1.2.17", "lang": "java", "sha1": "5af35056b4d257e4b64b9e8069c0746e8b08629f", "sha256": "1d31696445697720527091754369082a6651bd49781b6005deb94e56753406f9", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/log4j/log4j/1.2.17/log4j-1.2.17.jar", "source": {"sha1": "677abe279b68c5e7490d6d50c6951376238d7d3e", "sha256": "4d9ba787af1692aa88417c2a47a37a98125d645b91ab556252dbee0f45225493", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/log4j/log4j/1.2.17/log4j-1.2.17-sources.jar"} , "name": "log4j_log4j", "actual": "@log4j_log4j//jar", "bind": "jar/log4j/log4j"}, + {"artifact": "net.java.dev.jna:jna:4.1.0", "lang": "java", "sha1": "1c12d070e602efd8021891cdd7fd18bc129372d4", "sha256": "1aa37e9ea6baa0ee152d89509f758f0847eac66ec179b955cafe0919e540a92e", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/net/java/dev/jna/jna/4.1.0/jna-4.1.0.jar", "source": {"sha1": "28a5d592b494fe54c120c64689087e779a7bf55b", "sha256": "a7d37342b0447865934bc74c19be90565e3b10a0c0251f1652647e9437533414", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/net/java/dev/jna/jna/4.1.0/jna-4.1.0-sources.jar"} , "name": "net_java_dev_jna_jna", "actual": "@net_java_dev_jna_jna//jar", "bind": "jar/net/java/dev/jna/jna"}, + {"artifact": "net.lingala.zip4j:zip4j:1.3.2", "lang": "java", "sha1": "4ba84e98ee017b74cb52f45962f929a221f3074c", "sha256": "c67098d430c574311432728ebd4c7c45672f9ccf5c64702eb6afb8816c22ad08", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/net/lingala/zip4j/zip4j/1.3.2/zip4j-1.3.2.jar", "source": {"sha1": "b2a62723195cf14964e7937f60da7c4daa0f61be", "sha256": "a55009fecaf47d8c4b01a30823de8d8ede3b4df6cf388e348687e0d2a9b0a143", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/net/lingala/zip4j/zip4j/1.3.2/zip4j-1.3.2-sources.jar"} , "name": "net_lingala_zip4j_zip4j", "actual": "@net_lingala_zip4j_zip4j//jar", "bind": "jar/net/lingala/zip4j/zip4j"}, + {"artifact": "net.sf.saxon:Saxon-HE:9.9.0-2", "lang": "java", "sha1": "c3437344acc4d2f1b5d1a638166e6ce52b300595", "sha256": "34661e9c6dd69caff14ec31e3ec47a9e7dbbc08be9da2c6326daf4e0038ed97c", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/net/sf/saxon/Saxon-HE/9.9.0-2/Saxon-HE-9.9.0-2.jar", "source": {"sha1": "294ea53dfb38f001372deee809c375e8736f16f4", "sha256": "b229a09b43c67690eeba0344840cd3a053bb1946ebcd668b9764ef454ed7d1cf", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/net/sf/saxon/Saxon-HE/9.9.0-2/Saxon-HE-9.9.0-2-sources.jar"} , "name": "net_sf_saxon_Saxon_HE", "actual": "@net_sf_saxon_Saxon_HE//jar", "bind": "jar/net/sf/saxon/Saxon_HE"}, + {"artifact": "org.antlr:antlr4-runtime:4.7.1", "lang": "java", "sha1": "946f8aa9daa917dd81a8b818111bec7e288f821a", "sha256": "43516d19beae35909e04d06af6c0c58c17bc94e0070c85e8dc9929ca640dc91d", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/antlr/antlr4-runtime/4.7.1/antlr4-runtime-4.7.1.jar", "source": {"sha1": "1e68e18aa14f3229b95820d354a594846134af38", "sha256": "a33d52d0d64e68c60d5e3ae2c1098fe7200d57cff59032c19930fd9d487fc7d4", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/antlr/antlr4-runtime/4.7.1/antlr4-runtime-4.7.1-sources.jar"} , "name": "org_antlr_antlr4_runtime", "actual": "@org_antlr_antlr4_runtime//jar", "bind": "jar/org/antlr/antlr4_runtime"}, + {"artifact": "org.apache.ant:ant-launcher:1.7.0", "lang": "java", "sha1": "e7e30789211e074aa70ef3eaea59bd5b22a7fa7a", "sha256": "72b3d03e0d7d86a56513ec38dd4cd6abe3da6620189be222ab255352cb6eba4a", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/apache/ant/ant-launcher/1.7.0/ant-launcher-1.7.0.jar", "name": "org_apache_ant_ant_launcher", "actual": "@org_apache_ant_ant_launcher//jar", "bind": "jar/org/apache/ant/ant_launcher"}, + {"artifact": "org.apache.ant:ant:1.7.0", "lang": "java", "sha1": "9746af1a485e50cf18dcb232489032a847067066", "sha256": "92f72307e7440f1e352c916f2438d2bbab3ffd2cf730c71316117ad04abadea8", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/apache/ant/ant/1.7.0/ant-1.7.0.jar", "source": {"sha1": "58ec0e378f1d33cb72a92ee3a9442909789090db", "sha256": "81e4de3e9e9d254b010809d679f902e7703eb4056097641861c07687e3f1fecf", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/apache/ant/ant/1.7.0/ant-1.7.0-sources.jar"} , "name": "org_apache_ant_ant", "actual": "@org_apache_ant_ant//jar", "bind": "jar/org/apache/ant/ant"}, + {"artifact": "org.apache.commons:commons-lang3:3.4", "lang": "java", "sha1": "5fe28b9518e58819180a43a850fbc0dd24b7c050", "sha256": "734c8356420cc8e30c795d64fd1fcd5d44ea9d90342a2cc3262c5158fbc6d98b", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/apache/commons/commons-lang3/3.4/commons-lang3-3.4.jar", "source": {"sha1": "b49dafc9cfef24c356827f322e773e7c26725dd2", "sha256": "4709f16a9e0f8fd83ae155083d63044e23045aac8f6f0183a2db09f492491b12", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/apache/commons/commons-lang3/3.4/commons-lang3-3.4-sources.jar"} , "name": "org_apache_commons_commons_lang3", "actual": "@org_apache_commons_commons_lang3//jar", "bind": "jar/org/apache/commons/commons_lang3"}, + {"artifact": "org.apache.commons:commons-pool2:2.3", "lang": "java", "sha1": "62a559a025fd890c30364296ece14643ba9c8c5b", "sha256": "3e794e75be78f63cc31b6676697e45f07085903a1039169459132513d85c62ae", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/apache/commons/commons-pool2/2.3/commons-pool2-2.3.jar", "source": {"sha1": "9e03327e766c3d16ed34a617818ab082192e83a7", "sha256": "3b75f7ce4fa929928b4a6e48d23779ce6065e0965c1fb934b5a6c5a54c8e84ea", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/apache/commons/commons-pool2/2.3/commons-pool2-2.3-sources.jar"} , "name": "org_apache_commons_commons_pool2", "actual": "@org_apache_commons_commons_pool2//jar", "bind": "jar/org/apache/commons/commons_pool2"}, + {"artifact": "org.beanshell:bsh:2.0b4", "lang": "java", "sha1": "a05f0a0feefa8d8467ac80e16e7de071489f0d9c", "sha256": "91395c07885839a8c6986d5b7c577cd9bacf01bf129c89141f35e8ea858427b6", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/beanshell/bsh/2.0b4/bsh-2.0b4.jar", "name": "org_beanshell_bsh", "actual": "@org_beanshell_bsh//jar", "bind": "jar/org/beanshell/bsh"}, + {"artifact": "org.checkerframework:checker-qual:2.5.2", "lang": "java", "sha1": "cea74543d5904a30861a61b4643a5f2bb372efc4", "sha256": "64b02691c8b9d4e7700f8ee2e742dce7ea2c6e81e662b7522c9ee3bf568c040a", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar", "source": {"sha1": "ebb8ebccd42218434674f3e1d9022c13df1c19f8", "sha256": "821c5c63a6f156a3bb498c5bbb613580d9d8f4134131a5627d330fc4018669d2", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2-sources.jar"} , "name": "org_checkerframework_checker_qual", "actual": "@org_checkerframework_checker_qual//jar", "bind": "jar/org/checkerframework/checker_qual"}, + {"artifact": "org.codehaus.mojo:animal-sniffer-annotations:1.17", "lang": "java", "sha1": "f97ce6decaea32b36101e37979f8b647f00681fb", "sha256": "92654f493ecfec52082e76354f0ebf87648dc3d5cec2e3c3cdb947c016747a53", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar", "source": {"sha1": "8fb5b5ad9c9723951b9fccaba5bb657fa6064868", "sha256": "2571474a676f775a8cdd15fb9b1da20c4c121ed7f42a5d93fca0e7b6e2015b40", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17-sources.jar"} , "name": "org_codehaus_mojo_animal_sniffer_annotations", "actual": "@org_codehaus_mojo_animal_sniffer_annotations//jar", "bind": "jar/org/codehaus/mojo/animal_sniffer_annotations"}, + {"artifact": "org.hamcrest:hamcrest-core:1.1", "lang": "java", "sha1": "860340562250678d1a344907ac75754e259cdb14", "sha256": "0361d1493ff0d94f861359efea91200720635072134792efb7217f6e09d5cffb", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/hamcrest/hamcrest-core/1.1/hamcrest-core-1.1.jar", "source": {"sha1": "2ccf1154d1a8936042a8a742dc3e611d02ac7213", "sha256": "bcc20dca5b6f9bd6a6f7611b4d5734a75f5e4ede272ed3e1b156669d44e493d6", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/hamcrest/hamcrest-core/1.1/hamcrest-core-1.1-sources.jar"} , "name": "org_hamcrest_hamcrest_core", "actual": "@org_hamcrest_hamcrest_core//jar", "bind": "jar/org/hamcrest/hamcrest_core"}, + {"artifact": "org.ini4j:ini4j:0.5.2", "lang": "java", "sha1": "16561cb11c221b5928119e10d7636c95ee5c960d", "sha256": "631656eb38639b0ae41161f706ff7fbe04313b5b8f42892da5ec656390031fc6", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/ini4j/ini4j/0.5.2/ini4j-0.5.2.jar", "source": {"sha1": "17a383f897ae0ec8fca2f3effc2a2b8dbb336fc2", "sha256": "10f11429b7bfc298ed776e6d1ed8a9365f73a1df90737cd178169cd5b7849ba7", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/ini4j/ini4j/0.5.2/ini4j-0.5.2-sources.jar"} , "name": "org_ini4j_ini4j", "actual": "@org_ini4j_ini4j//jar", "bind": "jar/org/ini4j/ini4j"}, + {"artifact": "org.javassist:javassist:3.19.0-GA", "lang": "java", "sha1": "50120f69224dd8684b445a6f3a5b08fe9b5c60f6", "sha256": "d19c1ef43ccd9cb1b39466bb2f1c8e45c2b6752f1e13a3dfb60096543d1791fa", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/javassist/javassist/3.19.0-GA/javassist-3.19.0-GA.jar", "source": {"sha1": "37e6508aeea3eef21df5b7ee3ea3c708618c1b7b", "sha256": "26690d98e9b005eecebf0eba2b7331b8f7cb9f3d3d9e41975a41574cfe83a656", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/javassist/javassist/3.19.0-GA/javassist-3.19.0-GA-sources.jar"} , "name": "org_javassist_javassist", "actual": "@org_javassist_javassist//jar", "bind": "jar/org/javassist/javassist"}, + {"artifact": "org.mockito:mockito-all:1.10.19", "lang": "java", "sha1": "539df70269cc254a58cccc5d8e43286b4a73bf30", "sha256": "d1a7a7ef14b3db5c0fc3e0a63a81b374b510afe85add9f7984b97911f4c70605", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar", "source": {"sha1": "8269667b73d9616600359a9b0ba1b1c7d0cf7a97", "sha256": "6c8680afe9cfdb6e0fe40aca1e27ded721252bd53107390f166f2077b9e5b16e", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/mockito/mockito-all/1.10.19/mockito-all-1.10.19-sources.jar"} , "name": "org_mockito_mockito_all", "actual": "@org_mockito_mockito_all//jar", "bind": "jar/org/mockito/mockito_all"}, + {"artifact": "org.objenesis:objenesis:2.4", "lang": "java", "sha1": "2916b6c96b50c5b3ec4452ed99401db745aabb27", "sha256": "090416e160a7e398f79e0149a7b05ef328b1843898a34e08c1cc170128d326d0", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/objenesis/objenesis/2.4/objenesis-2.4.jar", "source": {"sha1": "19b316d8da7da3ae2539ee4ee2cf73a1e127fa73", "sha256": "f5d6ea639bb9d1e7260447ad9a7bb9e14e16a7c98912db3c401c8468a9382871", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/objenesis/objenesis/2.4/objenesis-2.4-sources.jar"} , "name": "org_objenesis_objenesis", "actual": "@org_objenesis_objenesis//jar", "bind": "jar/org/objenesis/objenesis"}, + {"artifact": "org.ow2.asm:asm:6.0", "lang": "java", "sha1": "bc6fa6b19424bb9592fe43bbc20178f92d403105", "sha256": "dd8971c74a4e697899a8e95caae4ea8760ea6c486dc6b97b1795e75760420461", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/ow2/asm/asm/6.0/asm-6.0.jar", "source": {"sha1": "066e3ec9f4ff0d70713868456142040111cc201c", "sha256": "0687933fe7b7989ec5110946b5792a01784011fe7cbfe555f7e416da7984a1ca", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/ow2/asm/asm/6.0/asm-6.0-sources.jar"} , "name": "org_ow2_asm_asm", "actual": "@org_ow2_asm_asm//jar", "bind": "jar/org/ow2/asm/asm"}, + {"artifact": "org.slf4j:slf4j-api:1.7.25", "lang": "java", "sha1": "da76ca59f6a57ee3102f8f9bd9cee742973efa8a", "sha256": "18c4a0095d5c1da6b817592e767bb23d29dd2f560ad74df75ff3961dbde25b79", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar", "source": {"sha1": "962153db4a9ea71b79d047dfd1b2a0d80d8f4739", "sha256": "c4bc93180a4f0aceec3b057a2514abe04a79f06c174bbed910a2afb227b79366", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25-sources.jar"} , "name": "org_slf4j_slf4j_api", "actual": "@org_slf4j_slf4j_api//jar", "bind": "jar/org/slf4j/slf4j_api"}, + {"artifact": "org.slf4j:slf4j-log4j12:1.7.25", "lang": "java", "sha1": "110cefe2df103412849d72ef7a67e4e91e4266b4", "sha256": "ddb343954deb6f046f862606c534178730c02ed23d0b7f6ca1012c1e3fa74273", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/slf4j/slf4j-log4j12/1.7.25/slf4j-log4j12-1.7.25.jar", "source": {"sha1": "d6f907c4254a49f40fa46005b65bc97261ad9e46", "sha256": "3c57ac91d961ab2b68b5246275ddc8d6fc700eadc8deb113b2d353176112e0c8", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/slf4j/slf4j-log4j12/1.7.25/slf4j-log4j12-1.7.25-sources.jar"} , "name": "org_slf4j_slf4j_log4j12", "actual": "@org_slf4j_slf4j_log4j12//jar", "bind": "jar/org/slf4j/slf4j_log4j12"}, + {"artifact": "org.testng:testng:6.9.9", "lang": "java", "sha1": "1bf509349476d6a48978cc2b04af9caa907781ab", "sha256": "e89d524ca6dba6ea6c9619c2537f4669524eb9306bb6fd9b7a52240cb3b4b690", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/testng/testng/6.9.9/testng-6.9.9.jar", "source": {"sha1": "045c07e00b4cea80ac8715deb749a54d784f7cb6", "sha256": "304face92c85dcf3418be273713499cadc8a10c2735852496f69a0e9faf55b57", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/testng/testng/6.9.9/testng-6.9.9-sources.jar"} , "name": "org_testng_testng", "actual": "@org_testng_testng//jar", "bind": "jar/org/testng/testng"}, + {"artifact": "org.yaml:snakeyaml:1.15", "lang": "java", "sha1": "3b132bea69e8ee099f416044970997bde80f4ea6", "sha256": "79ea8aac6590f49ee8390c2f17ed9343079e85b44158a097b301dfee42af86ec", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/yaml/snakeyaml/1.15/snakeyaml-1.15.jar", "source": {"sha1": "7dc55271cad1cf73bb495e372164269e75d9345a", "sha256": "cd92585a477bf81e3c8f2bcbfff182dd67999fdc8fd880c3a1531c0626c7a4ee", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/org/yaml/snakeyaml/1.15/snakeyaml-1.15-sources.jar"} , "name": "org_yaml_snakeyaml", "actual": "@org_yaml_snakeyaml//jar", "bind": "jar/org/yaml/snakeyaml"}, + {"artifact": "redis.clients:jedis:2.8.0", "lang": "java", "sha1": "a9486e200b2195f31ce9b8779a70a5e2450a73ba", "sha256": "e8194b42cbdbe990f6cc1ec7a91a6b553d901abaf155dd6cc599139eb3f5d68d", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/redis/clients/jedis/2.8.0/jedis-2.8.0.jar", "source": {"sha1": "356324b58a8d6dfaeb6ddd6289f3a972551c915b", "sha256": "c71f6d176af72617a18bb685f4a658fa0ea6f35f872d9d073b29792e842541ca", "repository": "https://repo.maven.apache.org/maven2/", "url": "https://repo.maven.apache.org/maven2/redis/clients/jedis/2.8.0/jedis-2.8.0-sources.jar"} , "name": "redis_clients_jedis", "actual": "@redis_clients_jedis//jar", "bind": "jar/redis/clients/jedis"}, + ] + +def maven_dependencies(callback = jar_artifact_callback): + for hash in list_dependencies(): + callback(hash) diff --git a/src/ray/test/run_gcs_tests.sh b/src/ray/test/run_gcs_tests.sh index b564f5c7e1a3..2aaf88da72c9 100644 --- a/src/ray/test/run_gcs_tests.sh +++ b/src/ray/test/run_gcs_tests.sh @@ -6,7 +6,7 @@ set -e set -x -bazel build "//:gcs_client_test" "//:asio_test" "//:ray_redis_module.so" -c opt +bazel build "//:gcs_client_test" "//:asio_test" "//:libray_redis_module.so" # Start Redis. if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then @@ -16,9 +16,9 @@ if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then --loadmodule ./src/ray/gcs/redis_module/libray_redis_module.so \ --port 6379 & else - ./bazel-genfiles/ray_pkg/ray/core/src/ray/thirdparty/redis/src/redis-server \ + ./bazel-genfiles/redis-server \ --loglevel warning \ - --loadmodule ./bazel-bin/ray_redis_module.so \ + --loadmodule ./bazel-bin/libray_redis_module.so \ --port 6379 & fi sleep 1s @@ -26,5 +26,5 @@ sleep 1s ./bazel-bin/gcs_client_test ./bazel-bin/asio_test -./bazel-genfiles/ray_pkg/ray/core/src/ray/thirdparty/redis/src/redis-cli -p 6379 shutdown +./bazel-genfiles/redis-cli -p 6379 shutdown sleep 1s diff --git a/src/ray/test/run_object_manager_tests.sh b/src/ray/test/run_object_manager_tests.sh index 37fe59f0aa30..a739f6eeec0a 100644 --- a/src/ray/test/run_object_manager_tests.sh +++ b/src/ray/test/run_object_manager_tests.sh @@ -6,7 +6,7 @@ set -e set -x -bazel build -c opt "//:object_manager_stress_test" "//:object_manager_test" "@plasma//:plasma_store_server" +bazel build "//:object_manager_stress_test" "//:object_manager_test" "@plasma//:plasma_store_server" # Get the directory in which this script is executing. SCRIPT_DIR="`dirname \"$0\"`" @@ -22,21 +22,21 @@ if [ ! -d "$RAY_ROOT/python" ]; then exit 1 fi -REDIS_MODULE="./bazel-genfiles/ray_pkg/ray/core/src/ray/gcs/redis_module/libray_redis_module.so" +REDIS_MODULE="./bazel-bin/libray_redis_module.so" LOAD_MODULE_ARGS="--loadmodule ${REDIS_MODULE}" STORE_EXEC="./bazel-bin/external/plasma/plasma_store_server" # Allow cleanup commands to fail. -bazel run -c opt //:redis-cli -- -p 6379 shutdown || true +bazel run //:redis-cli -- -p 6379 shutdown || true sleep 1s -bazel run -c opt //:redis-server -- --loglevel warning ${LOAD_MODULE_ARGS} --port 6379 & +bazel run //:redis-server -- --loglevel warning ${LOAD_MODULE_ARGS} --port 6379 & sleep 1s # Run tests. ./bazel-bin/object_manager_stress_test $STORE_EXEC sleep 1s # Use timeout=1000ms for the Wait tests. ./bazel-bin/object_manager_test $STORE_EXEC 1000 -bazel run -c opt //:redis-cli -- -p 6379 shutdown +bazel run //:redis-cli -- -p 6379 shutdown sleep 1s # Include raylet integration test once it's ready. diff --git a/src/ray/test/run_object_manager_valgrind.sh b/src/ray/test/run_object_manager_valgrind.sh index f7e708e30401..1fcdb77f367b 100644 --- a/src/ray/test/run_object_manager_valgrind.sh +++ b/src/ray/test/run_object_manager_valgrind.sh @@ -6,7 +6,7 @@ set -e set -x -bazel build -c opt "//:object_manager_stress_test" "//:object_manager_test" "@plasma//:plasma_store_server" +bazel build "//:object_manager_stress_test" "//:object_manager_test" "@plasma//:plasma_store_server" # Get the directory in which this script is executing. SCRIPT_DIR="`dirname \"$0\"`" @@ -22,7 +22,7 @@ if [ ! -d "$RAY_ROOT/python" ]; then exit 1 fi -REDIS_MODULE="./bazel-genfiles/ray_pkg/ray/core/src/ray/gcs/redis_module/libray_redis_module.so" +REDIS_MODULE="./bazel-bin/libray_redis_module.so" LOAD_MODULE_ARGS="--loadmodule ${REDIS_MODULE}" STORE_EXEC="./bazel-bin/external/plasma/plasma_store_server" @@ -30,9 +30,9 @@ VALGRIND_CMD="valgrind --track-origins=yes --leak-check=full --show-leak-kinds=a # Allow cleanup commands to fail. killall plasma_store || true -bazel run -c opt //:redis-cli -- -p 6379 shutdown || true +bazel run //:redis-cli -- -p 6379 shutdown || true sleep 1s -bazel run -c opt //:redis-server -- --loglevel warning ${LOAD_MODULE_ARGS} --port 6379 & +bazel run //:redis-server -- --loglevel warning ${LOAD_MODULE_ARGS} --port 6379 & sleep 1s # Run tests. Use timeout=10000ms for the Wait tests since tests run slower @@ -40,7 +40,7 @@ sleep 1s $VALGRIND_CMD ./bazel-bin/object_manager_test $STORE_EXEC 10000 sleep 1s $VALGRIND_CMD ./bazel-bin/object_manager_stress_test $STORE_EXEC -bazel run -c opt //:redis-cli -- -p 6379 shutdown +bazel run //:redis-cli -- -p 6379 shutdown sleep 1s # Include raylet integration test once it's ready.