diff --git a/ci/build/build-manylinux-ray.sh b/ci/build/build-manylinux-ray.sh
index f7c84add0751..477d69ede103 100755
--- a/ci/build/build-manylinux-ray.sh
+++ b/ci/build/build-manylinux-ray.sh
@@ -8,7 +8,7 @@ fi
 
 # Build ray java
 if [[ "${RAY_INSTALL_JAVA}" == "1" ]]; then
-  bazel build //java:ray_java_pkg
+  bazel run //java:gen_ray_java_pkg
 fi
 
 export PATH="/usr/local/node/bin:$PATH"
diff --git a/ci/docker/ray-java.Dockerfile b/ci/docker/ray-java.Dockerfile
index 0b09e8328a3a..d45ad8aef6ec 100644
--- a/ci/docker/ray-java.Dockerfile
+++ b/ci/docker/ray-java.Dockerfile
@@ -20,7 +20,7 @@ if [[ "${BUILDKITE_CACHE_READONLY:-}" == "true" ]]; then
   echo "build --remote_upload_local_results=false" >> "$HOME/.bazelrc"
 fi
 
-bazelisk build --config=ci //java:ray_java_pkg
+bazelisk run --config=ci //java:gen_ray_java_pkg
 
 cp bazel-bin/java/ray_java_pkg.zip /home/forge/ray_java_pkg.zip
 
diff --git a/java/BUILD.bazel b/java/BUILD.bazel
index d9cefe4ec28f..38307e930d4d 100644
--- a/java/BUILD.bazel
+++ b/java/BUILD.bazel
@@ -231,7 +231,6 @@ java_test(
     args = ["java/testng.xml"],
     data = [
         "testng.xml",
-        ":ray_java_pkg",
         "//:ray_pkg_zip",
     ],
     main_class = "org.testng.TestNG",
@@ -254,7 +253,6 @@ java_test(
     args = ["java/testng_custom.xml"],
     data = [
         "testng_custom.xml",
-        ":ray_java_pkg",
         "//:ray_pkg_zip",
     ],
     main_class = "org.testng.TestNG",
@@ -497,24 +495,6 @@ pkg_zip(
     visibility = ["//visibility:private"],
 )
 
-genrule(
-    name = "ray_java_pkg",
-    srcs = [
-        "ray_java_pkg.zip",
-        ":gen_maven_deps",
-    ],
-    outs = ["ray_java_pkg.out"],
-    cmd = """
-        WORK_DIR="$$(pwd)" &&
-        rm -rf "$$WORK_DIR/python/ray/jars" && mkdir -p "$$WORK_DIR/python/ray/jars" &&
-        unzip -q $(location ray_java_pkg.zip) -d "$$WORK_DIR/python" &&
-        if [[ "$$OSTYPE" =~ ^darwin ]]; then shasum $(SRCS) > $@ ; else sha1sum $(SRCS) > $@ ; fi
-    """,
-    local = 1,
-    tags = ["no-cache"],
-    visibility = ["//visibility:private"],
-)
-
 py_binary(
     name = "gen_ray_java_pkg",
     srcs = ["gen_ray_java_pkg.py"],