diff --git a/thirdparty/scripts/build_arrow.sh b/thirdparty/scripts/build_arrow.sh
index 7640a9ee1462..b3ba49d52b5e 100755
--- a/thirdparty/scripts/build_arrow.sh
+++ b/thirdparty/scripts/build_arrow.sh
@@ -41,9 +41,16 @@ else
 fi
 
 # Download and compile arrow if it isn't already present.
-if [[ ! -d $TP_DIR/../python/ray/pyarrow_files/pyarrow ]]; then
+if [[ ! -d $TP_DIR/../python/ray/pyarrow_files/pyarrow || \
+  "$LANGUAGE" == "java" && ! -f $TP_DIR/build/arrow/cpp/build/release/libplasma_java.* ]]; then
     echo "building arrow"
 
+    if [[ "$LANGUAGE" == "java" && ! -f $TP_DIR/build/arrow/cpp/build/release/libplasma_java.* ]]; then
+      rm -rf $TP_DIR/build/arrow
+      rm -rf $TP_DIR/pkg/arrow
+      rm -rf $TP_DIR/build/parquet-cpp
+    fi
+
     if [[ ! -d $TP_DIR/build/arrow ]]; then
       git clone https://github.com/apache/arrow.git "$TP_DIR/build/arrow"
     fi