From 618b50bc49014eca41cbf63f09f4ffd3de6a282a Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Fri, 19 Jan 2024 16:03:08 +0800
Subject: [PATCH 1/4] init

---
 dev/deps/spark-deps-hadoop-3-hive-2.3 | 12 +++++++-----
 pom.xml                               | 13 ++++++++++++-
 project/SparkBuild.scala              | 11 +++++++----
 3 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3
index 6220626069af..4ee0f5a41191 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -16,10 +16,10 @@ antlr4-runtime/4.13.1//antlr4-runtime-4.13.1.jar
 aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar
 arpack/3.0.3//arpack-3.0.3.jar
 arpack_combined_all/0.1//arpack_combined_all-0.1.jar
-arrow-format/14.0.2//arrow-format-14.0.2.jar
-arrow-memory-core/14.0.2//arrow-memory-core-14.0.2.jar
-arrow-memory-netty/14.0.2//arrow-memory-netty-14.0.2.jar
-arrow-vector/14.0.2//arrow-vector-14.0.2.jar
+arrow-format/15.0.0//arrow-format-15.0.0.jar
+arrow-memory-core/15.0.0//arrow-memory-core-15.0.0.jar
+arrow-memory-netty/15.0.0//arrow-memory-netty-15.0.0.jar
+arrow-vector/15.0.0//arrow-vector-15.0.0.jar
 audience-annotations/0.12.0//audience-annotations-0.12.0.jar
 avro-ipc/1.11.3//avro-ipc-1.11.3.jar
 avro-mapred/1.11.3//avro-mapred-1.11.3.jar
@@ -63,7 +63,9 @@ derby/10.16.1.1//derby-10.16.1.1.jar
 derbyshared/10.16.1.1//derbyshared-10.16.1.1.jar
 derbytools/10.16.1.1//derbytools-10.16.1.1.jar
 dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
-flatbuffers-java/1.12.0//flatbuffers-java-1.12.0.jar
+eclipse-collections-api/11.1.0//eclipse-collections-api-11.1.0.jar
+eclipse-collections/11.1.0//eclipse-collections-11.1.0.jar
+flatbuffers-java/23.5.26//flatbuffers-java-23.5.26.jar
 gcs-connector/hadoop3-2.2.18/shaded/gcs-connector-hadoop3-2.2.18-shaded.jar
 gmetric4j/1.0.10//gmetric4j-1.0.10.jar
 gson/2.2.4//gson-2.2.4.jar
diff --git a/pom.xml b/pom.xml
index 3eb8b0917bf5..767dbcd92645 100644
--- a/pom.xml
+++ b/pom.xml
@@ -230,7 +230,7 @@
     If you are changing Arrow version specification, please check
     ./python/pyspark/sql/pandas/utils.py, and ./python/setup.py too.
     -->
-    <arrow.version>14.0.2</arrow.version>
+    <arrow.version>15.0.0</arrow.version>
     <ammonite.version>2.5.11</ammonite.version>
 
     <!-- org.fusesource.leveldbjni will be used except on arm64 platform. -->
@@ -322,6 +322,17 @@
     </extraJavaTestArgs>
   </properties>
   <repositories>
+    <repository>
+      <id>asf-staging</id>
+      <name>ASF</name>
+      <url>https://repository.apache.org/content/groups/staging/</url>
+      <releases>
+        <enabled>true</enabled>
+      </releases>
+      <snapshots>
+        <enabled>false</enabled>
+      </snapshots>
+    </repository>
     <repository>
       <id>gcs-maven-central-mirror</id>
       <!--
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 5523932ac0ad..0081c225898c 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -269,6 +269,7 @@ object SparkBuild extends PomBuild {
     resolvers := Seq(
       // Google Mirror of Maven Central, placed first so that it's used instead of flaky Maven Central.
       // See https://storage-download.googleapis.com/maven-central/index.html for more info.
+      "asf-staging" at "https://repository.apache.org/content/groups/staging/",
       "gcs-maven-central-mirror" at "https://maven-central.storage-download.googleapis.com/maven2/",
       DefaultMavenRepository,
       Resolver.mavenLocal,
@@ -756,10 +757,12 @@ object SparkConnect {
     // `netty-*.jar` and `unused-1.0.0.jar` from assembly.
     (assembly / assemblyExcludedJars) := {
       val cp = (assembly / fullClasspath).value
-      cp filter { v =>
-        val name = v.data.getName
-        name.startsWith("pmml-model-") || name.startsWith("scala-collection-compat_") ||
-          name.startsWith("jsr305-") || name.startsWith("netty-") || name == "unused-1.0.0.jar"
+      val validPrefixes = Set("spark-connect", "unused-", "guava-", "failureaccess-",
+        "annotations-", "grpc-", "protobuf-", "gson", "error_prone_annotations",
+        "j2objc-annotations", "animal-sniffer-annotations", "perfmark-api",
+        "proto-google-common-protos")
+      cp filterNot { v =>
+        validPrefixes.exists(v.data.getName.startsWith)
       }
     },
 

From 820536ffe4725cc52917695e935c6c58efea146d Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Mon, 22 Jan 2024 13:46:16 +0800
Subject: [PATCH 2/4] remove staging

---
 pom.xml                  | 11 -----------
 project/SparkBuild.scala |  1 -
 2 files changed, 12 deletions(-)

diff --git a/pom.xml b/pom.xml
index ade150456992..5f33dd7d8ebc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -322,17 +322,6 @@
     </extraJavaTestArgs>
   </properties>
   <repositories>
-    <repository>
-      <id>asf-staging</id>
-      <name>ASF</name>
-      <url>https://repository.apache.org/content/groups/staging/</url>
-      <releases>
-        <enabled>true</enabled>
-      </releases>
-      <snapshots>
-        <enabled>false</enabled>
-      </snapshots>
-    </repository>
     <repository>
       <id>gcs-maven-central-mirror</id>
       <!--
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 9fcafcb8e4c1..3123f931cd88 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -269,7 +269,6 @@ object SparkBuild extends PomBuild {
     resolvers := Seq(
       // Google Mirror of Maven Central, placed first so that it's used instead of flaky Maven Central.
       // See https://storage-download.googleapis.com/maven-central/index.html for more info.
-      "asf-staging" at "https://repository.apache.org/content/groups/staging/",
       "gcs-maven-central-mirror" at "https://maven-central.storage-download.googleapis.com/maven2/",
       DefaultMavenRepository,
       Resolver.mavenLocal,

From b702f40fa0135069bb7d661441df48031c6ef1c6 Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Mon, 22 Jan 2024 16:03:05 +0800
Subject: [PATCH 3/4] exclude eclipse-collections

---
 dev/deps/spark-deps-hadoop-3-hive-2.3 | 2 --
 pom.xml                               | 4 ++--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3
index 4ee0f5a41191..31f16024c29e 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -63,8 +63,6 @@ derby/10.16.1.1//derby-10.16.1.1.jar
 derbyshared/10.16.1.1//derbyshared-10.16.1.1.jar
 derbytools/10.16.1.1//derbytools-10.16.1.1.jar
 dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
-eclipse-collections-api/11.1.0//eclipse-collections-api-11.1.0.jar
-eclipse-collections/11.1.0//eclipse-collections-11.1.0.jar
 flatbuffers-java/23.5.26//flatbuffers-java-23.5.26.jar
 gcs-connector/hadoop3-2.2.18/shaded/gcs-connector-hadoop3-2.2.18-shaded.jar
 gmetric4j/1.0.10//gmetric4j-1.0.10.jar
diff --git a/pom.xml b/pom.xml
index 5f33dd7d8ebc..b077a999042a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2790,8 +2790,8 @@
             <artifactId>jackson-core</artifactId>
           </exclusion>
           <exclusion>
-            <groupId>io.netty</groupId>
-            <artifactId>netty-common</artifactId>
+            <groupId>org.eclipse.collections</groupId>
+            <artifactId>eclipse-collections</artifactId>
           </exclusion>
         </exclusions>
       </dependency>

From 951b7e57034e65be21eaf8b6717c3881ffa38fe5 Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Mon, 22 Jan 2024 17:04:12 +0800
Subject: [PATCH 4/4] Revert "exclude eclipse-collections"

This reverts commit b702f40fa0135069bb7d661441df48031c6ef1c6.
---
 dev/deps/spark-deps-hadoop-3-hive-2.3 | 2 ++
 pom.xml                               | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3
index 31f16024c29e..4ee0f5a41191 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -63,6 +63,8 @@ derby/10.16.1.1//derby-10.16.1.1.jar
 derbyshared/10.16.1.1//derbyshared-10.16.1.1.jar
 derbytools/10.16.1.1//derbytools-10.16.1.1.jar
 dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
+eclipse-collections-api/11.1.0//eclipse-collections-api-11.1.0.jar
+eclipse-collections/11.1.0//eclipse-collections-11.1.0.jar
 flatbuffers-java/23.5.26//flatbuffers-java-23.5.26.jar
 gcs-connector/hadoop3-2.2.18/shaded/gcs-connector-hadoop3-2.2.18-shaded.jar
 gmetric4j/1.0.10//gmetric4j-1.0.10.jar
diff --git a/pom.xml b/pom.xml
index b077a999042a..5f33dd7d8ebc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2790,8 +2790,8 @@
             <artifactId>jackson-core</artifactId>
           </exclusion>
           <exclusion>
-            <groupId>org.eclipse.collections</groupId>
-            <artifactId>eclipse-collections</artifactId>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-common</artifactId>
           </exclusion>
         </exclusions>
       </dependency>