Align package dependencies to iceberg and revert iceberg back to 1.4.3 (

#276) Align package dependencies to iceberg - Align Iceberg spark runtime - Align AWS and GCP dependencies - And Jackson dependency
memiiso · Mar 1, 2024 · 9a3fb24 · 9a3fb24
1 parent 71fc531
commit 9a3fb24
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 11 deletions.
diff --git a/debezium-server-iceberg-sink/pom.xml b/debezium-server-iceberg-sink/pom.xml
@@ -50,7 +50,7 @@
         <!-- Iceberg  -->
         <dependency>
             <groupId>org.apache.iceberg</groupId>
-            <artifactId>iceberg-spark-runtime-3.3_2.13</artifactId>
+            <artifactId>iceberg-spark-runtime-${version.spark.major}_${version.spark.scala}</artifactId>
             <version>${version.iceberg}</version>
         </dependency>
         <dependency>
@@ -87,22 +87,20 @@
         <dependency>
             <groupId>com.google.cloud</groupId>
             <artifactId>google-cloud-storage</artifactId>
-            <version>2.22.1</version>
         </dependency>
         <dependency>
             <groupId>com.google.cloud</groupId>
             <artifactId>google-cloud-nio</artifactId>
-            <version>0.127.7</version>
         </dependency>
         <dependency>
             <groupId>com.google.cloud.bigdataoss</groupId>
             <artifactId>gcs-connector</artifactId>
-            <version>hadoop3-2.2.8</version>
+            <version>hadoop3-${version.googlebigdataoss}</version>
         </dependency>
         <dependency>
             <groupId>com.google.cloud.bigdataoss</groupId>
             <artifactId>gcsio</artifactId>
-            <version>2.2.16</version>
+            <version>${version.googlebigdataoss}</version>
         </dependency>
         <!-- AWS  -->
         <dependency>
@@ -146,6 +144,7 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <!-- aws-java-sdk-bundle Test dependency -->
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-java-sdk-bundle</artifactId>
@@ -203,7 +202,7 @@
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
-            <artifactId>spark-core_2.13</artifactId>
+            <artifactId>spark-core_${version.spark.scala}</artifactId>
             <version>${version.spark}</version>
             <scope>test</scope>
             <exclusions>
@@ -227,7 +226,7 @@
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
-            <artifactId>spark-sql_2.13</artifactId>
+            <artifactId>spark-sql_${version.spark.scala}</artifactId>
             <version>${version.spark}</version>
             <scope>test</scope>
         </dependency>

diff --git a/pom.xml b/pom.xml
@@ -31,11 +31,19 @@
         <version.assembly.plugin>3.6.0</version.assembly.plugin>
         <!-- Use same version as iceberg https://github.com/apache/iceberg/blob/main/gradle/libs.versions.toml#L53-->
         <version.jackson>2.14.2</version.jackson>
-        <version.iceberg>1.5.0</version.iceberg>
-        <version.spark>3.3.4</version.spark>
+        <version.iceberg>1.4.2</version.iceberg>
+        <!-- Following two properties defines which version of iceberg-spark-runtime is used -->
+        <!-- Example https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-spark-runtime-3.4_2.13/1.4.3 -->
+        <version.spark.major>3.5</version.spark.major>
+        <version.spark.scala>2.13</version.spark.scala>
+        <version.spark>${version.spark.major}.1</version.spark>
         <version.hadoop>3.3.6</version.hadoop>
         <version.hive>3.1.3</version.hive>
-        <version.awssdk>2.22.9</version.awssdk>
+        <!-- Use same version as iceberg https://github.com/apache/iceberg/blob/main/gradle/libs.versions.toml#L31-->  
+        <version.awssdk>2.24.5</version.awssdk>
+        <!-- Use same version as iceberg https://github.com/apache/iceberg/blob/main/gradle/libs.versions.toml#L44-->  
+        <version.googlelibraries>26.28.0</version.googlelibraries>
+        <version.googlebigdataoss>2.2.20</version.googlebigdataoss>
         <version.testcontainers>1.19.3</version.testcontainers>
         <!-- Debezium -->
         <version.debezium>2.5.2.Final</version.debezium>
@@ -81,7 +89,14 @@
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
-
+            <!-- gcp -->
+            <dependency>
+                <groupId>com.google.cloud</groupId>
+                <artifactId>libraries-bom</artifactId>
+                <version>${version.googlelibraries}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
             <!-- MySQL JDBC Driver, Binlog reader, Geometry support -->
             <dependency>
                 <groupId>mysql</groupId>