From 0d5e9ceaa244085a61e4f5391ea4f4d05a8a59b8 Mon Sep 17 00:00:00 2001
From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com>
Date: Sun, 3 Mar 2024 21:36:06 +0100
Subject: [PATCH]  Switch to iceberg cloud bundle dependecies (#288)

* Use cloud bundle dependencies

* Use cloud bundle dependencies

* Use cloud bundle dependencies
---
 debezium-server-iceberg-sink/pom.xml          | 52 +++++--------------
 .../iceberg/testresources/BaseSparkTest.java  | 19 +------
 pom.xml                                       | 40 ++------------
 3 files changed, 18 insertions(+), 93 deletions(-)

diff --git a/debezium-server-iceberg-sink/pom.xml b/debezium-server-iceberg-sink/pom.xml
index f83f70cb..cfdcf96e 100644
--- a/debezium-server-iceberg-sink/pom.xml
+++ b/debezium-server-iceberg-sink/pom.xml
@@ -93,11 +93,21 @@
             <artifactId>iceberg-gcp</artifactId>
             <version>${version.iceberg}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.iceberg</groupId>
+            <artifactId>iceberg-gcp-bundle</artifactId>
+            <version>${version.iceberg}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.iceberg</groupId>
             <artifactId>iceberg-aws</artifactId>
             <version>${version.iceberg}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.iceberg</groupId>
+            <artifactId>iceberg-aws-bundle</artifactId>
+            <version>${version.iceberg}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.iceberg</groupId>
             <artifactId>iceberg-dell</artifactId>
@@ -113,15 +123,12 @@
             <artifactId>iceberg-azure</artifactId>
             <version>${version.iceberg}</version>
         </dependency>
-        <!-- Google -->
-        <dependency>
-            <groupId>com.google.cloud</groupId>
-            <artifactId>google-cloud-storage</artifactId>
-        </dependency>
         <dependency>
-            <groupId>com.google.cloud</groupId>
-            <artifactId>google-cloud-nio</artifactId>
+            <groupId>org.apache.iceberg</groupId>
+            <artifactId>iceberg-azure-bundle</artifactId>
+            <version>${version.iceberg}</version>
         </dependency>
+        <!-- Google -->
         <dependency>
             <groupId>com.google.cloud.bigdataoss</groupId>
             <artifactId>gcs-connector</artifactId>
@@ -132,37 +139,6 @@
             <artifactId>gcsio</artifactId>
             <version>${version.googlebigdataoss}</version>
         </dependency>
-        <!-- AWS  -->
-        <dependency>
-            <groupId>software.amazon.awssdk</groupId>
-            <artifactId>core</artifactId>
-            <version>${version.awssdk}</version>
-            <type>pom</type>
-        </dependency>
-        <dependency>
-            <groupId>software.amazon.awssdk</groupId>
-            <artifactId>glue</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>software.amazon.awssdk</groupId>
-            <artifactId>s3</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>software.amazon.awssdk</groupId>
-            <artifactId>dynamodb</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>software.amazon.awssdk</groupId>
-            <artifactId>kms</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>software.amazon.awssdk</groupId>
-            <artifactId>sts</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>software.amazon.awssdk</groupId>
-            <artifactId>url-connection-client</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-aws</artifactId>
diff --git a/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/testresources/BaseSparkTest.java b/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/testresources/BaseSparkTest.java
index f53131eb..f81fd743 100644
--- a/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/testresources/BaseSparkTest.java
+++ b/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/testresources/BaseSparkTest.java
@@ -47,35 +47,18 @@ static void setup() {
         .set("spark.eventLog.enabled", "false")
         // enable iceberg SQL Extensions and Catalog
         .set("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions")
+        .set("spark.sql.warehouse.dir", S3_BUCKET)
         // hadoop catalog
         .set("spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog")
         .set("spark.sql.catalog.spark_catalog.type", "hadoop")
         .set("spark.sql.catalog.spark_catalog.warehouse", S3_BUCKET)
         .set("spark.sql.catalog.spark_catalog.default-namespaces", CATALOG_TABLE_NAMESPACE)
         .set("spark.sql.catalog.spark_catalog.io-impl", "org.apache.iceberg.aws.s3.S3FileIO")
-        .set("spark.sql.warehouse.dir", S3_BUCKET)
         .set("spark.sql.catalog.spark_catalog.s3.endpoint", "http://localhost:" + S3Minio.getMappedPort().toString())
         .set("spark.sql.catalog.spark_catalog.s3.path-style-access", "true")
         .set("spark.sql.catalog.spark_catalog.s3.access-key-id", S3Minio.MINIO_ACCESS_KEY)
         .set("spark.sql.catalog.spark_catalog.s3.secret-access-key", S3Minio.MINIO_SECRET_KEY)
         .set("spark.sql.catalog.spark_catalog.client.region", TestConfigSource.S3_REGION)
-        .set("spark.sql.catalog.spark_catalog.io-impl", "org.apache.iceberg.aws.s3.S3FileIO")
-        .set("spark.sql.catalog.spark_catalog.warehouse", S3_BUCKET)
-//        // JdbcCatalog catalog, add additional catalog
-//        .set("spark.sql.defaultCatalog", ICEBERG_CATALOG_NAME)
-//        .set("spark.sql.catalog." + ICEBERG_CATALOG_NAME, "org.apache.iceberg.spark.SparkCatalog")
-//        .set("spark.sql.catalog." + ICEBERG_CATALOG_NAME + ".warehouse", S3_BUCKET)
-//        .set("spark.sql.catalog." + ICEBERG_CATALOG_NAME + ".cache-enabled", "false")
-//        .set("spark.sql.catalog." + ICEBERG_CATALOG_NAME + ".catalog-impl", JdbcCatalog.class.getName())
-//        .set("spark.sql.catalog." + ICEBERG_CATALOG_NAME + ".default-namespaces", CATALOG_TABLE_NAMESPACE)
-//        .set("spark.sql.catalog." + ICEBERG_CATALOG_NAME + ".uri", JdbcCatalogDB.container.getJdbcUrl())
-//        .set("spark.sql.catalog." + ICEBERG_CATALOG_NAME + ".jdbc.user", JdbcCatalogDB.container.getUsername())
-//        .set("spark.sql.catalog." + ICEBERG_CATALOG_NAME + ".jdbc.password", JdbcCatalogDB.container.getPassword())
-//        .set("spark.sql.catalog." + ICEBERG_CATALOG_NAME + ".io-impl", "org.apache.iceberg.aws.s3.S3FileIO")
-//        .set("spark.sql.catalog." + ICEBERG_CATALOG_NAME + ".s3.endpoint", "http://localhost:" + S3Minio.getMappedPort().toString())
-//        .set("spark.sql.catalog." + ICEBERG_CATALOG_NAME + ".s3.path-style-access", "true")
-//        .set("spark.sql.catalog." + ICEBERG_CATALOG_NAME + ".s3.access-key-id", S3Minio.MINIO_ACCESS_KEY)
-//        .set("spark.sql.catalog." + ICEBERG_CATALOG_NAME + ".s3.secret-access-key", S3Minio.MINIO_SECRET_KEY)
     ;
 
     BaseSparkTest.spark = SparkSession
diff --git a/pom.xml b/pom.xml
index 447feb29..4e348ff4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -39,10 +39,6 @@
         <version.spark>${version.spark.major}.1</version.spark>
         <version.hadoop>3.3.6</version.hadoop>
         <version.hive>3.1.3</version.hive>
-        <!-- Use same version as iceberg https://github.com/apache/iceberg/blob/main/gradle/libs.versions.toml#L31-->  
-        <version.awssdk>2.24.5</version.awssdk>
-        <!-- Use same version as iceberg https://github.com/apache/iceberg/blob/main/gradle/libs.versions.toml#L44-->  
-        <version.googlelibraries>26.28.0</version.googlelibraries>
         <version.googlebigdataoss>2.2.20</version.googlebigdataoss>
         <version.testcontainers>1.19.6</version.testcontainers>
         <!-- Debezium -->
@@ -74,14 +70,6 @@
                 <artifactId>slf4j-jboss-logmanager</artifactId>
                 <version>[1.2.0.Final,)</version>
             </dependency>
-            <!-- aws -->
-            <dependency>
-                <groupId>software.amazon.awssdk</groupId>
-                <artifactId>bom</artifactId>
-                <version>${version.awssdk}</version>
-                <type>pom</type>
-                <scope>import</scope>
-            </dependency>
             <dependency>
                 <groupId>io.quarkus</groupId>
                 <artifactId>quarkus-bom</artifactId>
@@ -89,14 +77,6 @@
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
-            <!-- gcp -->
-            <dependency>
-                <groupId>com.google.cloud</groupId>
-                <artifactId>libraries-bom</artifactId>
-                <version>${version.googlelibraries}</version>
-                <type>pom</type>
-                <scope>import</scope>
-            </dependency>
             <!-- MySQL JDBC Driver, Binlog reader, Geometry support -->
             <dependency>
                 <groupId>mysql</groupId>
@@ -111,27 +91,13 @@
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
-
-            <dependency>
-                <groupId>io.debezium</groupId>
-                <artifactId>debezium-server-batch</artifactId>
-                <version>${project.version}</version>
-            </dependency>
             <!-- Debezium Scripting -->
             <dependency>
                 <groupId>org.codehaus.groovy</groupId>
-                <artifactId>groovy</artifactId>
-                <version>${version.groovy}</version>
-            </dependency>
-            <dependency>
-                <groupId>org.codehaus.groovy</groupId>
-                <artifactId>groovy-json</artifactId>
-                <version>${version.groovy}</version>
-            </dependency>
-            <dependency>
-                <groupId>org.codehaus.groovy</groupId>
-                <artifactId>groovy-jsr223</artifactId>
+                <artifactId>groovy-bom</artifactId>
                 <version>${version.groovy}</version>
+                <type>pom</type>
+                <scope>import</scope>
             </dependency>
             <!-- Test dependencies -->
             <dependency>