From 9091adfb3721f1692e9cffabdbb93f979ea9e51c Mon Sep 17 00:00:00 2001
From: ismail simsek <gel.yine.gel@hotmail.com>
Date: Fri, 15 Oct 2021 12:39:01 +0200
Subject: [PATCH] Update aws dependencies (#29)

---
 .gitignore                                    |  3 +++
 .../conf/application.properties.example       | 25 ++++++++++++++----
 debezium-server-iceberg-sink/pom.xml          | 26 +++++++++----------
 pom.xml                                       |  9 ++-----
 4 files changed, 38 insertions(+), 25 deletions(-)

diff --git a/.gitignore b/.gitignore
index f966bb21..e0d82d3c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,6 @@
+
+deploy.sh
+
 # Debezium gitignore
 
 activemq-data/
diff --git a/debezium-server-dist/src/main/resources/distro/conf/application.properties.example b/debezium-server-dist/src/main/resources/distro/conf/application.properties.example
index 9502bdf1..b27540bf 100644
--- a/debezium-server-dist/src/main/resources/distro/conf/application.properties.example
+++ b/debezium-server-dist/src/main/resources/distro/conf/application.properties.example
@@ -1,5 +1,23 @@
-debezium.sink.type=kinesis
-debezium.sink.kinesis.region=eu-central-1
+debezium.sink.type=iceberg
+debezium.sink.iceberg.table-prefix=debeziumcdc_
+debezium.sink.iceberg.catalog-name=mycatalog
+debezium.sink.iceberg.table-namespace=debeziumevents
+debezium.sink.iceberg.fs.defaultFS=s3a://my-bucket
+debezium.sink.iceberg.warehouse=s3a://my-bucket/iceberg_warehouse
+debezium.sink.iceberg.type=hadoop
+debezium.sink.iceberg.com.amazonaws.services.s3.enableV4=true
+debezium.sink.iceberg.com.amazonaws.services.s3a.enableV4=true
+debezium.sink.iceberg.fs.s3a.aws.credentials.provider=com.amazonaws.auth.DefaultAWSCredentialsProviderChain
+debezium.sink.iceberg.fs.s3a.access.key=my-aws-access-key
+debezium.sink.iceberg.fs.s3a.secret.key=my-secret-access-key
+debezium.sink.iceberg.fs.s3a.path.style.access=true
+debezium.sink.iceberg.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem
+debezium.sink.iceberg.upsert=true
+debezium.sink.iceberg.upsert-keep-deletes=true
+debezium.format.value.schemas.enable=true
+debezium.format.key.schemas.enable=true
+
+# source db
 debezium.source.connector.class=io.debezium.connector.postgresql.PostgresConnector
 debezium.source.offset.storage.file.filename=data/offsets.dat
 debezium.source.offset.flush.interval.ms=0
@@ -11,11 +29,8 @@ debezium.source.database.dbname=postgres
 debezium.source.database.server.name=tutorial
 debezium.source.schema.include.list=inventory
 
-
 # ############ SET LOG LEVELS ############
 quarkus.log.level=INFO
-# Change this to set Spark log level
-quarkus.log.category."org.apache.spark".level=WARN
 # hadoop, parquet
 quarkus.log.category."org.apache.hadoop".level=WARN
 quarkus.log.category."org.apache.parquet".level=WARN
diff --git a/debezium-server-iceberg-sink/pom.xml b/debezium-server-iceberg-sink/pom.xml
index c5d595df..dc87c79d 100644
--- a/debezium-server-iceberg-sink/pom.xml
+++ b/debezium-server-iceberg-sink/pom.xml
@@ -49,36 +49,42 @@
             <artifactId>iceberg-spark3-runtime</artifactId>
             <version>${version.iceberg}</version>
         </dependency>
+
+        <!-- AWS  -->
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
-            <artifactId>glue</artifactId>
+            <artifactId>core</artifactId>
             <version>${version.awssdk}</version>
+            <type>pom</type>
+        </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>glue</artifactId>
         </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>s3</artifactId>
-            <version>${version.awssdk}</version>
         </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>dynamodb</artifactId>
-            <version>${version.awssdk}</version>
         </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>kms</artifactId>
-            <version>${version.awssdk}</version>
         </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>sts</artifactId>
-            <version>${version.awssdk}</version>
         </dependency>
-
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>url-connection-client</artifactId>
-            <version>${version.awssdk}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-aws</artifactId>
+            <version>${version.hadoop}</version>
         </dependency>
 
         <!-- needed for quarkus jboss logging -->
@@ -109,12 +115,6 @@
             <artifactId>hadoop-client</artifactId>
             <version>${version.hadoop}</version>
         </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-aws</artifactId>
-            <version>${version.hadoop}</version>
-            <scope>test</scope>
-        </dependency>
 
         <!-- Testing -->
         <!-- spark for tests -->
diff --git a/pom.xml b/pom.xml
index 2e4a45dc..b8d33ca3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -48,15 +48,10 @@
             <!-- aws -->
             <dependency>
                 <groupId>software.amazon.awssdk</groupId>
-                <artifactId>bundle</artifactId>
-                <version>${version.awssdk}</version>
-            </dependency>
-            <dependency>
-                <groupId>software.amazon.awssdk</groupId>
-                <artifactId>url-connection-client</artifactId>
+                <artifactId>bom</artifactId>
                 <version>${version.awssdk}</version>
+                <type>pom</type>
             </dependency>
-
             <!-- parquet -->
             <dependency>
                 <groupId>org.apache.parquet</groupId>