Update aws dependencies (#29)

memiiso · Oct 15, 2021 · 9091adf · 9091adf
1 parent 3fab5c4
commit 9091adf
Show file tree

Hide file tree

Showing 4 changed files with 38 additions and 25 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,6 @@
+
+deploy.sh
+
 # Debezium gitignore
 
 activemq-data/

diff --git a/debezium-server-dist/src/main/resources/distro/conf/application.properties.example b/debezium-server-dist/src/main/resources/distro/conf/application.properties.example
@@ -1,5 +1,23 @@
-debezium.sink.type=kinesis
-debezium.sink.kinesis.region=eu-central-1
+debezium.sink.type=iceberg
+debezium.sink.iceberg.table-prefix=debeziumcdc_
+debezium.sink.iceberg.catalog-name=mycatalog
+debezium.sink.iceberg.table-namespace=debeziumevents
+debezium.sink.iceberg.fs.defaultFS=s3a://my-bucket
+debezium.sink.iceberg.warehouse=s3a://my-bucket/iceberg_warehouse
+debezium.sink.iceberg.type=hadoop
+debezium.sink.iceberg.com.amazonaws.services.s3.enableV4=true
+debezium.sink.iceberg.com.amazonaws.services.s3a.enableV4=true
+debezium.sink.iceberg.fs.s3a.aws.credentials.provider=com.amazonaws.auth.DefaultAWSCredentialsProviderChain
+debezium.sink.iceberg.fs.s3a.access.key=my-aws-access-key
+debezium.sink.iceberg.fs.s3a.secret.key=my-secret-access-key
+debezium.sink.iceberg.fs.s3a.path.style.access=true
+debezium.sink.iceberg.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem
+debezium.sink.iceberg.upsert=true
+debezium.sink.iceberg.upsert-keep-deletes=true
+debezium.format.value.schemas.enable=true
+debezium.format.key.schemas.enable=true
+
+# source db
 debezium.source.connector.class=io.debezium.connector.postgresql.PostgresConnector
 debezium.source.offset.storage.file.filename=data/offsets.dat
 debezium.source.offset.flush.interval.ms=0
@@ -11,11 +29,8 @@ debezium.source.database.dbname=postgres
 debezium.source.database.server.name=tutorial
 debezium.source.schema.include.list=inventory
 
-
 # ############ SET LOG LEVELS ############
 quarkus.log.level=INFO
-# Change this to set Spark log level
-quarkus.log.category."org.apache.spark".level=WARN
 # hadoop, parquet
 quarkus.log.category."org.apache.hadoop".level=WARN
 quarkus.log.category."org.apache.parquet".level=WARN

diff --git a/debezium-server-iceberg-sink/pom.xml b/debezium-server-iceberg-sink/pom.xml
@@ -49,36 +49,42 @@
             <artifactId>iceberg-spark3-runtime</artifactId>
             <version>${version.iceberg}</version>
         </dependency>
+
+        <!-- AWS  -->
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
-            <artifactId>glue</artifactId>
+            <artifactId>core</artifactId>
             <version>${version.awssdk}</version>
+            <type>pom</type>
+        </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>glue</artifactId>
         </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>s3</artifactId>
-            <version>${version.awssdk}</version>
         </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>dynamodb</artifactId>
-            <version>${version.awssdk}</version>
         </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>kms</artifactId>
-            <version>${version.awssdk}</version>
         </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>sts</artifactId>
-            <version>${version.awssdk}</version>
         </dependency>
-
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>url-connection-client</artifactId>
-            <version>${version.awssdk}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-aws</artifactId>
+            <version>${version.hadoop}</version>
         </dependency>
 
         <!-- needed for quarkus jboss logging -->
@@ -109,12 +115,6 @@
             <artifactId>hadoop-client</artifactId>
             <version>${version.hadoop}</version>
         </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-aws</artifactId>
-            <version>${version.hadoop}</version>
-            <scope>test</scope>
-        </dependency>
 
         <!-- Testing -->
         <!-- spark for tests -->

diff --git a/pom.xml b/pom.xml
@@ -48,15 +48,10 @@
             <!-- aws -->
             <dependency>
                 <groupId>software.amazon.awssdk</groupId>
-                <artifactId>bundle</artifactId>
-                <version>${version.awssdk}</version>
-            </dependency>
-            <dependency>
-                <groupId>software.amazon.awssdk</groupId>
-                <artifactId>url-connection-client</artifactId>
+                <artifactId>bom</artifactId>
                 <version>${version.awssdk}</version>
+                <type>pom</type>
             </dependency>
-
             <!-- parquet -->
             <dependency>
                 <groupId>org.apache.parquet</groupId>