Package google cloud storage client libs (#103)

* Minor python fix * Add google cloud client libs * Optimize dependencies
memiiso · Aug 20, 2022 · 8f0497c · 8f0497c
1 parent a323b76
commit 8f0497c
Show file tree

Hide file tree

Showing 7 changed files with 90 additions and 21 deletions.
diff --git a/debezium-server-iceberg-dist/pom.xml b/debezium-server-iceberg-dist/pom.xml
@@ -89,10 +89,6 @@
                     <groupId>io.debezium</groupId>
                     <artifactId>debezium-connector-oracle</artifactId>
                 </dependency>
-                <dependency>
-                    <groupId>io.debezium</groupId>
-                    <artifactId>debezium-connector-oracle</artifactId>
-                </dependency>
                 <dependency>
                     <groupId>io.debezium</groupId>
                     <artifactId>debezium-connector-db2</artifactId>

diff --git a/debezium-server-iceberg-dist/src/main/resources/assemblies/server-distribution.xml b/debezium-server-iceberg-dist/src/main/resources/assemblies/server-distribution.xml
@@ -30,6 +30,8 @@
                 <exclude>org.glassfish.jersey.*:*:*</exclude>
                 <exclude>org.eclipse.jetty:*:*</exclude>
                 <exclude>org.apache.maven:*:*</exclude>
+                <exclude>log4j:log4j:*</exclude>
+                <exclude>ch.qos.reload4j:reload4j</exclude>
             </excludes>
         </dependencySet>
     </dependencySets>

diff --git a/debezium-server-iceberg-dist/src/main/resources/distro/conf/application.properties.example b/debezium-server-iceberg-dist/src/main/resources/distro/conf/application.properties.example
@@ -1,38 +1,62 @@
+# Use iceberg sink
 debezium.sink.type=iceberg
+
+# Run without Kafka, use local file to store checkpoints
+debezium.source.database.history=io.debezium.relational.history.FileDatabaseHistory
+debezium.source.database.history.file.filename=data/status.dat
+
+# Iceberg sink config
 debezium.sink.iceberg.table-prefix=debeziumcdc_
 debezium.sink.iceberg.upsert=true
 debezium.sink.iceberg.upsert-keep-deletes=true
 debezium.sink.iceberg.write.format.default=parquet
 debezium.sink.iceberg.catalog-name=mycatalog
+# Hadoop catalog, you can use other catalog supported by iceberg as well
 debezium.sink.iceberg.type=hadoop
 debezium.sink.iceberg.warehouse=s3a://my-bucket/iceberg_warehouse
 debezium.sink.iceberg.table-namespace=debeziumevents
-# s3 conf
+
+# S3 config
 debezium.sink.iceberg.fs.defaultFS=s3a://my-bucket
 debezium.sink.iceberg.com.amazonaws.services.s3.enableV4=true
 debezium.sink.iceberg.com.amazonaws.services.s3a.enableV4=true
 debezium.sink.iceberg.fs.s3a.aws.credentials.provider=com.amazonaws.auth.DefaultAWSCredentialsProviderChain
-debezium.sink.iceberg.fs.s3a.access.key=my-aws-access-key
-debezium.sink.iceberg.fs.s3a.secret.key=my-secret-access-key
+debezium.sink.iceberg.fs.s3a.access.key=AWS_ACCESS_KEY
+debezium.sink.iceberg.fs.s3a.secret.key=AWS_SECRET_ACCESS_KEY
 debezium.sink.iceberg.fs.s3a.path.style.access=true
 debezium.sink.iceberg.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem
-# enable event schemas
+
+# enable event schemas - mandate
 debezium.format.value.schemas.enable=true
 debezium.format.key.schemas.enable=true
 debezium.format.value=json
 debezium.format.key=json
 
-# source db
-debezium.source.connector.class=io.debezium.connector.postgresql.PostgresConnector
-debezium.source.offset.storage.file.filename=data/offsets.dat
-debezium.source.offset.flush.interval.ms=0
-debezium.source.database.hostname=localhost
-debezium.source.database.port=5432
-debezium.source.database.user=postgres
-debezium.source.database.password=postgres
-debezium.source.database.dbname=postgres
-debezium.source.database.server.name=tutorial
-debezium.source.schema.include.list=inventory
+# postgres source
+#debezium.source.connector.class=io.debezium.connector.postgresql.PostgresConnector
+#debezium.source.offset.storage.file.filename=data/offsets.dat
+#debezium.source.offset.flush.interval.ms=0
+#debezium.source.database.hostname=localhost
+#debezium.source.database.port=5432
+#debezium.source.database.user=postgres
+#debezium.source.database.password=postgres
+#debezium.source.database.dbname=postgres
+#debezium.source.database.server.name=tutorial
+#debezium.source.schema.include.list=inventory
+
+# sql server source
+#debezium.source.connector.class=io.debezium.connector.sqlserver.SqlServerConnector
+#debezium.source.offset.storage.file.filename=data/offsets.dat
+#debezium.source.offset.flush.interval.ms=0
+#debezium.source.database.hostname=localhost
+#debezium.source.database.port=5432
+#debezium.source.database.user=debezium
+#debezium.source.database.password=debezium
+#debezium.source.database.dbname=debezium
+#debezium.source.database.server.name=tutorial
+#debezium.source.schema.include.list=inventory
+# mandate for sql server source, avoid error when snapshot and schema change
+#debezium.source.include.schema.changes=false
 
 # do event flattening. unwrap message!
 debezium.transforms=unwrap
@@ -43,6 +67,7 @@ debezium.transforms.unwrap.drop.tombstones=true
 
 # ############ SET LOG LEVELS ############
 quarkus.log.level=INFO
+quarkus.log.console.json=false
 # hadoop, parquet
 quarkus.log.category."org.apache.hadoop".level=WARN
 quarkus.log.category."org.apache.parquet".level=WARN

diff --git a/debezium-server-iceberg-sink/pom.xml b/debezium-server-iceberg-sink/pom.xml
@@ -53,6 +53,27 @@
             <artifactId>iceberg-spark-runtime-3.2_2.13</artifactId>
             <version>${version.iceberg}</version>
         </dependency>
+        <!-- Google -->
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-storage</artifactId>
+            <version>2.11.3</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-nio</artifactId>
+            <version>0.124.14</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.cloud.bigdataoss</groupId>
+            <artifactId>gcs-connector</artifactId>
+            <version>hadoop3-2.2.7</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.cloud.bigdataoss</groupId>
+            <artifactId>gcsio</artifactId>
+            <version>2.2.7</version>
+        </dependency>
         <!-- AWS  -->
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
@@ -103,12 +124,30 @@
                     <groupId>log4j</groupId>
                     <artifactId>log4j</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-reload4j</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client</artifactId>
             <version>${version.hadoop}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>log4j</groupId>
+                    <artifactId>log4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-reload4j</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <!-- Testing -->

diff --git a/debezium-server-iceberg-sink/src/main/resources/conf/application.properties.example b/debezium-server-iceberg-sink/src/main/resources/conf/application.properties.example
@@ -67,6 +67,7 @@ debezium.transforms.unwrap.drop.tombstones=true
 
 # ############ SET LOG LEVELS ############
 quarkus.log.level=INFO
+quarkus.log.console.json=false
 # hadoop, parquet
 quarkus.log.category."org.apache.hadoop".level=WARN
 quarkus.log.category."org.apache.parquet".level=WARN

diff --git a/pom.xml b/pom.xml
@@ -59,6 +59,12 @@
                 <artifactId>kafka-clients</artifactId>
                 <version>${version.kafkaclients}</version>
             </dependency>
+            <!-- log -->
+            <dependency>
+                <groupId>org.jboss.slf4j</groupId>
+                <artifactId>slf4j-jboss-logmanager</artifactId>
+                <version>[1.2.0.Final,)</version>
+            </dependency>
             <!-- aws -->
             <dependency>
                 <groupId>software.amazon.awssdk</groupId>

diff --git a/python/setup.py b/python/setup.py
@@ -1,4 +1,5 @@
 import os
+
 from setuptools import setup, find_packages
 
 setup_py_dir = os.path.dirname(os.path.abspath(__file__))
@@ -18,7 +19,6 @@
     url='https://debezium.io/',
     include_package_data=True,
     license="Apache License 2.0",
-    test_suite='tests',
-    install_requires=["pyjnius==1.4.0"],
+    install_requires=["pyjnius==1.4.2"],
     python_requires='>=3',
 )