Skip to content

Commit

Permalink
Simplify example config, removed hadoop catalog example
Browse files Browse the repository at this point in the history
  • Loading branch information
ismailsimsek committed May 17, 2024
1 parent a860be2 commit a4dd4e3
Showing 1 changed file with 25 additions and 48 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -6,72 +6,48 @@ debezium.sink.iceberg.table-prefix=debeziumcdc_
debezium.sink.iceberg.upsert=true
debezium.sink.iceberg.upsert-keep-deletes=true
debezium.sink.iceberg.write.format.default=parquet
debezium.sink.iceberg.catalog-name=mycatalog
# Hadoop catalog, you can use other catalog supported by iceberg as well

# S3 config with hadoop and hadoop catalog
debezium.sink.iceberg.type=hadoop
debezium.sink.iceberg.warehouse=s3a://my-bucket/iceberg_warehouse
debezium.sink.iceberg.table-namespace=debeziumevents
debezium.sink.iceberg.fs.defaultFS=s3a://my-bucket
debezium.sink.iceberg.com.amazonaws.services.s3.enableV4=true
debezium.sink.iceberg.com.amazonaws.services.s3a.enableV4=true
debezium.sink.iceberg.fs.s3a.aws.credentials.provider=com.amazonaws.auth.DefaultAWSCredentialsProviderChain
debezium.sink.iceberg.fs.s3a.access.key=AWS_ACCESS_KEY
debezium.sink.iceberg.fs.s3a.secret.key=AWS_SECRET_ACCESS_KEY
debezium.sink.iceberg.fs.s3a.path.style.access=true
debezium.sink.iceberg.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem
debezium.sink.iceberg.catalog-name=iceberg

# S3 config using JdbcCatalog catalog And S3FileIO
debezium.sink.iceberg.catalog-impl=org.apache.iceberg.jdbc.JdbcCatalog
debezium.sink.iceberg.uri=jdbc_db_url
debezium.sink.iceberg.jdbc.user=my_user
debezium.sink.iceberg.jdbc.password=my_password
debezium.sink.iceberg.table-namespace=debeziumdata
debezium.sink.iceberg.catalog-name=iceberg
debezium.sink.iceberg.warehouse=s3a://my_bucket/iceberg_warehouse
debezium.sink.iceberg.warehouse=s3://my_bucket/iceberg_warehouse
# S3FileIO
debezium.sink.iceberg.io-impl=org.apache.iceberg.aws.s3.S3FileIO
debezium.sink.iceberg.s3.access-key-id=MY_ACCESS_KEY
debezium.sink.iceberg.s3.secret-access-key=MY_SECRET_KEY
debezium.sink.iceberg.s3.path-style-access=true

# Config with hive meatastore catalogs
# debezium.sink.iceberg.type=hive
# debezium.sink.iceberg.uri=thrift://xx.xxx.xx.xxx:9083
# debezium.sink.iceberg.clients=5
# debezium.sink.iceberg.warehouse=s3a://datalake/datawarehouse
# debezium.sink.iceberg.warehouse=s3://datalake/datawarehouse
# debezium.sink.iceberg.catalog.io-impl=org.apache.iceberg.aws.s3.S3FileIO
# debezium.sink.iceberg.s3.access-key-id=MY_ACCESS_KEY
# debezium.sink.iceberg.s3.secret-access-key=MY_SECRET_KEY
# debezium.sink.iceberg.engine.hive.enabled=true
# debezium.sink.iceberg.iceberg.engine.hive.enabled=true
# debezium.sink.hive.metastore.sasl.enabled=false
# debezium.sink.iceberg.hive.metastore.sasl.enabled=false

# Use S3FileIO
debezium.sink.iceberg.io-impl=org.apache.iceberg.aws.s3.S3FileIO
debezium.sink.iceberg.s3.endpoint=http://localhost:9000
debezium.sink.iceberg.s3.path-style-access=true
debezium.sink.iceberg.s3.access-key-id=MY_ACCESS_KEY
debezium.sink.iceberg.s3.secret-access-key=MY_SECRET_KEY

# S3 config without hadoop catalog. Using InMemoryCatalog catalog And S3FileIO
### using mino as S3
debezium.sink.iceberg.s3.endpoint=http://localhost:9000;
debezium.sink.iceberg.s3.path-style-access=true
debezium.sink.iceberg.s3.access-key-id=MY_ACCESS_KEY
debezium.sink.iceberg.s3.secret-access-key=MY_SECRET_KEY
debezium.sink.iceberg.io-impl=org.apache.iceberg.aws.s3.S3FileIO
debezium.sink.iceberg.warehouse=s3a://my-bucket/iceberg_warehouse
debezium.sink.iceberg.catalog-impl=org.apache.iceberg.inmemory.InMemoryCatalog

# enable event schemas - mandatory
debezium.format.value.schemas.enable=true
debezium.format.key.schemas.enable=true
debezium.format.value=json
debezium.format.key=json

# saving debezium state data to destination, iceberg tables
# saving debezium state data to destination iceberg tables
# see https://debezium.io/documentation/reference/stable/development/engine.html#advanced-consuming
debezium.source.offset.storage=io.debezium.server.iceberg.offset.IcebergOffsetBackingStore
debezium.source.offset.storage.iceberg.table-name=debezium_offset_storage_custom_table
debezium.source.offset.storage.iceberg.table-name=debezium_offset_storage_table
# see https://debezium.io/documentation/reference/stable/development/engine.html#database-history-properties
debezium.source.schema.history.internal=io.debezium.server.iceberg.history.IcebergSchemaHistory
debezium.source.schema.history.internal.iceberg.table-name=debezium_database_history_storage_test
debezium.source.schema.history.internal.iceberg.table-name=debezium_database_history_storage_table

# postgres source
debezium.source.connector.class=io.debezium.connector.postgresql.PostgresConnector
Expand All @@ -87,17 +63,17 @@ debezium.source.schema.include.list=inventory
debezium.source.topic.prefix=dbz_

# sql server source
#debezium.source.connector.class=io.debezium.connector.sqlserver.SqlServerConnector
#debezium.source.offset.flush.interval.ms=0
#debezium.source.database.hostname=localhost
#debezium.source.database.port=5432
#debezium.source.database.user=debezium
#debezium.source.database.password=debezium
#debezium.source.database.dbname=debezium
#debezium.source.database.server.name=tutorial
#debezium.source.schema.include.list=inventory
# debezium.source.connector.class=io.debezium.connector.sqlserver.SqlServerConnector
# debezium.source.offset.flush.interval.ms=0
# debezium.source.database.hostname=localhost
# debezium.source.database.port=5432
# debezium.source.database.user=debezium
# debezium.source.database.password=debezium
# debezium.source.database.dbname=debezium
# debezium.source.database.server.name=tutorial
# debezium.source.schema.include.list=inventory
# mandate for sql server source, avoid error when snapshot and schema change
#debezium.source.include.schema.changes=false
# debezium.source.include.schema.changes=false

# do event flattening. unwrap message!
debezium.transforms=unwrap
Expand All @@ -109,8 +85,9 @@ debezium.transforms.unwrap.drop.tombstones=true
# ############ SET LOG LEVELS ############
quarkus.log.level=INFO
quarkus.log.console.json=false
# set log level for libs
# hadoop, parquet
quarkus.log.category."org.apache.hadoop".level=WARN
quarkus.log.category."org.apache.parquet".level=WARN
# Ignore messages below warning level from Jetty, because it's a bit verbose
quarkus.log.category."org.eclipse.jetty".level=WARN
quarkus.log.category."org.eclipse.jetty".level=WARN

0 comments on commit a4dd4e3

Please sign in to comment.