From a4dd4e39e61d6b07c7a693fe90b5b049129a2348 Mon Sep 17 00:00:00 2001 From: Ismail Simsek <6005685+ismailsimsek@users.noreply.github.com> Date: Fri, 17 May 2024 19:19:32 +0200 Subject: [PATCH] Simplify example config, removed hadoop catalog example --- .../conf/application.properties.example | 73 +++++++------------ 1 file changed, 25 insertions(+), 48 deletions(-) diff --git a/debezium-server-iceberg-dist/src/main/resources/distro/conf/application.properties.example b/debezium-server-iceberg-dist/src/main/resources/distro/conf/application.properties.example index d9367c2d..7fbb9e12 100644 --- a/debezium-server-iceberg-dist/src/main/resources/distro/conf/application.properties.example +++ b/debezium-server-iceberg-dist/src/main/resources/distro/conf/application.properties.example @@ -6,21 +6,7 @@ debezium.sink.iceberg.table-prefix=debeziumcdc_ debezium.sink.iceberg.upsert=true debezium.sink.iceberg.upsert-keep-deletes=true debezium.sink.iceberg.write.format.default=parquet -debezium.sink.iceberg.catalog-name=mycatalog -# Hadoop catalog, you can use other catalog supported by iceberg as well - -# S3 config with hadoop and hadoop catalog -debezium.sink.iceberg.type=hadoop -debezium.sink.iceberg.warehouse=s3a://my-bucket/iceberg_warehouse -debezium.sink.iceberg.table-namespace=debeziumevents -debezium.sink.iceberg.fs.defaultFS=s3a://my-bucket -debezium.sink.iceberg.com.amazonaws.services.s3.enableV4=true -debezium.sink.iceberg.com.amazonaws.services.s3a.enableV4=true -debezium.sink.iceberg.fs.s3a.aws.credentials.provider=com.amazonaws.auth.DefaultAWSCredentialsProviderChain -debezium.sink.iceberg.fs.s3a.access.key=AWS_ACCESS_KEY -debezium.sink.iceberg.fs.s3a.secret.key=AWS_SECRET_ACCESS_KEY -debezium.sink.iceberg.fs.s3a.path.style.access=true -debezium.sink.iceberg.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem +debezium.sink.iceberg.catalog-name=iceberg # S3 config using JdbcCatalog catalog And S3FileIO debezium.sink.iceberg.catalog-impl=org.apache.iceberg.jdbc.JdbcCatalog @@ -28,36 +14,26 @@ debezium.sink.iceberg.uri=jdbc_db_url debezium.sink.iceberg.jdbc.user=my_user debezium.sink.iceberg.jdbc.password=my_password debezium.sink.iceberg.table-namespace=debeziumdata -debezium.sink.iceberg.catalog-name=iceberg -debezium.sink.iceberg.warehouse=s3a://my_bucket/iceberg_warehouse +debezium.sink.iceberg.warehouse=s3://my_bucket/iceberg_warehouse +# S3FileIO +debezium.sink.iceberg.io-impl=org.apache.iceberg.aws.s3.S3FileIO +debezium.sink.iceberg.s3.access-key-id=MY_ACCESS_KEY +debezium.sink.iceberg.s3.secret-access-key=MY_SECRET_KEY +debezium.sink.iceberg.s3.path-style-access=true # Config with hive meatastore catalogs # debezium.sink.iceberg.type=hive # debezium.sink.iceberg.uri=thrift://xx.xxx.xx.xxx:9083 # debezium.sink.iceberg.clients=5 -# debezium.sink.iceberg.warehouse=s3a://datalake/datawarehouse +# debezium.sink.iceberg.warehouse=s3://datalake/datawarehouse # debezium.sink.iceberg.catalog.io-impl=org.apache.iceberg.aws.s3.S3FileIO +# debezium.sink.iceberg.s3.access-key-id=MY_ACCESS_KEY +# debezium.sink.iceberg.s3.secret-access-key=MY_SECRET_KEY # debezium.sink.iceberg.engine.hive.enabled=true # debezium.sink.iceberg.iceberg.engine.hive.enabled=true # debezium.sink.hive.metastore.sasl.enabled=false # debezium.sink.iceberg.hive.metastore.sasl.enabled=false -# Use S3FileIO -debezium.sink.iceberg.io-impl=org.apache.iceberg.aws.s3.S3FileIO -debezium.sink.iceberg.s3.endpoint=http://localhost:9000 -debezium.sink.iceberg.s3.path-style-access=true -debezium.sink.iceberg.s3.access-key-id=MY_ACCESS_KEY -debezium.sink.iceberg.s3.secret-access-key=MY_SECRET_KEY - -# S3 config without hadoop catalog. Using InMemoryCatalog catalog And S3FileIO -### using mino as S3 -debezium.sink.iceberg.s3.endpoint=http://localhost:9000; -debezium.sink.iceberg.s3.path-style-access=true -debezium.sink.iceberg.s3.access-key-id=MY_ACCESS_KEY -debezium.sink.iceberg.s3.secret-access-key=MY_SECRET_KEY -debezium.sink.iceberg.io-impl=org.apache.iceberg.aws.s3.S3FileIO -debezium.sink.iceberg.warehouse=s3a://my-bucket/iceberg_warehouse -debezium.sink.iceberg.catalog-impl=org.apache.iceberg.inmemory.InMemoryCatalog # enable event schemas - mandatory debezium.format.value.schemas.enable=true @@ -65,13 +41,13 @@ debezium.format.key.schemas.enable=true debezium.format.value=json debezium.format.key=json -# saving debezium state data to destination, iceberg tables +# saving debezium state data to destination iceberg tables # see https://debezium.io/documentation/reference/stable/development/engine.html#advanced-consuming debezium.source.offset.storage=io.debezium.server.iceberg.offset.IcebergOffsetBackingStore -debezium.source.offset.storage.iceberg.table-name=debezium_offset_storage_custom_table +debezium.source.offset.storage.iceberg.table-name=debezium_offset_storage_table # see https://debezium.io/documentation/reference/stable/development/engine.html#database-history-properties debezium.source.schema.history.internal=io.debezium.server.iceberg.history.IcebergSchemaHistory -debezium.source.schema.history.internal.iceberg.table-name=debezium_database_history_storage_test +debezium.source.schema.history.internal.iceberg.table-name=debezium_database_history_storage_table # postgres source debezium.source.connector.class=io.debezium.connector.postgresql.PostgresConnector @@ -87,17 +63,17 @@ debezium.source.schema.include.list=inventory debezium.source.topic.prefix=dbz_ # sql server source -#debezium.source.connector.class=io.debezium.connector.sqlserver.SqlServerConnector -#debezium.source.offset.flush.interval.ms=0 -#debezium.source.database.hostname=localhost -#debezium.source.database.port=5432 -#debezium.source.database.user=debezium -#debezium.source.database.password=debezium -#debezium.source.database.dbname=debezium -#debezium.source.database.server.name=tutorial -#debezium.source.schema.include.list=inventory +# debezium.source.connector.class=io.debezium.connector.sqlserver.SqlServerConnector +# debezium.source.offset.flush.interval.ms=0 +# debezium.source.database.hostname=localhost +# debezium.source.database.port=5432 +# debezium.source.database.user=debezium +# debezium.source.database.password=debezium +# debezium.source.database.dbname=debezium +# debezium.source.database.server.name=tutorial +# debezium.source.schema.include.list=inventory # mandate for sql server source, avoid error when snapshot and schema change -#debezium.source.include.schema.changes=false +# debezium.source.include.schema.changes=false # do event flattening. unwrap message! debezium.transforms=unwrap @@ -109,8 +85,9 @@ debezium.transforms.unwrap.drop.tombstones=true # ############ SET LOG LEVELS ############ quarkus.log.level=INFO quarkus.log.console.json=false +# set log level for libs # hadoop, parquet quarkus.log.category."org.apache.hadoop".level=WARN quarkus.log.category."org.apache.parquet".level=WARN # Ignore messages below warning level from Jetty, because it's a bit verbose -quarkus.log.category."org.eclipse.jetty".level=WARN +quarkus.log.category."org.eclipse.jetty".level=WARN \ No newline at end of file