From 7f0cf402e1da0f7cdd10a636b97b2b33d6457922 Mon Sep 17 00:00:00 2001
From: Ismail Simsek <6005685+ismailsimsek@users.noreply.github.com>
Date: Sun, 31 Mar 2024 22:37:00 +0200
Subject: [PATCH] Add example setup with docker-compose

---
 examples/conf/application.properties | 64 ++++++++++++++++++++++
 examples/docker-compose.yml          | 81 ++++++++++++++++++++++++++++
 2 files changed, 145 insertions(+)
 create mode 100644 examples/conf/application.properties
 create mode 100644 examples/docker-compose.yml

diff --git a/examples/conf/application.properties b/examples/conf/application.properties
new file mode 100644
index 00000000..6621ce58
--- /dev/null
+++ b/examples/conf/application.properties
@@ -0,0 +1,64 @@
+# Use iceberg sink
+debezium.sink.type=iceberg
+
+# Iceberg sink config
+debezium.sink.iceberg.table-prefix=debeziumcdc_
+debezium.sink.iceberg.upsert=true
+debezium.sink.iceberg.upsert-keep-deletes=true
+debezium.sink.iceberg.write.format.default=parquet
+
+# S3 config using JdbcCatalog catalog And S3FileIO
+#debezium.sink.iceberg.catalog-impl=org.apache.iceberg.rest.RESTCatalog
+#debezium.sink.iceberg.uri=http://restcatalog:8181
+debezium.sink.iceberg.type=jdbc
+debezium.sink.iceberg.catalog-name=iceberg
+debezium.sink.iceberg.table-namespace=icebergdata
+debezium.sink.iceberg.warehouse=s3://warehouse
+debezium.sink.iceberg.uri=jdbc:postgresql://jdbccatalog:5432/test
+debezium.sink.iceberg.jdbc.user=testuser
+debezium.sink.iceberg.jdbc.password=testpass
+# Use S3FileIO
+debezium.sink.iceberg.io-impl=org.apache.iceberg.aws.s3.S3FileIO
+debezium.sink.iceberg.s3.endpoint=http://minio:9000
+debezium.sink.iceberg.s3.path-style-access=true
+debezium.sink.iceberg.s3.access-key-id=admin
+debezium.sink.iceberg.s3.secret-access-key=password
+
+# postgres source
+debezium.source.connector.class=io.debezium.connector.postgresql.PostgresConnector
+debezium.source.offset.flush.interval.ms=0
+debezium.source.database.hostname=postgresqlsourcedb
+debezium.source.database.port=5432
+debezium.source.database.user=postgres
+debezium.source.database.password=postgres
+debezium.source.database.dbname=postgres
+debezium.source.database.server.name=tutorial
+debezium.source.database.server.id=1234
+debezium.source.schema.include.list=inventory
+debezium.source.topic.prefix=dbz_
+# saving debezium state data to destination, iceberg tables
+# see https://debezium.io/documentation/reference/stable/development/engine.html#advanced-consuming
+debezium.source.offset.storage=io.debezium.server.iceberg.offset.IcebergOffsetBackingStore
+debezium.source.offset.storage.iceberg.table-name=debezium_offset_storage_table
+# see https://debezium.io/documentation/reference/stable/development/engine.html#database-history-properties
+debezium.source.schema.history.internal=io.debezium.server.iceberg.history.IcebergSchemaHistory
+debezium.source.schema.history.internal.iceberg.table-name=debezium_database_history_storage_table
+
+# enable event schemas - mandatory
+debezium.format.value.schemas.enable=true
+debezium.format.key.schemas.enable=true
+debezium.format.value=json
+debezium.format.key=json
+
+# do event flattening. unwrap message!
+debezium.transforms=unwrap
+debezium.transforms.unwrap.type=io.debezium.transforms.ExtractNewRecordState
+debezium.transforms.unwrap.add.fields=op,table,source.ts_ms,db
+debezium.transforms.unwrap.delete.handling.mode=rewrite
+debezium.transforms.unwrap.drop.tombstones=true
+
+# #######
+quarkus.http.port=8082
+# ############ SET LOG LEVELS ############
+quarkus.log.level=INFO
+quarkus.log.console.json=false
diff --git a/examples/docker-compose.yml b/examples/docker-compose.yml
new file mode 100644
index 00000000..988141eb
--- /dev/null
+++ b/examples/docker-compose.yml
@@ -0,0 +1,81 @@
+version: "3"
+
+services:
+  debezium-iceberg:
+    image: ghcr.io/memiiso/debezium-server-iceberg:latest
+    container_name: debezium-server-iceberg
+    networks:
+      iceberg_net:
+    depends_on:
+      - jdbccatalog
+      - minio
+      - postgresqlsourcedb
+      - mc
+    volumes:
+      - ./conf/application.properties:/app/conf/application.properties
+    environment:
+      - AWS_ACCESS_KEY_ID=admin
+      - AWS_SECRET_ACCESS_KEY=password
+      - AWS_REGION=us-east-1
+    links:
+      - jdbccatalog
+      - minio
+      - postgresqlsourcedb
+  postgresqlsourcedb:
+    image: debezium/example-postgres:2.5
+    container_name: postgresql-source-db
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=postgres
+      - POSTGRES_INITDB_ARGS="-E UTF8"
+      - LANG=en_US.utf8
+    networks:
+      iceberg_net:
+    ports:
+      - 5432:5432
+  jdbccatalog:
+    image: postgres:14-alpine
+    container_name: iceberg-jdbc-catalog
+    networks:
+      iceberg_net:
+    ports:
+      - 5433:5432
+    environment:
+      - POSTGRES_DB=test
+      - POSTGRES_USER=testuser
+      - POSTGRES_PASSWORD=testpass
+  minio:
+    image: minio/minio
+    container_name: minio
+    environment:
+      - MINIO_ROOT_USER=admin
+      - MINIO_ROOT_PASSWORD=password
+      - MINIO_DOMAIN=minio
+      - MINIO_BUCKET=warehouse
+    networks:
+      iceberg_net:
+    ports:
+      - 9001:9001
+      - 9000:9000
+    command: ["server", "/data", "--console-address", ":9001"]
+  mc:
+    image: minio/mc
+    container_name: mc
+    networks:
+      iceberg_net:
+    depends_on:
+      - minio
+    environment:
+      - AWS_ACCESS_KEY_ID=demo
+      - AWS_SECRET_ACCESS_KEY=password
+      - AWS_REGION=us-east-1
+    entrypoint: >
+      /bin/sh -c "
+      until (/usr/bin/mc config host add minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
+      /usr/bin/mc mb --ignore-existing minio/warehouse;
+      /usr/bin/mc policy set public minio/warehouse;
+      exit 0;
+      "
+networks:
+  iceberg_net:
\ No newline at end of file