From 69210280eb3606965775bcf6abc2a9c778265cf1 Mon Sep 17 00:00:00 2001 From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com> Date: Thu, 23 May 2024 20:30:41 +0200 Subject: [PATCH 1/5] Update spark version and enable Mysql tests --- pom.xml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 5a6095fb..792ce151 100644 --- a/pom.xml +++ b/pom.xml @@ -34,9 +34,10 @@ 1.5.2 - 3.5 + 4.0 + 3.5 2.13 - ${version.spark.major}.1 + ${version.spark.major}.0-preview1 3.3.6 3.1.3 2.2.20 @@ -46,13 +47,11 @@ 8.0.32 3.10.0 - - 4.9.3 - iceberg-stg-repo - https://repository.apache.org/content/repositories/orgapacheiceberg-1163/ + orgapachespark-stg-repo + https://repository.apache.org/content/repositories/orgapachespark-1454/ From ec0fe42e598b0653f285c748fa00ddff834261b3 Mon Sep 17 00:00:00 2001 From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com> Date: Thu, 23 May 2024 20:31:24 +0200 Subject: [PATCH 2/5] Update pom.xml --- debezium-server-iceberg-sink/pom.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/debezium-server-iceberg-sink/pom.xml b/debezium-server-iceberg-sink/pom.xml index 136d3e95..6caf36fe 100644 --- a/debezium-server-iceberg-sink/pom.xml +++ b/debezium-server-iceberg-sink/pom.xml @@ -212,11 +212,11 @@ - + - org.antlr - antlr4-runtime - ${version.antlr} + jakarta.servlet + jakarta.servlet-api + 5.0.0 test @@ -251,13 +251,13 @@ org.apache.iceberg - iceberg-spark-${version.spark.major}_${version.spark.scala} + iceberg-spark-${version.spark.major.iceebrgtemp}_${version.spark.scala} ${version.iceberg} test org.apache.iceberg - iceberg-spark-extensions-${version.spark.major}_${version.spark.scala} + iceberg-spark-extensions-${version.spark.major.iceebrgtemp}_${version.spark.scala} ${version.iceberg} test From 907b4bf6c0f615c0eece94e34351cbe1cc0826e2 Mon Sep 17 00:00:00 2001 From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com> Date: Thu, 23 May 2024 20:31:55 +0200 Subject: [PATCH 3/5] Update IcebergChangeConsumerMysqlTest.java --- .../debezium/server/iceberg/IcebergChangeConsumerMysqlTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergChangeConsumerMysqlTest.java b/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergChangeConsumerMysqlTest.java index 80850aa7..22b515bf 100644 --- a/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergChangeConsumerMysqlTest.java +++ b/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergChangeConsumerMysqlTest.java @@ -31,7 +31,6 @@ * @author Ismail Simsek */ @QuarkusTest -@Disabled // @TODO remove spark with antlr4 version @QuarkusTestResource(value = S3Minio.class, restrictToAnnotatedClass = true) @QuarkusTestResource(value = SourceMysqlDB.class, restrictToAnnotatedClass = true) @TestProfile(IcebergChangeConsumerMysqlTest.TestProfile.class) From ba47d5c1021a77ae4e8e2e4d8c2df1e0ba634dcf Mon Sep 17 00:00:00 2001 From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com> Date: Thu, 23 May 2024 20:32:19 +0200 Subject: [PATCH 4/5] Update IcebergEventsChangeConsumerTest.java --- .../iceberg/IcebergEventsChangeConsumerTest.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergEventsChangeConsumerTest.java b/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergEventsChangeConsumerTest.java index a4b31aa1..1d167c3b 100644 --- a/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergEventsChangeConsumerTest.java +++ b/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergEventsChangeConsumerTest.java @@ -10,6 +10,7 @@ import io.debezium.server.iceberg.testresources.BaseSparkTest; import io.debezium.server.iceberg.testresources.S3Minio; +import io.debezium.server.iceberg.testresources.SourceMysqlDB; import io.debezium.server.iceberg.testresources.SourcePostgresqlDB; import io.quarkus.test.common.QuarkusTestResource; import io.quarkus.test.junit.QuarkusTest; @@ -33,7 +34,7 @@ */ @QuarkusTest @QuarkusTestResource(value = S3Minio.class, restrictToAnnotatedClass = true) -@QuarkusTestResource(value = SourcePostgresqlDB.class, restrictToAnnotatedClass = true) +@QuarkusTestResource(value = SourceMysqlDB.class, restrictToAnnotatedClass = true) @TestProfile(IcebergEventsChangeConsumerTest.TestProfile.class) public class IcebergEventsChangeConsumerTest extends BaseSparkTest { @ConfigProperty(name = "debezium.sink.type") @@ -61,8 +62,16 @@ public static class TestProfile implements QuarkusTestProfile { public Map getConfigOverrides() { Map config = new HashMap<>(); config.put("debezium.sink.type", "icebergevents"); + config.put("quarkus.profile", "mysql"); + config.put("%mysql.debezium.source.connector.class", "io.debezium.connector.mysql.MySqlConnector"); + config.put("%mysql.debezium.source.table.whitelist", "inventory.customers,inventory.test_delete_table"); return config; } + + @Override + public String getConfigProfile() { + return "mysql"; + } } } From 1a9d206520a477a8fb04a9e2a63eaaa26f9f873a Mon Sep 17 00:00:00 2001 From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com> Date: Thu, 23 May 2024 20:33:23 +0200 Subject: [PATCH 5/5] Update pom.xml --- debezium-server-iceberg-sink/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debezium-server-iceberg-sink/pom.xml b/debezium-server-iceberg-sink/pom.xml index 6caf36fe..35a05d9a 100644 --- a/debezium-server-iceberg-sink/pom.xml +++ b/debezium-server-iceberg-sink/pom.xml @@ -212,7 +212,7 @@ - + jakarta.servlet jakarta.servlet-api