From 69210280eb3606965775bcf6abc2a9c778265cf1 Mon Sep 17 00:00:00 2001
From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com>
Date: Thu, 23 May 2024 20:30:41 +0200
Subject: [PATCH 1/5] Update spark version and enable Mysql tests
---
pom.xml | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/pom.xml b/pom.xml
index 5a6095fb..792ce151 100644
--- a/pom.xml
+++ b/pom.xml
@@ -34,9 +34,10 @@
1.5.2
- 3.5
+ 4.0
+ 3.5
2.13
- ${version.spark.major}.1
+ ${version.spark.major}.0-preview1
3.3.6
3.1.3
2.2.20
@@ -46,13 +47,11 @@
8.0.32
3.10.0
-
- 4.9.3
- iceberg-stg-repo
- https://repository.apache.org/content/repositories/orgapacheiceberg-1163/
+ orgapachespark-stg-repo
+ https://repository.apache.org/content/repositories/orgapachespark-1454/
From ec0fe42e598b0653f285c748fa00ddff834261b3 Mon Sep 17 00:00:00 2001
From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com>
Date: Thu, 23 May 2024 20:31:24 +0200
Subject: [PATCH 2/5] Update pom.xml
---
debezium-server-iceberg-sink/pom.xml | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/debezium-server-iceberg-sink/pom.xml b/debezium-server-iceberg-sink/pom.xml
index 136d3e95..6caf36fe 100644
--- a/debezium-server-iceberg-sink/pom.xml
+++ b/debezium-server-iceberg-sink/pom.xml
@@ -212,11 +212,11 @@
-
+
- org.antlr
- antlr4-runtime
- ${version.antlr}
+ jakarta.servlet
+ jakarta.servlet-api
+ 5.0.0
test
@@ -251,13 +251,13 @@
org.apache.iceberg
- iceberg-spark-${version.spark.major}_${version.spark.scala}
+ iceberg-spark-${version.spark.major.iceebrgtemp}_${version.spark.scala}
${version.iceberg}
test
org.apache.iceberg
- iceberg-spark-extensions-${version.spark.major}_${version.spark.scala}
+ iceberg-spark-extensions-${version.spark.major.iceebrgtemp}_${version.spark.scala}
${version.iceberg}
test
From 907b4bf6c0f615c0eece94e34351cbe1cc0826e2 Mon Sep 17 00:00:00 2001
From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com>
Date: Thu, 23 May 2024 20:31:55 +0200
Subject: [PATCH 3/5] Update IcebergChangeConsumerMysqlTest.java
---
.../debezium/server/iceberg/IcebergChangeConsumerMysqlTest.java | 1 -
1 file changed, 1 deletion(-)
diff --git a/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergChangeConsumerMysqlTest.java b/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergChangeConsumerMysqlTest.java
index 80850aa7..22b515bf 100644
--- a/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergChangeConsumerMysqlTest.java
+++ b/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergChangeConsumerMysqlTest.java
@@ -31,7 +31,6 @@
* @author Ismail Simsek
*/
@QuarkusTest
-@Disabled // @TODO remove spark with antlr4 version
@QuarkusTestResource(value = S3Minio.class, restrictToAnnotatedClass = true)
@QuarkusTestResource(value = SourceMysqlDB.class, restrictToAnnotatedClass = true)
@TestProfile(IcebergChangeConsumerMysqlTest.TestProfile.class)
From ba47d5c1021a77ae4e8e2e4d8c2df1e0ba634dcf Mon Sep 17 00:00:00 2001
From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com>
Date: Thu, 23 May 2024 20:32:19 +0200
Subject: [PATCH 4/5] Update IcebergEventsChangeConsumerTest.java
---
.../iceberg/IcebergEventsChangeConsumerTest.java | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergEventsChangeConsumerTest.java b/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergEventsChangeConsumerTest.java
index a4b31aa1..1d167c3b 100644
--- a/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergEventsChangeConsumerTest.java
+++ b/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergEventsChangeConsumerTest.java
@@ -10,6 +10,7 @@
import io.debezium.server.iceberg.testresources.BaseSparkTest;
import io.debezium.server.iceberg.testresources.S3Minio;
+import io.debezium.server.iceberg.testresources.SourceMysqlDB;
import io.debezium.server.iceberg.testresources.SourcePostgresqlDB;
import io.quarkus.test.common.QuarkusTestResource;
import io.quarkus.test.junit.QuarkusTest;
@@ -33,7 +34,7 @@
*/
@QuarkusTest
@QuarkusTestResource(value = S3Minio.class, restrictToAnnotatedClass = true)
-@QuarkusTestResource(value = SourcePostgresqlDB.class, restrictToAnnotatedClass = true)
+@QuarkusTestResource(value = SourceMysqlDB.class, restrictToAnnotatedClass = true)
@TestProfile(IcebergEventsChangeConsumerTest.TestProfile.class)
public class IcebergEventsChangeConsumerTest extends BaseSparkTest {
@ConfigProperty(name = "debezium.sink.type")
@@ -61,8 +62,16 @@ public static class TestProfile implements QuarkusTestProfile {
public Map getConfigOverrides() {
Map config = new HashMap<>();
config.put("debezium.sink.type", "icebergevents");
+ config.put("quarkus.profile", "mysql");
+ config.put("%mysql.debezium.source.connector.class", "io.debezium.connector.mysql.MySqlConnector");
+ config.put("%mysql.debezium.source.table.whitelist", "inventory.customers,inventory.test_delete_table");
return config;
}
+
+ @Override
+ public String getConfigProfile() {
+ return "mysql";
+ }
}
}
From 1a9d206520a477a8fb04a9e2a63eaaa26f9f873a Mon Sep 17 00:00:00 2001
From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com>
Date: Thu, 23 May 2024 20:33:23 +0200
Subject: [PATCH 5/5] Update pom.xml
---
debezium-server-iceberg-sink/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/debezium-server-iceberg-sink/pom.xml b/debezium-server-iceberg-sink/pom.xml
index 6caf36fe..35a05d9a 100644
--- a/debezium-server-iceberg-sink/pom.xml
+++ b/debezium-server-iceberg-sink/pom.xml
@@ -212,7 +212,7 @@
-
+
jakarta.servlet
jakarta.servlet-api