Merge pull request #331 from memiiso/ismailsimsek-patch-1

Upgrade spark to 4.0.x version and enable Mysql tests
memiiso · May 24, 2024 · ff64621 · ff64621
2 parents ae32fd6 + 1a9d206
commit ff64621
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 14 deletions.
diff --git a/debezium-server-iceberg-sink/pom.xml b/debezium-server-iceberg-sink/pom.xml
@@ -212,11 +212,11 @@
 
         <!-- Testing -->
         <!-- spark for tests -->
-        <!-- antlr4 version compatible with spark, testing only -->
+        <!-- jakarta version compatible with spark, for testing only -->
         <dependency>
-            <groupId>org.antlr</groupId>
-            <artifactId>antlr4-runtime</artifactId>
-            <version>${version.antlr}</version>
+            <groupId>jakarta.servlet</groupId>
+            <artifactId>jakarta.servlet-api</artifactId>
+            <version>5.0.0</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -251,13 +251,13 @@
         </dependency>
         <dependency>
             <groupId>org.apache.iceberg</groupId>
-            <artifactId>iceberg-spark-${version.spark.major}_${version.spark.scala}</artifactId>
+            <artifactId>iceberg-spark-${version.spark.major.iceebrgtemp}_${version.spark.scala}</artifactId>
             <version>${version.iceberg}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.iceberg</groupId>
-            <artifactId>iceberg-spark-extensions-${version.spark.major}_${version.spark.scala}</artifactId>
+            <artifactId>iceberg-spark-extensions-${version.spark.major.iceebrgtemp}_${version.spark.scala}</artifactId>
             <version>${version.iceberg}</version>
             <scope>test</scope>
         </dependency>

diff --git a/...iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergChangeConsumerMysqlTest.java b/...iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergChangeConsumerMysqlTest.java
@@ -31,7 +31,6 @@
  * @author Ismail Simsek
  */
 @QuarkusTest
-@Disabled // @TODO remove spark with antlr4 version
 @QuarkusTestResource(value = S3Minio.class, restrictToAnnotatedClass = true)
 @QuarkusTestResource(value = SourceMysqlDB.class, restrictToAnnotatedClass = true)
 @TestProfile(IcebergChangeConsumerMysqlTest.TestProfile.class)

diff --git a/...ceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergEventsChangeConsumerTest.java b/...ceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergEventsChangeConsumerTest.java
@@ -10,6 +10,7 @@
 
 import io.debezium.server.iceberg.testresources.BaseSparkTest;
 import io.debezium.server.iceberg.testresources.S3Minio;
+import io.debezium.server.iceberg.testresources.SourceMysqlDB;
 import io.debezium.server.iceberg.testresources.SourcePostgresqlDB;
 import io.quarkus.test.common.QuarkusTestResource;
 import io.quarkus.test.junit.QuarkusTest;
@@ -33,7 +34,7 @@
  */
 @QuarkusTest
 @QuarkusTestResource(value = S3Minio.class, restrictToAnnotatedClass = true)
-@QuarkusTestResource(value = SourcePostgresqlDB.class, restrictToAnnotatedClass = true)
+@QuarkusTestResource(value = SourceMysqlDB.class, restrictToAnnotatedClass = true)
 @TestProfile(IcebergEventsChangeConsumerTest.TestProfile.class)
 public class IcebergEventsChangeConsumerTest extends BaseSparkTest {
   @ConfigProperty(name = "debezium.sink.type")
@@ -61,8 +62,16 @@ public static class TestProfile implements QuarkusTestProfile {
     public Map<String, String> getConfigOverrides() {
       Map<String, String> config = new HashMap<>();
       config.put("debezium.sink.type", "icebergevents");
+      config.put("quarkus.profile", "mysql");
+      config.put("%mysql.debezium.source.connector.class", "io.debezium.connector.mysql.MySqlConnector");
+      config.put("%mysql.debezium.source.table.whitelist", "inventory.customers,inventory.test_delete_table");
       return config;
     }
+
+    @Override
+    public String getConfigProfile() {
+      return "mysql";
+    }
   }
 
 }
diff --git a/pom.xml b/pom.xml
@@ -34,9 +34,10 @@
         <version.iceberg>1.5.2</version.iceberg>
         <!-- Following two properties defines which version of iceberg-spark-runtime is used -->
         <!-- Example https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-spark-runtime-3.4_2.13/1.4.3 -->
-        <version.spark.major>3.5</version.spark.major>
+        <version.spark.major>4.0</version.spark.major>
+        <version.spark.major.iceebrgtemp>3.5</version.spark.major.iceebrgtemp>
         <version.spark.scala>2.13</version.spark.scala>
-        <version.spark>${version.spark.major}.1</version.spark>
+        <version.spark>${version.spark.major}.0-preview1</version.spark>
         <version.hadoop>3.3.6</version.hadoop>
         <version.hive>3.1.3</version.hive>
         <version.googlebigdataoss>2.2.20</version.googlebigdataoss>
@@ -46,13 +47,11 @@
         <version.mysql.driver>8.0.32</version.mysql.driver>
         <!-- Quarkus -->
         <version.quarkus>3.10.0</version.quarkus>
-        <!-- FOR TESTS Align with SPARK Antlr version -->
-        <version.antlr>4.9.3</version.antlr>
     </properties>
     <repositories>
         <repository>
-            <id>iceberg-stg-repo</id>
-            <url>https://repository.apache.org/content/repositories/orgapacheiceberg-1163/</url>
+            <id>orgapachespark-stg-repo</id>
+            <url>https://repository.apache.org/content/repositories/orgapachespark-1454/</url>
         </repository>
     </repositories>
     <dependencyManagement>