Upgrade iceberg Version to 0.12 (#17)

Upgrade iceberg Version to 0.12 spark to 3.1.2
memiiso · Aug 4, 2021 · 9a15fe7 · 9a15fe7
1 parent e71c447
commit 9a15fe7
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 5 deletions.
diff --git a/debezium-server-iceberg-sink/src/main/java/io/debezium/server/iceberg/IcebergUtil.java b/debezium-server-iceberg-sink/src/main/java/io/debezium/server/iceberg/IcebergUtil.java
@@ -17,6 +17,7 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.iceberg.Schema;
 import org.apache.iceberg.data.GenericRecord;
 import org.apache.iceberg.types.Types;
@@ -190,4 +191,13 @@ public static Map<String, String> getConfigSubset(Config config, String prefix)
     return ret;
   }
 
+  public static Map<String, String> getConfigurationAsMap(Configuration conf) {
+    Map<String, String> config = new HashMap<String, String>();
+    for (Map.Entry<String, String> entry : conf) {
+      config.put(entry.getKey(), entry.getValue());
+    }
+    return config;
+  }
+
+
 }
diff --git a/...rver-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergChangeConsumerTest.java b/...rver-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergChangeConsumerTest.java
@@ -32,6 +32,7 @@
 import org.eclipse.microprofile.config.ConfigProvider;
 import org.eclipse.microprofile.config.inject.ConfigProperty;
 import org.junit.jupiter.api.Test;
+import static io.debezium.server.iceberg.IcebergUtil.getConfigurationAsMap;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 /**
@@ -71,7 +72,9 @@ private HadoopCatalog getIcebergCatalog() {
             ConfigProvider.getConfig().getValue(name, String.class));
       }
     }
-    HadoopCatalog icebergCatalog = new HadoopCatalog("iceberg", hadoopConf, warehouseLocation);
+    HadoopCatalog icebergCatalog = new HadoopCatalog();
+    icebergCatalog.setConf(hadoopConf);
+    icebergCatalog.initialize("iceberg", getConfigurationAsMap(hadoopConf));
     return icebergCatalog;
   }
 

diff --git a/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/TestIcebergUtil.java b/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/TestIcebergUtil.java
@@ -63,4 +63,5 @@ public void valuePayloadWithSchemaAsJsonNode() {
     assertFalse(deserializedSchema.has("schema"));
   }
 
+
 }
diff --git a/pom.xml b/pom.xml
@@ -17,6 +17,13 @@
     <version>${revision}</version>
     <packaging>pom</packaging>
 
+    <repositories>
+        <repository>
+            <id>nexus-orgapacheiceberg</id>
+            <url>https://repository.apache.org/content/repositories/orgapacheiceberg-1017/</url>
+        </repository>
+    </repositories>
+
     <properties>
         <revision>0.1.0-SNAPSHOT</revision>
 
@@ -28,12 +35,11 @@
         <skipITs>true</skipITs>
 
         <version.groovy>3.0.7</version.groovy>
-        <version.debezium>1.5.0.Final</version.debezium>
         <version.assembly.plugin>3.1.1</version.assembly.plugin>
         <version.jackson>2.12.2</version.jackson>
-        <version.iceberg>0.11.1</version.iceberg>
-        <version.spark>3.0.2</version.spark>
-        <version.hadoop>3.3.0</version.hadoop>
+        <version.iceberg>0.12.0</version.iceberg>
+        <version.spark>3.1.2</version.spark>
+        <version.hadoop>3.3.1</version.hadoop>
         <version.awssdk>2.16.88</version.awssdk>
         <version.parquet>1.11.1</version.parquet>
         <!-- Debezium -->