From da1921882e8f8d5c5cf2510bf90f085814cf2f8d Mon Sep 17 00:00:00 2001 From: ismail simsek Date: Wed, 4 Aug 2021 10:15:11 +0200 Subject: [PATCH] Upgrade iceberg Version to 0.12 (#17) Upgrade iceberg Version to 0.12 spark to 3.1.2 --- .../io/debezium/server/iceberg/IcebergUtil.java | 10 ++++++++++ .../server/iceberg/IcebergChangeConsumerTest.java | 5 ++++- .../debezium/server/iceberg/TestIcebergUtil.java | 1 + pom.xml | 14 ++++++++++---- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/debezium-server-iceberg-sink/src/main/java/io/debezium/server/iceberg/IcebergUtil.java b/debezium-server-iceberg-sink/src/main/java/io/debezium/server/iceberg/IcebergUtil.java index 09acb8fb..714bc2e7 100644 --- a/debezium-server-iceberg-sink/src/main/java/io/debezium/server/iceberg/IcebergUtil.java +++ b/debezium-server-iceberg-sink/src/main/java/io/debezium/server/iceberg/IcebergUtil.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.Schema; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.types.Types; @@ -190,4 +191,13 @@ public static Map getConfigSubset(Config config, String prefix) return ret; } + public static Map getConfigurationAsMap(Configuration conf) { + Map config = new HashMap(); + for (Map.Entry entry : conf) { + config.put(entry.getKey(), entry.getValue()); + } + return config; + } + + } diff --git a/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergChangeConsumerTest.java b/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergChangeConsumerTest.java index 32c37d46..e1da209e 100644 --- a/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergChangeConsumerTest.java +++ b/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/IcebergChangeConsumerTest.java @@ -32,6 +32,7 @@ import org.eclipse.microprofile.config.ConfigProvider; import org.eclipse.microprofile.config.inject.ConfigProperty; import org.junit.jupiter.api.Test; +import static io.debezium.server.iceberg.IcebergUtil.getConfigurationAsMap; import static org.junit.jupiter.api.Assertions.assertEquals; /** @@ -71,7 +72,9 @@ private HadoopCatalog getIcebergCatalog() { ConfigProvider.getConfig().getValue(name, String.class)); } } - HadoopCatalog icebergCatalog = new HadoopCatalog("iceberg", hadoopConf, warehouseLocation); + HadoopCatalog icebergCatalog = new HadoopCatalog(); + icebergCatalog.setConf(hadoopConf); + icebergCatalog.initialize("iceberg", getConfigurationAsMap(hadoopConf)); return icebergCatalog; } diff --git a/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/TestIcebergUtil.java b/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/TestIcebergUtil.java index a615ed91..66fc1df8 100644 --- a/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/TestIcebergUtil.java +++ b/debezium-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/TestIcebergUtil.java @@ -63,4 +63,5 @@ public void valuePayloadWithSchemaAsJsonNode() { assertFalse(deserializedSchema.has("schema")); } + } diff --git a/pom.xml b/pom.xml index d4cf39c6..cac89510 100644 --- a/pom.xml +++ b/pom.xml @@ -17,6 +17,13 @@ ${revision} pom + + + nexus-orgapacheiceberg + https://repository.apache.org/content/repositories/orgapacheiceberg-1017/ + + + 0.1.0-SNAPSHOT @@ -28,12 +35,11 @@ true 3.0.7 - 1.5.0.Final 3.1.1 2.12.2 - 0.11.1 - 3.0.2 - 3.3.0 + 0.12.0 + 3.1.2 + 3.3.1 2.16.88 1.11.1