From b6c1e3f58a5d0fa01d19b2f13e2b8e66d506d731 Mon Sep 17 00:00:00 2001
From: Ismail Simsek <6005685+ismailsimsek@users.noreply.github.com>
Date: Sun, 3 Mar 2024 11:51:56 +0100
Subject: [PATCH] Replace spark runtime
---
debezium-server-iceberg-sink/pom.xml | 58 ++++++++++++++++++-
.../debezium/server/iceberg/IcebergUtil.java | 2 +-
.../tableoperator/BaseDeltaTaskWriter.java | 2 +-
.../tableoperator/PartitionedDeltaWriter.java | 4 +-
4 files changed, 61 insertions(+), 5 deletions(-)
diff --git a/debezium-server-iceberg-sink/pom.xml b/debezium-server-iceberg-sink/pom.xml
index 40742b6f..f83f70cb 100644
--- a/debezium-server-iceberg-sink/pom.xml
+++ b/debezium-server-iceberg-sink/pom.xml
@@ -63,6 +63,56 @@
iceberg-parquet
${version.iceberg}
+
+ org.apache.iceberg
+ iceberg-orc
+ ${version.iceberg}
+
+
+ org.apache.iceberg
+ iceberg-arrow
+ ${version.iceberg}
+
+
+ org.apache.hive
+ hive-metastore
+ ${version.hive}
+
+
+ org.apache.parquet
+ parquet-hadoop-bundle
+
+
+ org.apache.arrow
+ arrow-*
+
+
+
+
+ org.apache.iceberg
+ iceberg-gcp
+ ${version.iceberg}
+
+
+ org.apache.iceberg
+ iceberg-aws
+ ${version.iceberg}
+
+
+ org.apache.iceberg
+ iceberg-dell
+ ${version.iceberg}
+
+
+ org.apache.iceberg
+ iceberg-aliyun
+ ${version.iceberg}
+
+
+ org.apache.iceberg
+ iceberg-azure
+ ${version.iceberg}
+
com.google.cloud
@@ -212,7 +262,13 @@
org.apache.iceberg
- iceberg-spark-runtime-3.2_2.13
+ iceberg-spark-${version.spark.major}_${version.spark.scala}
+ ${version.iceberg}
+ test
+
+
+ org.apache.iceberg
+ iceberg-spark-extensions-${version.spark.major}_${version.spark.scala}
${version.iceberg}
test
diff --git a/debezium-server-iceberg-sink/src/main/java/io/debezium/server/iceberg/IcebergUtil.java b/debezium-server-iceberg-sink/src/main/java/io/debezium/server/iceberg/IcebergUtil.java
index 2547db30..ed0614db 100644
--- a/debezium-server-iceberg-sink/src/main/java/io/debezium/server/iceberg/IcebergUtil.java
+++ b/debezium-server-iceberg-sink/src/main/java/io/debezium/server/iceberg/IcebergUtil.java
@@ -28,7 +28,7 @@
import org.apache.iceberg.data.GenericAppenderFactory;
import org.apache.iceberg.exceptions.NoSuchTableException;
import org.apache.iceberg.io.OutputFileFactory;
-import org.apache.iceberg.relocated.com.google.common.primitives.Ints;
+import com.google.common.primitives.Ints;
import org.eclipse.microprofile.config.Config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/debezium-server-iceberg-sink/src/main/java/io/debezium/server/iceberg/tableoperator/BaseDeltaTaskWriter.java b/debezium-server-iceberg-sink/src/main/java/io/debezium/server/iceberg/tableoperator/BaseDeltaTaskWriter.java
index 8268214c..6718516e 100644
--- a/debezium-server-iceberg-sink/src/main/java/io/debezium/server/iceberg/tableoperator/BaseDeltaTaskWriter.java
+++ b/debezium-server-iceberg-sink/src/main/java/io/debezium/server/iceberg/tableoperator/BaseDeltaTaskWriter.java
@@ -3,7 +3,6 @@
import java.io.IOException;
import java.util.List;
-import com.google.common.collect.Sets;
import org.apache.iceberg.*;
import org.apache.iceberg.data.InternalRecordWrapper;
import org.apache.iceberg.data.Record;
@@ -11,6 +10,7 @@
import org.apache.iceberg.io.FileAppenderFactory;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.io.OutputFileFactory;
+import com.google.common.collect.Sets;
import org.apache.iceberg.types.TypeUtil;
abstract class BaseDeltaTaskWriter extends BaseTaskWriter {
diff --git a/debezium-server-iceberg-sink/src/main/java/io/debezium/server/iceberg/tableoperator/PartitionedDeltaWriter.java b/debezium-server-iceberg-sink/src/main/java/io/debezium/server/iceberg/tableoperator/PartitionedDeltaWriter.java
index aff40809..2530b693 100644
--- a/debezium-server-iceberg-sink/src/main/java/io/debezium/server/iceberg/tableoperator/PartitionedDeltaWriter.java
+++ b/debezium-server-iceberg-sink/src/main/java/io/debezium/server/iceberg/tableoperator/PartitionedDeltaWriter.java
@@ -2,7 +2,6 @@
import java.io.IOException;
import java.io.UncheckedIOException;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -14,13 +13,14 @@
import org.apache.iceberg.io.FileAppenderFactory;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.io.OutputFileFactory;
+import com.google.common.collect.Maps;
import org.apache.iceberg.util.Tasks;
class PartitionedDeltaWriter extends BaseDeltaTaskWriter {
private final PartitionKey partitionKey;
- private final Map writers = new HashMap<>();
+ private final Map writers = Maps.newHashMap();
PartitionedDeltaWriter(PartitionSpec spec,
FileFormat format,