diff --git a/pom.xml b/pom.xml index cec34e7e382a..6a28c3349b87 100644 --- a/pom.xml +++ b/pom.xml @@ -69,7 +69,7 @@ 0.10.0 0.19.0 2.3.1 - 0.10.1 + 0.11.0 1.15.1 3.2.12 diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/HudiRealtimeBootstrapBaseFileSplitConverter.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/HudiRealtimeBootstrapBaseFileSplitConverter.java index 8c4f6119beac..c52c98beb37e 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/HudiRealtimeBootstrapBaseFileSplitConverter.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/HudiRealtimeBootstrapBaseFileSplitConverter.java @@ -17,7 +17,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.FileSplit; import org.apache.hudi.common.model.HoodieLogFile; -import org.apache.hudi.hadoop.realtime.RealtimeBootstrapBaseFileSplit; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.hadoop.realtime.HoodieRealtimeBootstrapBaseFileSplit; import java.io.IOException; import java.util.Arrays; @@ -45,11 +46,11 @@ public class HudiRealtimeBootstrapBaseFileSplitConverter @Override public Optional> extractCustomSplitInfo(FileSplit split) { - if (split instanceof RealtimeBootstrapBaseFileSplit) { + if (split instanceof HoodieRealtimeBootstrapBaseFileSplit) { ImmutableMap.Builder customSplitInfo = ImmutableMap.builder(); - RealtimeBootstrapBaseFileSplit hudiSplit = (RealtimeBootstrapBaseFileSplit) split; + HoodieRealtimeBootstrapBaseFileSplit hudiSplit = (HoodieRealtimeBootstrapBaseFileSplit) split; - customSplitInfo.put(CUSTOM_FILE_SPLIT_CLASS_KEY, RealtimeBootstrapBaseFileSplit.class.getName()); + customSplitInfo.put(CUSTOM_FILE_SPLIT_CLASS_KEY, HoodieRealtimeBootstrapBaseFileSplit.class.getName()); customSplitInfo.put(BASE_PATH_KEY, hudiSplit.getBasePath()); customSplitInfo.put(MAX_COMMIT_TIME_KEY, hudiSplit.getMaxCommitTime()); customSplitInfo.put(DELTA_FILE_PATHS_KEY, String.join(",", hudiSplit.getDeltaLogPaths())); @@ -67,7 +68,7 @@ public Optional recreateFileSplitWithCustomInfo(FileSplit split, Map< { requireNonNull(customSplitInfo); String customFileSplitClass = customSplitInfo.get(CUSTOM_FILE_SPLIT_CLASS_KEY); - if (!isNullOrEmpty(customFileSplitClass) && RealtimeBootstrapBaseFileSplit.class.getName().equals(customFileSplitClass)) { + if (!isNullOrEmpty(customFileSplitClass) && HoodieRealtimeBootstrapBaseFileSplit.class.getName().equals(customFileSplitClass)) { String deltaFilePaths = customSplitInfo.get(DELTA_FILE_PATHS_KEY); List deltaLogPaths = isNullOrEmpty(deltaFilePaths) ? Collections.emptyList() : Arrays.asList(deltaFilePaths.split(",")); List deltaLogFiles = deltaLogPaths.stream().map(p -> new HoodieLogFile(new Path(p))).collect(Collectors.toList()); @@ -76,8 +77,8 @@ public Optional recreateFileSplitWithCustomInfo(FileSplit split, Map< parseLong(customSplitInfo.get(BOOTSTRAP_FILE_SPLIT_START)), parseLong(customSplitInfo.get(BOOTSTRAP_FILE_SPLIT_LEN)), (String[]) null); - split = new RealtimeBootstrapBaseFileSplit(split, customSplitInfo.get(BASE_PATH_KEY), deltaLogFiles, - customSplitInfo.get(MAX_COMMIT_TIME_KEY), bootstrapFileSplit); + split = new HoodieRealtimeBootstrapBaseFileSplit(split, customSplitInfo.get(BASE_PATH_KEY), deltaLogFiles, + customSplitInfo.get(MAX_COMMIT_TIME_KEY), bootstrapFileSplit, false, Option.empty()); return Optional.of(split); } return Optional.empty(); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/HudiRealtimeSplitConverter.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/HudiRealtimeSplitConverter.java index a5b60ef8e2a5..cd2ee66955d0 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/HudiRealtimeSplitConverter.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/HudiRealtimeSplitConverter.java @@ -73,6 +73,7 @@ public Optional recreateFileSplitWithCustomInfo(FileSplit split, Map< requireNonNull(customSplitInfo.get(HUDI_BASEPATH_KEY), "HUDI_BASEPATH_KEY is missing"), deltaLogFiles, requireNonNull(customSplitInfo.get(HUDI_MAX_COMMIT_TIME_KEY), "HUDI_MAX_COMMIT_TIME_KEY is missing"), + false, // incremental query not supported currently Option.empty())); } return Optional.empty(); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/util/TestCustomSplitConversionUtils.java b/presto-hive/src/test/java/com/facebook/presto/hive/util/TestCustomSplitConversionUtils.java index 5b27c75390ac..2f333ac805a9 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/util/TestCustomSplitConversionUtils.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/util/TestCustomSplitConversionUtils.java @@ -19,8 +19,8 @@ import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.util.Option; import org.apache.hudi.hadoop.BootstrapBaseFileSplit; +import org.apache.hudi.hadoop.realtime.HoodieRealtimeBootstrapBaseFileSplit; import org.apache.hudi.hadoop.realtime.HoodieRealtimeFileSplit; -import org.apache.hudi.hadoop.realtime.RealtimeBootstrapBaseFileSplit; import org.testng.annotations.Test; import java.io.IOException; @@ -48,7 +48,7 @@ public void testHudiRealtimeSplitConverterRoundTrip() String expectedMaxCommitTime = "max_commit_time"; FileSplit baseSplit = new FileSplit(FILE_PATH, SPLIT_START_POS, SPLIT_LENGTH, SPLIT_HOSTS); - FileSplit hudiSplit = new HoodieRealtimeFileSplit(baseSplit, BASE_PATH, deltaLogFiles, expectedMaxCommitTime, Option.empty()); + FileSplit hudiSplit = new HoodieRealtimeFileSplit(baseSplit, BASE_PATH, deltaLogFiles, expectedMaxCommitTime, false, Option.empty()); // Test conversion of HudiSplit -> customSplitInfo Map customSplitInfo = CustomSplitConversionUtils.extractCustomSplitInfo(hudiSplit); @@ -74,7 +74,7 @@ public void testHudiRealtimeSplitConverterNoLogRoundTrip() String expectedMaxCommitTime = "max_commit_time"; FileSplit baseSplit = new FileSplit(FILE_PATH, SPLIT_START_POS, SPLIT_LENGTH, SPLIT_HOSTS); - FileSplit hudiSplit = new HoodieRealtimeFileSplit(baseSplit, BASE_PATH, deltaLogFiles, expectedMaxCommitTime, Option.empty()); + FileSplit hudiSplit = new HoodieRealtimeFileSplit(baseSplit, BASE_PATH, deltaLogFiles, expectedMaxCommitTime, false, Option.empty()); // Test conversion of HudiSplit -> customSplitInfo Map customSplitInfo = CustomSplitConversionUtils.extractCustomSplitInfo(hudiSplit); @@ -135,14 +135,14 @@ public void testHudiRealtimeBootstrapBaseFileSplitConverter() FileSplit baseSplit = new FileSplit(FILE_PATH, SPLIT_START_POS, SPLIT_LENGTH, SPLIT_HOSTS); FileSplit bootstrapSourceSplit = new FileSplit(bootstrapSourceFilePath, bootstrapSourceSplitStartPos, bootstrapSourceSplitLength, new String[0]); - FileSplit hudiSplit = new RealtimeBootstrapBaseFileSplit(baseSplit, BASE_PATH, deltaLogFiles, maxCommitTime, - bootstrapSourceSplit); + FileSplit hudiSplit = new HoodieRealtimeBootstrapBaseFileSplit(baseSplit, BASE_PATH, deltaLogFiles, maxCommitTime, + bootstrapSourceSplit, false, Option.empty()); // Test conversion of HudiSplit -> customSplitInfo Map customSplitInfo = CustomSplitConversionUtils.extractCustomSplitInfo(hudiSplit); // Test conversion of (customSplitInfo + baseSplit) -> HudiSplit - RealtimeBootstrapBaseFileSplit recreatedSplit = (RealtimeBootstrapBaseFileSplit) CustomSplitConversionUtils + HoodieRealtimeBootstrapBaseFileSplit recreatedSplit = (HoodieRealtimeBootstrapBaseFileSplit) CustomSplitConversionUtils .recreateSplitWithCustomInfo(baseSplit, customSplitInfo); assertEquals(FILE_PATH, recreatedSplit.getPath());