readCommits = readRecords.stream().map(r -> (GenericRecord) r)
.map(r -> readCommit(r, skipMetadata)).collect(Collectors.toList());
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ClusteringCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ClusteringCommand.java
index 9adae1daa5336..4163f0cb5a6a4 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ClusteringCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ClusteringCommand.java
@@ -116,4 +116,40 @@ public String runClustering(
}
return "Succeeded to run clustering for " + clusteringInstantTime;
}
+
+ /**
+ * Run clustering table service.
+ *
+ * Example:
+ * > connect --path {path to hudi table}
+ * > clustering scheduleAndExecute --sparkMaster local --sparkMemory 2g
+ */
+ @CliCommand(value = "clustering scheduleAndExecute", help = "Run Clustering. Make a cluster plan first and execute that plan immediately")
+ public String runClustering(
+ @CliOption(key = "sparkMaster", unspecifiedDefaultValue = SparkUtil.DEFAULT_SPARK_MASTER, help = "Spark master") final String master,
+ @CliOption(key = "sparkMemory", help = "Spark executor memory", unspecifiedDefaultValue = "4g") final String sparkMemory,
+ @CliOption(key = "parallelism", help = "Parallelism for hoodie clustering", unspecifiedDefaultValue = "1") final String parallelism,
+ @CliOption(key = "retry", help = "Number of retries", unspecifiedDefaultValue = "1") final String retry,
+ @CliOption(key = "propsFilePath", help = "path to properties file on localfs or dfs with configurations for "
+ + "hoodie client for compacting", unspecifiedDefaultValue = "") final String propsFilePath,
+ @CliOption(key = "hoodieConfigs", help = "Any configuration that can be set in the properties file can be "
+ + "passed here in the form of an array", unspecifiedDefaultValue = "") final String[] configs) throws Exception {
+ HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
+ boolean initialized = HoodieCLI.initConf();
+ HoodieCLI.initFS(initialized);
+
+ String sparkPropertiesPath =
+ Utils.getDefaultPropertiesFile(JavaConverters.mapAsScalaMapConverter(System.getenv()).asScala());
+ SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
+ sparkLauncher.addAppArgs(SparkCommand.CLUSTERING_SCHEDULE_AND_EXECUTE.toString(), master, sparkMemory,
+ client.getBasePath(), client.getTableConfig().getTableName(), parallelism, retry, propsFilePath);
+ UtilHelpers.validateAndAddProperties(configs, sparkLauncher);
+ Process process = sparkLauncher.launch();
+ InputStreamConsumer.captureOutput(process);
+ int exitCode = process.waitFor();
+ if (exitCode != 0) {
+ return "Failed to run clustering for scheduleAndExecute.";
+ }
+ return "Succeeded to run clustering for scheduleAndExecute";
+ }
}
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java
index 9517234a0bb60..db1cd207df5a1 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java
@@ -232,7 +232,9 @@ public String rollbackCommit(@CliOption(key = {"commit"}, help = "Commit to roll
@CliOption(key = {"sparkProperties"}, help = "Spark Properties File Path") final String sparkPropertiesPath,
@CliOption(key = "sparkMaster", unspecifiedDefaultValue = "", help = "Spark Master") String master,
@CliOption(key = "sparkMemory", unspecifiedDefaultValue = "4G",
- help = "Spark executor memory") final String sparkMemory)
+ help = "Spark executor memory") final String sparkMemory,
+ @CliOption(key = "rollbackUsingMarkers", unspecifiedDefaultValue = "true",
+ help = "Enabling marker based rollback") final String rollbackUsingMarkers)
throws Exception {
HoodieActiveTimeline activeTimeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
HoodieTimeline completedTimeline = activeTimeline.getCommitsTimeline().filterCompletedInstants();
@@ -243,7 +245,7 @@ public String rollbackCommit(@CliOption(key = {"commit"}, help = "Commit to roll
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
sparkLauncher.addAppArgs(SparkMain.SparkCommand.ROLLBACK.toString(), master, sparkMemory, instantTime,
- HoodieCLI.getTableMetaClient().getBasePath());
+ HoodieCLI.getTableMetaClient().getBasePath(), rollbackUsingMarkers);
Process process = sparkLauncher.launch();
InputStreamConsumer.captureOutput(process);
int exitCode = process.waitFor();
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java
index b6a366bbb75ef..097c68a542c47 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java
@@ -264,6 +264,41 @@ public String compact(
return "Compaction successfully completed for " + compactionInstantTime;
}
+ @CliCommand(value = "compaction scheduleAndExecute", help = "Schedule compaction plan and execute this plan")
+ public String compact(
+ @CliOption(key = {"parallelism"}, mandatory = true,
+ help = "Parallelism for hoodie compaction") final String parallelism,
+ @CliOption(key = "schemaFilePath", mandatory = true,
+ help = "Path for Avro schema file") final String schemaFilePath,
+ @CliOption(key = "sparkMaster", unspecifiedDefaultValue = "local",
+ help = "Spark Master") String master,
+ @CliOption(key = "sparkMemory", unspecifiedDefaultValue = "4G",
+ help = "Spark executor memory") final String sparkMemory,
+ @CliOption(key = "retry", unspecifiedDefaultValue = "1", help = "Number of retries") final String retry,
+ @CliOption(key = "propsFilePath", help = "path to properties file on localfs or dfs with configurations for hoodie client for compacting",
+ unspecifiedDefaultValue = "") final String propsFilePath,
+ @CliOption(key = "hoodieConfigs", help = "Any configuration that can be set in the properties file can be passed here in the form of an array",
+ unspecifiedDefaultValue = "") final String[] configs)
+ throws Exception {
+ HoodieTableMetaClient client = checkAndGetMetaClient();
+ boolean initialized = HoodieCLI.initConf();
+ HoodieCLI.initFS(initialized);
+ String sparkPropertiesPath =
+ Utils.getDefaultPropertiesFile(scala.collection.JavaConversions.propertiesAsScalaMap(System.getProperties()));
+ SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
+ sparkLauncher.addAppArgs(SparkCommand.COMPACT_SCHEDULE_AND_EXECUTE.toString(), master, sparkMemory, client.getBasePath(),
+ client.getTableConfig().getTableName(), parallelism, schemaFilePath,
+ retry, propsFilePath);
+ UtilHelpers.validateAndAddProperties(configs, sparkLauncher);
+ Process process = sparkLauncher.launch();
+ InputStreamConsumer.captureOutput(process);
+ int exitCode = process.waitFor();
+ if (exitCode != 0) {
+ return "Failed to schedule and execute compaction ";
+ }
+ return "Schedule and execute compaction successfully completed";
+ }
+
/**
* Prints all compaction details.
*/
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java
index 119ccb0dcf039..1d8d6dcd6ae93 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java
@@ -34,14 +34,16 @@
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
+import org.apache.hudi.common.util.ClosableIterator;
import org.apache.hudi.exception.HoodieException;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
+
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
import org.apache.avro.specific.SpecificData;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
import org.springframework.shell.core.CommandMarker;
import org.springframework.shell.core.annotation.CliCommand;
import org.springframework.shell.core.annotation.CliOption;
@@ -69,8 +71,8 @@ public class ExportCommand implements CommandMarker {
@CliCommand(value = "export instants", help = "Export Instants and their metadata from the Timeline")
public String exportInstants(
@CliOption(key = {"limit"}, help = "Limit Instants", unspecifiedDefaultValue = "-1") final Integer limit,
- @CliOption(key = {"actions"}, help = "Comma seperated list of Instant actions to export",
- unspecifiedDefaultValue = "clean,commit,deltacommit,rollback,savepoint,restore") final String filter,
+ @CliOption(key = {"actions"}, help = "Comma separated list of Instant actions to export",
+ unspecifiedDefaultValue = "clean,commit,deltacommit,rollback,savepoint,restore") final String filter,
@CliOption(key = {"desc"}, help = "Ordering", unspecifiedDefaultValue = "false") final boolean descending,
@CliOption(key = {"localFolder"}, help = "Local Folder to export to", mandatory = true) String localFolder)
throws Exception {
@@ -122,44 +124,46 @@ private int copyArchivedInstants(List statuses, Set actionSe
// read the avro blocks
while (reader.hasNext() && copyCount < limit) {
HoodieAvroDataBlock blk = (HoodieAvroDataBlock) reader.next();
- for (IndexedRecord ir : blk.getRecords()) {
- // Archived instants are saved as arvo encoded HoodieArchivedMetaEntry records. We need to get the
- // metadata record from the entry and convert it to json.
- HoodieArchivedMetaEntry archiveEntryRecord = (HoodieArchivedMetaEntry) SpecificData.get()
- .deepCopy(HoodieArchivedMetaEntry.SCHEMA$, ir);
-
- final String action = archiveEntryRecord.get("actionType").toString();
- if (!actionSet.contains(action)) {
- continue;
- }
-
- GenericRecord metadata = null;
- switch (action) {
- case HoodieTimeline.CLEAN_ACTION:
- metadata = archiveEntryRecord.getHoodieCleanMetadata();
+ try (ClosableIterator recordItr = blk.getRecordItr()) {
+ while (recordItr.hasNext()) {
+ IndexedRecord ir = recordItr.next();
+ // Archived instants are saved as arvo encoded HoodieArchivedMetaEntry records. We need to get the
+ // metadata record from the entry and convert it to json.
+ HoodieArchivedMetaEntry archiveEntryRecord = (HoodieArchivedMetaEntry) SpecificData.get()
+ .deepCopy(HoodieArchivedMetaEntry.SCHEMA$, ir);
+ final String action = archiveEntryRecord.get("actionType").toString();
+ if (!actionSet.contains(action)) {
+ continue;
+ }
+
+ GenericRecord metadata = null;
+ switch (action) {
+ case HoodieTimeline.CLEAN_ACTION:
+ metadata = archiveEntryRecord.getHoodieCleanMetadata();
+ break;
+ case HoodieTimeline.COMMIT_ACTION:
+ case HoodieTimeline.DELTA_COMMIT_ACTION:
+ metadata = archiveEntryRecord.getHoodieCommitMetadata();
+ break;
+ case HoodieTimeline.ROLLBACK_ACTION:
+ metadata = archiveEntryRecord.getHoodieRollbackMetadata();
+ break;
+ case HoodieTimeline.SAVEPOINT_ACTION:
+ metadata = archiveEntryRecord.getHoodieSavePointMetadata();
+ break;
+ case HoodieTimeline.COMPACTION_ACTION:
+ metadata = archiveEntryRecord.getHoodieCompactionMetadata();
+ break;
+ default:
+ throw new HoodieException("Unknown type of action " + action);
+ }
+
+ final String instantTime = archiveEntryRecord.get("commitTime").toString();
+ final String outPath = localFolder + Path.SEPARATOR + instantTime + "." + action;
+ writeToFile(outPath, HoodieAvroUtils.avroToJson(metadata, true));
+ if (++copyCount == limit) {
break;
- case HoodieTimeline.COMMIT_ACTION:
- case HoodieTimeline.DELTA_COMMIT_ACTION:
- metadata = archiveEntryRecord.getHoodieCommitMetadata();
- break;
- case HoodieTimeline.ROLLBACK_ACTION:
- metadata = archiveEntryRecord.getHoodieRollbackMetadata();
- break;
- case HoodieTimeline.SAVEPOINT_ACTION:
- metadata = archiveEntryRecord.getHoodieSavePointMetadata();
- break;
- case HoodieTimeline.COMPACTION_ACTION:
- metadata = archiveEntryRecord.getHoodieCompactionMetadata();
- break;
- default:
- throw new HoodieException("Unknown type of action " + action);
- }
-
- final String instantTime = archiveEntryRecord.get("commitTime").toString();
- final String outPath = localFolder + Path.SEPARATOR + instantTime + "." + action;
- writeToFile(outPath, HoodieAvroUtils.avroToJson(metadata, true));
- if (++copyCount == limit) {
- break;
+ }
}
}
}
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java
index 27bcd81faefec..4a56858f3926a 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java
@@ -37,6 +37,7 @@
import org.apache.hudi.common.table.log.block.HoodieLogBlock;
import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType;
import org.apache.hudi.common.table.log.block.HoodieLogBlock.HoodieLogBlockType;
+import org.apache.hudi.common.util.ClosableIterator;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieMemoryConfig;
@@ -60,6 +61,7 @@
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import scala.Tuple2;
@@ -100,7 +102,7 @@ public String showLogFileCommits(
while (reader.hasNext()) {
HoodieLogBlock n = reader.next();
String instantTime;
- int recordCount = 0;
+ AtomicInteger recordCount = new AtomicInteger(0);
if (n instanceof HoodieCorruptBlock) {
try {
instantTime = n.getLogBlockHeader().get(HeaderMetadataType.INSTANT_TIME);
@@ -120,17 +122,19 @@ public String showLogFileCommits(
instantTime = "dummy_instant_time_" + dummyInstantTimeCount;
}
if (n instanceof HoodieDataBlock) {
- recordCount = ((HoodieDataBlock) n).getRecords().size();
+ try (ClosableIterator recordItr = ((HoodieDataBlock) n).getRecordItr()) {
+ recordItr.forEachRemaining(r -> recordCount.incrementAndGet());
+ }
}
}
if (commitCountAndMetadata.containsKey(instantTime)) {
commitCountAndMetadata.get(instantTime).add(
- new Tuple3<>(n.getBlockType(), new Tuple2<>(n.getLogBlockHeader(), n.getLogBlockFooter()), recordCount));
+ new Tuple3<>(n.getBlockType(), new Tuple2<>(n.getLogBlockHeader(), n.getLogBlockFooter()), recordCount.get()));
} else {
List, Map>, Integer>> list =
new ArrayList<>();
list.add(
- new Tuple3<>(n.getBlockType(), new Tuple2<>(n.getLogBlockHeader(), n.getLogBlockFooter()), recordCount));
+ new Tuple3<>(n.getBlockType(), new Tuple2<>(n.getLogBlockHeader(), n.getLogBlockFooter()), recordCount.get()));
commitCountAndMetadata.put(instantTime, list);
}
}
@@ -232,11 +236,12 @@ public String showLogFileRecords(
HoodieLogBlock n = reader.next();
if (n instanceof HoodieDataBlock) {
HoodieDataBlock blk = (HoodieDataBlock) n;
- List records = blk.getRecords();
- for (IndexedRecord record : records) {
- if (allRecords.size() < limit) {
- allRecords.add(record);
- }
+ try (ClosableIterator recordItr = blk.getRecordItr()) {
+ recordItr.forEachRemaining(record -> {
+ if (allRecords.size() < limit) {
+ allRecords.add(record);
+ }
+ });
}
}
}
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MarkersCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MarkersCommand.java
new file mode 100644
index 0000000000000..57a4ee1879855
--- /dev/null
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MarkersCommand.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.cli.commands;
+
+import org.apache.hudi.cli.HoodieCLI;
+import org.apache.hudi.cli.utils.InputStreamConsumer;
+import org.apache.hudi.cli.utils.SparkUtil;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.spark.launcher.SparkLauncher;
+import org.springframework.shell.core.CommandMarker;
+import org.springframework.shell.core.annotation.CliCommand;
+import org.springframework.shell.core.annotation.CliOption;
+import org.springframework.stereotype.Component;
+
+/**
+ * CLI command for marker options.
+ */
+@Component
+public class MarkersCommand implements CommandMarker {
+
+ @CliCommand(value = "marker delete", help = "Delete the marker")
+ public String deleteMarker(@CliOption(key = {"commit"}, help = "Delete a marker") final String instantTime,
+ @CliOption(key = {"sparkProperties"}, help = "Spark Properties File Path") final String sparkPropertiesPath,
+ @CliOption(key = "sparkMaster", unspecifiedDefaultValue = "", help = "Spark Master") String master,
+ @CliOption(key = "sparkMemory", unspecifiedDefaultValue = "1G",
+ help = "Spark executor memory") final String sparkMemory)
+ throws Exception {
+ HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
+ SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
+ sparkLauncher.addAppArgs(SparkMain.SparkCommand.DELETE_MARKER.toString(), master, sparkMemory, instantTime,
+ metaClient.getBasePath());
+ Process process = sparkLauncher.launch();
+ InputStreamConsumer.captureOutput(process);
+ int exitCode = process.waitFor();
+ // Refresh the current
+ HoodieCLI.refreshTableMetadata();
+ if (exitCode != 0) {
+ return String.format("Failed: Could not delete marker \"%s\".", instantTime);
+ }
+ return String.format("Marker \"%s\" deleted.", instantTime);
+ }
+}
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
index 2533562d8206e..6c068c898b9be 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
@@ -45,7 +45,6 @@
import org.springframework.stereotype.Component;
import scala.collection.JavaConverters;
-import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.List;
@@ -153,10 +152,12 @@ public String overwriteHoodieProperties(
HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
Properties newProps = new Properties();
- newProps.load(new FileInputStream(new File(overwriteFilePath)));
+ newProps.load(new FileInputStream(overwriteFilePath));
Map oldProps = client.getTableConfig().propsMap();
Path metaPathDir = new Path(client.getBasePath(), METAFOLDER_NAME);
HoodieTableConfig.create(client.getFs(), metaPathDir, newProps);
+ // reload new props as checksum would have been added
+ newProps = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient()).getTableConfig().getProps();
TreeSet allPropKeys = new TreeSet<>();
allPropKeys.addAll(newProps.keySet().stream().map(Object::toString).collect(Collectors.toSet()));
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java
index d1ee109f59042..0de1a1adfe0be 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java
@@ -18,6 +18,7 @@
package org.apache.hudi.cli.commands;
+import org.apache.hadoop.fs.Path;
import org.apache.hudi.DataSourceWriteOptions;
import org.apache.hudi.cli.DeDupeType;
import org.apache.hudi.cli.DedupeSparkJob;
@@ -25,6 +26,7 @@
import org.apache.hudi.client.SparkRDDWriteClient;
import org.apache.hudi.client.common.HoodieSparkEngineContext;
import org.apache.hudi.common.config.TypedProperties;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.HoodieTableVersion;
@@ -38,7 +40,9 @@
import org.apache.hudi.exception.HoodieSavepointException;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.keygen.constant.KeyGeneratorType;
+import org.apache.hudi.table.HoodieSparkTable;
import org.apache.hudi.table.action.compact.strategy.UnBoundedCompactionStrategy;
+import org.apache.hudi.table.marker.WriteMarkersFactory;
import org.apache.hudi.table.upgrade.SparkUpgradeDowngradeHelper;
import org.apache.hudi.table.upgrade.UpgradeDowngrade;
import org.apache.hudi.utilities.HDFSParquetImporter;
@@ -51,8 +55,6 @@
import org.apache.hudi.utilities.UtilHelpers;
import org.apache.hudi.utilities.deltastreamer.BootstrapExecutor;
import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer;
-
-import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.SQLContext;
@@ -74,9 +76,9 @@ public class SparkMain {
* Commands.
*/
enum SparkCommand {
- BOOTSTRAP, ROLLBACK, DEDUPLICATE, ROLLBACK_TO_SAVEPOINT, SAVEPOINT, IMPORT, UPSERT, COMPACT_SCHEDULE, COMPACT_RUN,
+ BOOTSTRAP, ROLLBACK, DEDUPLICATE, ROLLBACK_TO_SAVEPOINT, SAVEPOINT, IMPORT, UPSERT, COMPACT_SCHEDULE, COMPACT_RUN, COMPACT_SCHEDULE_AND_EXECUTE,
COMPACT_UNSCHEDULE_PLAN, COMPACT_UNSCHEDULE_FILE, COMPACT_VALIDATE, COMPACT_REPAIR, CLUSTERING_SCHEDULE,
- CLUSTERING_RUN, CLEAN, DELETE_SAVEPOINT, UPGRADE, DOWNGRADE
+ CLUSTERING_RUN, CLUSTERING_SCHEDULE_AND_EXECUTE, CLEAN, DELETE_MARKER, DELETE_SAVEPOINT, UPGRADE, DOWNGRADE
}
public static void main(String[] args) throws Exception {
@@ -92,8 +94,8 @@ public static void main(String[] args) throws Exception {
try {
switch (cmd) {
case ROLLBACK:
- assert (args.length == 5);
- returnCode = rollback(jsc, args[3], args[4]);
+ assert (args.length == 6);
+ returnCode = rollback(jsc, args[3], args[4], Boolean.parseBoolean(args[5]));
break;
case DEDUPLICATE:
assert (args.length == 8);
@@ -128,7 +130,21 @@ public static void main(String[] args) throws Exception {
configs.addAll(Arrays.asList(args).subList(9, args.length));
}
returnCode = compact(jsc, args[3], args[4], args[5], Integer.parseInt(args[6]), args[7],
- Integer.parseInt(args[8]), false, propsFilePath, configs);
+ Integer.parseInt(args[8]), HoodieCompactor.EXECUTE, propsFilePath, configs);
+ break;
+ case COMPACT_SCHEDULE_AND_EXECUTE:
+ assert (args.length >= 9);
+ propsFilePath = null;
+ if (!StringUtils.isNullOrEmpty(args[8])) {
+ propsFilePath = args[8];
+ }
+ configs = new ArrayList<>();
+ if (args.length > 9) {
+ configs.addAll(Arrays.asList(args).subList(8, args.length));
+ }
+
+ returnCode = compact(jsc, args[3], args[4], null, Integer.parseInt(args[5]), args[6],
+ Integer.parseInt(args[7]), HoodieCompactor.SCHEDULE_AND_EXECUTE, propsFilePath, configs);
break;
case COMPACT_SCHEDULE:
assert (args.length >= 7);
@@ -140,7 +156,7 @@ public static void main(String[] args) throws Exception {
if (args.length > 7) {
configs.addAll(Arrays.asList(args).subList(7, args.length));
}
- returnCode = compact(jsc, args[3], args[4], args[5], 1, "", 0, true, propsFilePath, configs);
+ returnCode = compact(jsc, args[3], args[4], args[5], 1, "", 0, HoodieCompactor.SCHEDULE, propsFilePath, configs);
break;
case COMPACT_VALIDATE:
assert (args.length == 7);
@@ -176,7 +192,20 @@ public static void main(String[] args) throws Exception {
configs.addAll(Arrays.asList(args).subList(9, args.length));
}
returnCode = cluster(jsc, args[3], args[4], args[5], Integer.parseInt(args[6]), args[2],
- Integer.parseInt(args[7]), false, propsFilePath, configs);
+ Integer.parseInt(args[7]), HoodieClusteringJob.EXECUTE, propsFilePath, configs);
+ break;
+ case CLUSTERING_SCHEDULE_AND_EXECUTE:
+ assert (args.length >= 8);
+ propsFilePath = null;
+ if (!StringUtils.isNullOrEmpty(args[7])) {
+ propsFilePath = args[7];
+ }
+ configs = new ArrayList<>();
+ if (args.length > 8) {
+ configs.addAll(Arrays.asList(args).subList(8, args.length));
+ }
+ returnCode = cluster(jsc, args[3], args[4], null, Integer.parseInt(args[5]), args[2],
+ Integer.parseInt(args[6]), HoodieClusteringJob.SCHEDULE_AND_EXECUTE, propsFilePath, configs);
break;
case CLUSTERING_SCHEDULE:
assert (args.length >= 7);
@@ -189,7 +218,7 @@ public static void main(String[] args) throws Exception {
configs.addAll(Arrays.asList(args).subList(7, args.length));
}
returnCode = cluster(jsc, args[3], args[4], args[5], 1, args[2],
- 0, true, propsFilePath, configs);
+ 0, HoodieClusteringJob.SCHEDULE, propsFilePath, configs);
break;
case CLEAN:
assert (args.length >= 5);
@@ -207,6 +236,10 @@ public static void main(String[] args) throws Exception {
assert (args.length == 7);
returnCode = createSavepoint(jsc, args[3], args[4], args[5], args[6]);
break;
+ case DELETE_MARKER:
+ assert (args.length == 5);
+ returnCode = deleteMarker(jsc, args[3], args[4]);
+ break;
case DELETE_SAVEPOINT:
assert (args.length == 5);
returnCode = deleteSavepoint(jsc, args[3], args[4]);
@@ -250,6 +283,21 @@ protected static void clean(JavaSparkContext jsc, String basePath, String propsF
new HoodieCleaner(cfg, jsc).run();
}
+ protected static int deleteMarker(JavaSparkContext jsc, String instantTime, String basePath) {
+ try {
+ SparkRDDWriteClient client = createHoodieClient(jsc, basePath);
+ HoodieWriteConfig config = client.getConfig();
+ HoodieEngineContext context = client.getEngineContext();
+ HoodieSparkTable table = HoodieSparkTable.create(config, context, true);
+ WriteMarkersFactory.get(config.getMarkersType(), table, instantTime)
+ .quietDeleteMarkerDir(context, config.getMarkersDeleteParallelism());
+ return 0;
+ } catch (Exception e) {
+ LOG.warn(String.format("Failed: Could not clean marker instantTime: \"%s\".", instantTime), e);
+ return -1;
+ }
+ }
+
private static int dataLoad(JavaSparkContext jsc, String command, String srcPath, String targetPath, String tableName,
String tableType, String rowKey, String partitionKey, int parallelism, String schemaFile,
int retry, String propsFilePath, List configs) {
@@ -320,7 +368,7 @@ private static void doCompactUnscheduleFile(JavaSparkContext jsc, String basePat
}
private static int compact(JavaSparkContext jsc, String basePath, String tableName, String compactionInstant,
- int parallelism, String schemaFile, int retry, boolean schedule, String propsFilePath,
+ int parallelism, String schemaFile, int retry, String mode, String propsFilePath,
List configs) {
HoodieCompactor.Config cfg = new HoodieCompactor.Config();
cfg.basePath = basePath;
@@ -330,20 +378,20 @@ private static int compact(JavaSparkContext jsc, String basePath, String tableNa
cfg.strategyClassName = UnBoundedCompactionStrategy.class.getCanonicalName();
cfg.parallelism = parallelism;
cfg.schemaFile = schemaFile;
- cfg.runSchedule = schedule;
+ cfg.runningMode = mode;
cfg.propsFilePath = propsFilePath;
cfg.configs = configs;
return new HoodieCompactor(jsc, cfg).compact(retry);
}
private static int cluster(JavaSparkContext jsc, String basePath, String tableName, String clusteringInstant,
- int parallelism, String sparkMemory, int retry, boolean schedule, String propsFilePath, List configs) {
+ int parallelism, String sparkMemory, int retry, String runningMode, String propsFilePath, List configs) {
HoodieClusteringJob.Config cfg = new HoodieClusteringJob.Config();
cfg.basePath = basePath;
cfg.tableName = tableName;
cfg.clusteringInstantTime = clusteringInstant;
cfg.parallelism = parallelism;
- cfg.runSchedule = schedule;
+ cfg.runningMode = runningMode;
cfg.propsFilePath = propsFilePath;
cfg.configs = configs;
jsc.getConf().set("spark.executor.memory", sparkMemory);
@@ -394,8 +442,8 @@ private static int doBootstrap(JavaSparkContext jsc, String tableName, String ta
return 0;
}
- private static int rollback(JavaSparkContext jsc, String instantTime, String basePath) throws Exception {
- SparkRDDWriteClient client = createHoodieClient(jsc, basePath);
+ private static int rollback(JavaSparkContext jsc, String instantTime, String basePath, Boolean rollbackUsingMarkers) throws Exception {
+ SparkRDDWriteClient client = createHoodieClient(jsc, basePath, rollbackUsingMarkers);
if (client.rollback(instantTime)) {
LOG.info(String.format("The commit \"%s\" rolled back.", instantTime));
return 0;
@@ -425,7 +473,7 @@ private static int rollbackToSavepoint(JavaSparkContext jsc, String savepointTim
LOG.info(String.format("The commit \"%s\" rolled back.", savepointTime));
return 0;
} catch (Exception e) {
- LOG.warn(String.format("The commit \"%s\" failed to roll back.", savepointTime));
+ LOG.warn(String.format("The commit \"%s\" failed to roll back.", savepointTime), e);
return -1;
}
}
@@ -437,7 +485,7 @@ private static int deleteSavepoint(JavaSparkContext jsc, String savepointTime, S
LOG.info(String.format("Savepoint \"%s\" deleted.", savepointTime));
return 0;
} catch (Exception e) {
- LOG.warn(String.format("Failed: Could not delete savepoint \"%s\".", savepointTime));
+ LOG.warn(String.format("Failed: Could not delete savepoint \"%s\".", savepointTime), e);
return -1;
}
}
@@ -452,11 +500,12 @@ private static int deleteSavepoint(JavaSparkContext jsc, String savepointTime, S
* @throws Exception
*/
protected static int upgradeOrDowngradeTable(JavaSparkContext jsc, String basePath, String toVersion) {
- HoodieWriteConfig config = getWriteConfig(basePath);
+ HoodieWriteConfig config = getWriteConfig(basePath, Boolean.parseBoolean(HoodieWriteConfig.ROLLBACK_USING_MARKERS_ENABLE.defaultValue()));
HoodieTableMetaClient metaClient =
HoodieTableMetaClient.builder().setConf(jsc.hadoopConfiguration()).setBasePath(config.getBasePath())
.setLoadActiveTimelineOnLoad(false).setConsistencyGuardConfig(config.getConsistencyGuardConfig())
- .setLayoutVersion(Option.of(new TimelineLayoutVersion(config.getTimelineLayoutVersion()))).build();
+ .setLayoutVersion(Option.of(new TimelineLayoutVersion(config.getTimelineLayoutVersion())))
+ .setFileSystemRetryConfig(config.getFileSystemRetryConfig()).build();
try {
new UpgradeDowngrade(metaClient, config, new HoodieSparkEngineContext(jsc), SparkUpgradeDowngradeHelper.getInstance())
.run(HoodieTableVersion.valueOf(toVersion), null);
@@ -468,13 +517,18 @@ protected static int upgradeOrDowngradeTable(JavaSparkContext jsc, String basePa
}
}
- private static SparkRDDWriteClient createHoodieClient(JavaSparkContext jsc, String basePath) throws Exception {
- HoodieWriteConfig config = getWriteConfig(basePath);
+ private static SparkRDDWriteClient createHoodieClient(JavaSparkContext jsc, String basePath, Boolean rollbackUsingMarkers) throws Exception {
+ HoodieWriteConfig config = getWriteConfig(basePath, rollbackUsingMarkers);
return new SparkRDDWriteClient(new HoodieSparkEngineContext(jsc), config);
}
- private static HoodieWriteConfig getWriteConfig(String basePath) {
+ private static SparkRDDWriteClient createHoodieClient(JavaSparkContext jsc, String basePath) throws Exception {
+ return createHoodieClient(jsc, basePath, Boolean.parseBoolean(HoodieWriteConfig.ROLLBACK_USING_MARKERS_ENABLE.defaultValue()));
+ }
+
+ private static HoodieWriteConfig getWriteConfig(String basePath, Boolean rollbackUsingMarkers) {
return HoodieWriteConfig.newBuilder().withPath(basePath)
+ .withRollbackUsingMarkers(rollbackUsingMarkers)
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build()).build();
}
}
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/CommitUtil.java b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/CommitUtil.java
index a95cc53df329c..5d58aa9d2e498 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/CommitUtil.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/CommitUtil.java
@@ -25,9 +25,6 @@
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import java.io.IOException;
-import java.text.ParseException;
-import java.time.Instant;
-import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.Date;
import java.util.List;
@@ -53,16 +50,4 @@ public static String getTimeDaysAgo(int numberOfDays) {
Date date = Date.from(ZonedDateTime.now().minusDays(numberOfDays).toInstant());
return HoodieActiveTimeline.formatDate(date);
}
-
- /**
- * Add hours to specified time. If hours <0, this acts as remove hours.
- * example, say compactionCommitTime: "20200202020000"
- * a) hours: +1, returns 20200202030000
- * b) hours: -1, returns 20200202010000
- */
- public static String addHours(String compactionCommitTime, int hours) throws ParseException {
- Instant instant = HoodieActiveTimeline.parseDateFromInstantTime(compactionCommitTime).toInstant();
- ZonedDateTime commitDateTime = ZonedDateTime.ofInstant(instant, ZoneId.systemDefault());
- return HoodieActiveTimeline.formatDate(Date.from(commitDateTime.plusHours(hours).toInstant()));
- }
}
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkTempViewProvider.java b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkTempViewProvider.java
index bbd8440448fd6..6f5a11ad6657f 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkTempViewProvider.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkTempViewProvider.java
@@ -20,8 +20,6 @@
import org.apache.hudi.exception.HoodieException;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
@@ -31,27 +29,34 @@
import org.apache.spark.sql.types.DataType;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.StructType;
+import org.springframework.shell.support.logging.HandlerUtils;
import java.util.List;
+import java.util.logging.Handler;
+import java.util.logging.Level;
+import java.util.logging.Logger;
import java.util.stream.Collectors;
public class SparkTempViewProvider implements TempViewProvider {
- private static final Logger LOG = LogManager.getLogger(SparkTempViewProvider.class);
+ private static final Logger LOG = HandlerUtils.getLogger(SparkTempViewProvider.class);
private JavaSparkContext jsc;
private SQLContext sqlContext;
public SparkTempViewProvider(String appName) {
try {
+ Handler handler = LOG.getParent().getHandlers()[0];
SparkConf sparkConf = new SparkConf().setAppName(appName)
.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer").setMaster("local[8]");
jsc = new JavaSparkContext(sparkConf);
- jsc.setLogLevel("ERROR");
-
sqlContext = new SQLContext(jsc);
+ if (handler != null) {
+ LOG.getParent().removeHandler(LOG.getParent().getHandlers()[0]);
+ LOG.getParent().addHandler(handler);
+ }
} catch (Throwable ex) {
// log full stack trace and rethrow. Without this its difficult to debug failures, if any
- LOG.error("unable to initialize spark context ", ex);
+ LOG.log(Level.WARNING, "unable to initialize spark context ", ex);
throw new HoodieException(ex);
}
}
@@ -90,7 +95,7 @@ public void createOrReplace(String tableName, List headers, List entry : data.entrySet()) {
String key = entry.getKey();
Integer[] value = entry.getValue();
@@ -279,8 +295,8 @@ public void testShowArchivedCommitsWithMultiCommitsFile() throws Exception {
HoodieSparkTable table = HoodieSparkTable.create(cfg, context(), metaClient);
// need to create multi archive files
- HoodieTimelineArchiveLog archiveLog = new HoodieTimelineArchiveLog(cfg, table);
- archiveLog.archiveIfRequired(context());
+ HoodieTimelineArchiver archiver = new HoodieTimelineArchiver(cfg, table);
+ archiver.archiveIfRequired(context());
}
CommandResult cr = shell().executeCommand(String.format("commits showarchived --startTs %s --endTs %s", "160", "174"));
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCompactionCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCompactionCommand.java
index 21841a5769450..17c1002f6b0dd 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCompactionCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCompactionCommand.java
@@ -24,7 +24,9 @@
import org.apache.hudi.cli.TableHeader;
import org.apache.hudi.cli.functional.CLIFunctionalTestHarness;
import org.apache.hudi.cli.testutils.HoodieTestCommitMetadataGenerator;
-import org.apache.hudi.common.config.HoodieMetadataConfig;
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.fs.HoodieWrapperFileSystem;
+import org.apache.hudi.common.fs.NoOpConsistencyGuard;
import org.apache.hudi.common.model.HoodieAvroPayload;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -35,12 +37,13 @@
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.testutils.CompactionTestUtils;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.table.HoodieSparkTable;
-import org.apache.hudi.table.HoodieTimelineArchiveLog;
+import org.apache.hudi.client.HoodieTimelineArchiver;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
@@ -152,7 +155,11 @@ private void generateCompactionInstances() throws IOException {
activeTimeline.transitionCompactionInflightToComplete(
new HoodieInstant(HoodieInstant.State.INFLIGHT, COMPACTION_ACTION, timestamp), Option.empty());
});
-
+ // Simulate a compaction commit in metadata table timeline
+ // so the archival in data table can happen
+ HoodieTestUtils.createCompactionCommitInMetadataTable(hadoopConf(),
+ new HoodieWrapperFileSystem(
+ FSUtils.getFs(tablePath, hadoopConf()), new NoOpConsistencyGuard()), tablePath, "007");
}
private void generateArchive() throws IOException {
@@ -162,13 +169,12 @@ private void generateArchive() throws IOException {
.withCompactionConfig(HoodieCompactionConfig.newBuilder().retainCommits(1).archiveCommitsWith(2, 3).build())
.withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder()
.withRemoteServerPort(timelineServicePort).build())
- .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build())
.forTable("test-trip-table").build();
// archive
HoodieTableMetaClient metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
HoodieSparkTable table = HoodieSparkTable.create(cfg, context(), metaClient);
- HoodieTimelineArchiveLog archiveLog = new HoodieTimelineArchiveLog(cfg, table);
- archiveLog.archiveIfRequired(context());
+ HoodieTimelineArchiver archiver = new HoodieTimelineArchiver(cfg, table);
+ archiver.archiveIfRequired(context());
}
/**
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java
index f2571ce3598d6..ee7fbda11b783 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java
@@ -108,7 +108,7 @@ public void init() throws IOException, InterruptedException, URISyntaxException
Map header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, INSTANT_TIME);
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, getSimpleSchema().toString());
- dataBlock = new HoodieAvroDataBlock(records, header);
+ dataBlock = new HoodieAvroDataBlock(records, header, HoodieRecord.RECORD_KEY_METADATA_FIELD);
writer.appendBlock(dataBlock);
}
}
@@ -188,7 +188,7 @@ public void testShowLogFileRecordsWithMerge() throws IOException, InterruptedExc
Map header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, INSTANT_TIME);
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString());
- HoodieAvroDataBlock dataBlock = new HoodieAvroDataBlock(records1, header);
+ HoodieAvroDataBlock dataBlock = new HoodieAvroDataBlock(records1, header, HoodieRecord.RECORD_KEY_METADATA_FIELD);
writer.appendBlock(dataBlock);
} finally {
if (writer != null) {
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java
index 048b2a20e6b2c..27cc31ccea2cf 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java
@@ -39,7 +39,6 @@
import org.junit.jupiter.api.Test;
import org.springframework.shell.core.CommandResult;
-import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URL;
@@ -51,6 +50,14 @@
import java.util.Properties;
import java.util.stream.Collectors;
+import static org.apache.hudi.common.table.HoodieTableConfig.ARCHIVELOG_FOLDER;
+import static org.apache.hudi.common.table.HoodieTableConfig.NAME;
+import static org.apache.hudi.common.table.HoodieTableConfig.TABLE_CHECKSUM;
+import static org.apache.hudi.common.table.HoodieTableConfig.TIMELINE_LAYOUT_VERSION;
+import static org.apache.hudi.common.table.HoodieTableConfig.TYPE;
+import static org.apache.hudi.common.table.HoodieTableConfig.VERSION;
+import static org.apache.hudi.common.table.HoodieTableConfig.generateChecksum;
+import static org.apache.hudi.common.table.HoodieTableConfig.validateChecksum;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -104,7 +111,7 @@ public void testAddPartitionMetaWithDryRun() throws IOException {
// expected all 'No'.
String[][] rows = FSUtils.getAllPartitionFoldersThreeLevelsDown(fs, tablePath)
.stream()
- .map(partition -> new String[]{partition, "No", "None"})
+ .map(partition -> new String[] {partition, "No", "None"})
.toArray(String[][]::new);
String expected = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_PARTITION_PATH,
HoodieTableHeaderFields.HEADER_METADATA_PRESENT, HoodieTableHeaderFields.HEADER_ACTION}, rows);
@@ -135,7 +142,7 @@ public void testAddPartitionMetaWithRealRun() throws IOException {
List paths = FSUtils.getAllPartitionFoldersThreeLevelsDown(fs, tablePath);
// after dry run, the action will be 'Repaired'
String[][] rows = paths.stream()
- .map(partition -> new String[]{partition, "No", "Repaired"})
+ .map(partition -> new String[] {partition, "No", "Repaired"})
.toArray(String[][]::new);
String expected = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_PARTITION_PATH,
HoodieTableHeaderFields.HEADER_METADATA_PRESENT, HoodieTableHeaderFields.HEADER_ACTION}, rows);
@@ -147,7 +154,7 @@ public void testAddPartitionMetaWithRealRun() throws IOException {
// after real run, Metadata is present now.
rows = paths.stream()
- .map(partition -> new String[]{partition, "Yes", "None"})
+ .map(partition -> new String[] {partition, "Yes", "None"})
.toArray(String[][]::new);
expected = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_PARTITION_PATH,
HoodieTableHeaderFields.HEADER_METADATA_PRESENT, HoodieTableHeaderFields.HEADER_ACTION}, rows);
@@ -170,19 +177,24 @@ public void testOverwriteHoodieProperties() throws IOException {
Map oldProps = HoodieCLI.getTableMetaClient().getTableConfig().propsMap();
// after overwrite, the stored value in .hoodie is equals to which read from properties.
- Map result = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient()).getTableConfig().propsMap();
+ HoodieTableConfig tableConfig = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient()).getTableConfig();
+ Map result = tableConfig.propsMap();
+ // validate table checksum
+ assertTrue(result.containsKey(TABLE_CHECKSUM.key()));
+ assertTrue(validateChecksum(tableConfig.getProps()));
Properties expectProps = new Properties();
- expectProps.load(new FileInputStream(new File(newProps.getPath())));
+ expectProps.load(new FileInputStream(newProps.getPath()));
Map expected = expectProps.entrySet().stream()
.collect(Collectors.toMap(e -> String.valueOf(e.getKey()), e -> String.valueOf(e.getValue())));
+ expected.putIfAbsent(TABLE_CHECKSUM.key(), String.valueOf(generateChecksum(tableConfig.getProps())));
assertEquals(expected, result);
// check result
- List allPropsStr = Arrays.asList("hoodie.table.name", "hoodie.table.type", "hoodie.table.version",
- "hoodie.archivelog.folder", "hoodie.timeline.layout.version");
- String[][] rows = allPropsStr.stream().sorted().map(key -> new String[]{key,
- oldProps.getOrDefault(key, "null"), result.getOrDefault(key, "null")})
+ List allPropsStr = Arrays.asList(NAME.key(), TYPE.key(), VERSION.key(),
+ ARCHIVELOG_FOLDER.key(), TIMELINE_LAYOUT_VERSION.key(), TABLE_CHECKSUM.key());
+ String[][] rows = allPropsStr.stream().sorted().map(key -> new String[] {key,
+ oldProps.getOrDefault(key, "null"), result.getOrDefault(key, "null")})
.toArray(String[][]::new);
String expect = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_HOODIE_PROPERTY,
HoodieTableHeaderFields.HEADER_OLD_VALUE, HoodieTableHeaderFields.HEADER_NEW_VALUE}, rows);
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java
index 17bc48f66f0c4..9a10893b35e89 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java
@@ -24,7 +24,7 @@
import org.apache.hudi.cli.HoodieTableHeaderFields;
import org.apache.hudi.cli.TableHeader;
import org.apache.hudi.cli.functional.CLIFunctionalTestHarness;
-import org.apache.hudi.client.AbstractHoodieWriteClient;
+import org.apache.hudi.client.BaseHoodieWriteClient;
import org.apache.hudi.client.SparkRDDWriteClient;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -93,7 +93,7 @@ public void init() throws Exception {
.withRollbackUsingMarkers(false)
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()).build();
- try (AbstractHoodieWriteClient client = new SparkRDDWriteClient(context(), config)) {
+ try (BaseHoodieWriteClient client = new SparkRDDWriteClient(context(), config)) {
// Rollback inflight commit3 and commit2
client.rollback("102");
client.rollback("101");
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java
index cba6d901b956d..b3650fa027626 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java
@@ -106,7 +106,7 @@ public void testDowngradeCommand() throws Exception {
assertEquals(metaClient.getTableConfig().getTableVersion().versionCode(), HoodieTableVersion.ZERO.versionCode());
assertTableVersionFromPropertyFile();
- // verify marker files are non existant
+ // verify marker files are non existent
for (String partitionPath : DEFAULT_PARTITION_PATHS) {
assertEquals(0, FileCreateUtils.getTotalMarkerFileCount(tablePath, partitionPath, "101", IOType.MERGE));
}
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestClusteringCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestClusteringCommand.java
new file mode 100644
index 0000000000000..17075f9d3dfb6
--- /dev/null
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestClusteringCommand.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.cli.integ;
+
+import org.apache.hudi.cli.HoodieCLI;
+import org.apache.hudi.cli.commands.TableCommand;
+import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest;
+import org.apache.hudi.client.SparkRDDWriteClient;
+import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.client.common.HoodieSparkEngineContext;
+import org.apache.hudi.common.model.HoodieAvroPayload;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.config.HoodieIndexConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.index.HoodieIndex;
+import org.apache.hudi.testutils.HoodieClientTestBase;
+
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.shell.core.CommandResult;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertAll;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Integration test class for {@link org.apache.hudi.cli.commands.ClusteringCommand}.
+ *
+ * A command use SparkLauncher need load jars under lib which generate during mvn package.
+ * Use integration test instead of unit test.
+ */
+public class ITTestClusteringCommand extends AbstractShellIntegrationTest {
+
+ private String tablePath;
+ private String tableName;
+
+ @BeforeEach
+ public void init() throws IOException {
+ tableName = "test_table_" + ITTestClusteringCommand.class.getName();
+ tablePath = Paths.get(basePath, tableName).toString();
+
+ HoodieCLI.conf = jsc.hadoopConfiguration();
+ // Create table and connect
+ new TableCommand().createTable(
+ tablePath, tableName, HoodieTableType.COPY_ON_WRITE.name(),
+ "", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload");
+ metaClient.setBasePath(tablePath);
+ metaClient = HoodieTableMetaClient.reload(metaClient);
+ }
+
+ /**
+ * Test case for command 'clustering schedule'.
+ */
+ @Test
+ public void testScheduleClustering() throws IOException {
+ // generate commits
+ generateCommits();
+
+ CommandResult cr = scheduleClustering();
+ assertAll("Command run failed",
+ () -> assertTrue(cr.isSuccess()),
+ () -> assertTrue(
+ cr.getResult().toString().startsWith("Succeeded to schedule clustering for")));
+
+ // there is 1 requested clustering
+ HoodieActiveTimeline timeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
+ assertEquals(1, timeline.filterPendingReplaceTimeline().countInstants());
+ }
+
+ /**
+ * Test case for command 'clustering run'.
+ */
+ @Test
+ public void testClustering() throws IOException {
+ // generate commits
+ generateCommits();
+
+ CommandResult cr1 = scheduleClustering();
+ assertTrue(cr1.isSuccess());
+
+ // get clustering instance
+ HoodieActiveTimeline timeline = HoodieCLI.getTableMetaClient().getActiveTimeline();
+ Option instance =
+ timeline.filterPendingReplaceTimeline().firstInstant().map(HoodieInstant::getTimestamp);
+ assertTrue(instance.isPresent(), "Must have pending clustering.");
+
+ CommandResult cr2 = getShell().executeCommand(
+ String.format("clustering run --parallelism %s --clusteringInstant %s --sparkMaster %s",
+ 2, instance, "local"));
+
+ assertAll("Command run failed",
+ () -> assertTrue(cr2.isSuccess()),
+ () -> assertTrue(
+ cr2.getResult().toString().startsWith("Succeeded to run clustering for ")));
+
+ // assert clustering complete
+ assertTrue(HoodieCLI.getTableMetaClient().getActiveTimeline().reload()
+ .filterCompletedInstants().getInstants()
+ .map(HoodieInstant::getTimestamp).collect(Collectors.toList()).contains(instance),
+ "Pending clustering must be completed");
+
+ assertTrue(HoodieCLI.getTableMetaClient().getActiveTimeline().reload()
+ .getCompletedReplaceTimeline().getInstants()
+ .map(HoodieInstant::getTimestamp).collect(Collectors.toList()).contains(instance),
+ "Pending clustering must be completed");
+ }
+
+ /**
+ * Test case for command 'clustering scheduleAndExecute'.
+ */
+ @Test
+ public void testClusteringScheduleAndExecute() throws IOException {
+ // generate commits
+ generateCommits();
+
+ CommandResult cr2 = getShell().executeCommand(
+ String.format("clustering scheduleAndExecute --parallelism %s --sparkMaster %s", 2, "local"));
+
+ assertAll("Command run failed",
+ () -> assertTrue(cr2.isSuccess()),
+ () -> assertTrue(
+ cr2.getResult().toString().startsWith("Succeeded to run clustering for scheduleAndExecute")));
+
+ // assert clustering complete
+ assertTrue(HoodieCLI.getTableMetaClient().getActiveTimeline().reload()
+ .getCompletedReplaceTimeline().getInstants()
+ .map(HoodieInstant::getTimestamp).count() > 0,
+ "Completed clustering couldn't be 0");
+ }
+
+ private CommandResult scheduleClustering() {
+ // generate requested clustering
+ return getShell().executeCommand(
+ String.format("clustering schedule --hoodieConfigs hoodie.clustering.inline.max.commits=1 --sparkMaster %s", "local"));
+ }
+
+ private void generateCommits() throws IOException {
+ HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator();
+
+ // Create the write client to write some records in
+ HoodieWriteConfig cfg = HoodieWriteConfig.newBuilder().withPath(tablePath)
+ .withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2)
+ .withDeleteParallelism(2).forTable(tableName)
+ .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build()).build();
+
+ SparkRDDWriteClient client = new SparkRDDWriteClient<>(new HoodieSparkEngineContext(jsc), cfg);
+
+ insert(jsc, client, dataGen, "001");
+ insert(jsc, client, dataGen, "002");
+ }
+
+ private List insert(JavaSparkContext jsc, SparkRDDWriteClient client,
+ HoodieTestDataGenerator dataGen, String newCommitTime) throws IOException {
+ // inserts
+ client.startCommitWithTime(newCommitTime);
+
+ List records = dataGen.generateInserts(newCommitTime, 10);
+ JavaRDD writeRecords = jsc.parallelize(records, 1);
+ operateFunc(SparkRDDWriteClient::insert, client, writeRecords, newCommitTime);
+ return records;
+ }
+
+ private JavaRDD operateFunc(
+ HoodieClientTestBase.Function3, SparkRDDWriteClient, JavaRDD, String> writeFn,
+ SparkRDDWriteClient client, JavaRDD writeRecords, String commitTime)
+ throws IOException {
+ return writeFn.apply(client, writeRecords, commitTime);
+ }
+}
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java
index b3c5c06be9a29..18f4a387d474e 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java
@@ -100,5 +100,18 @@ public void testRollbackCommit() throws Exception {
HoodieActiveTimeline timeline = metaClient.reloadActiveTimeline();
assertEquals(2, timeline.getCommitsTimeline().countInstants(), "There should have 2 instants.");
+
+ // rollback complete commit
+ CommandResult cr2 = getShell().executeCommand(String.format("commit rollback --commit %s --sparkMaster %s --sparkMemory %s",
+ "101", "local", "4G"));
+ assertTrue(cr2.isSuccess());
+
+ metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
+
+ HoodieActiveTimeline rollbackTimeline2 = new RollbacksCommand.RollbackTimeline(metaClient);
+ assertEquals(1, rollbackTimeline2.getRollbackTimeline().countInstants(), "There should have 2 rollback instant.");
+
+ HoodieActiveTimeline timeline2 = metaClient.reloadActiveTimeline();
+ assertEquals(2, timeline2.getCommitsTimeline().countInstants(), "There should have 1 instants.");
}
}
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCompactionCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCompactionCommand.java
index 37a2098d0cd18..4734f45e7074b 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCompactionCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCompactionCommand.java
@@ -140,6 +140,33 @@ public void testCompact() throws IOException {
"Pending compaction must be completed");
}
+ /**
+ * Test case for command 'compaction scheduleAndExecute'.
+ */
+ @Test
+ public void testCompactScheduleAndExecute() throws IOException {
+ // generate commits
+ generateCommits();
+
+ String schemaPath = Paths.get(basePath, "compaction.schema").toString();
+ writeSchemaToTmpFile(schemaPath);
+
+ CommandResult cr2 = getShell().executeCommand(
+ String.format("compaction scheduleAndExecute --parallelism %s --schemaFilePath %s --sparkMaster %s",
+ 2, schemaPath, "local"));
+
+ assertAll("Command run failed",
+ () -> assertTrue(cr2.isSuccess()),
+ () -> assertTrue(
+ cr2.getResult().toString().startsWith("Schedule and execute compaction successfully completed")));
+
+ // assert compaction complete
+ assertTrue(HoodieCLI.getTableMetaClient().getActiveTimeline().reload()
+ .filterCompletedInstants().getInstants()
+ .map(HoodieInstant::getTimestamp).count() > 0,
+ "Completed compaction couldn't be 0");
+ }
+
/**
* Test case for command 'compaction validate'.
*/
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestMarkersCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestMarkersCommand.java
new file mode 100644
index 0000000000000..221a29f5250d2
--- /dev/null
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestMarkersCommand.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.cli.integ;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hudi.cli.commands.TableCommand;
+import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.model.IOType;
+import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
+import org.apache.hudi.common.testutils.FileCreateUtils;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.shell.core.CommandResult;
+
+import java.io.IOException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Integration test class for {@link org.apache.hudi.cli.commands.MarkersCommand}.
+ *
+ * A command use SparkLauncher need load jars under lib which generate during mvn package.
+ * Use integration test instead of unit test.
+ */
+public class ITTestMarkersCommand extends AbstractShellIntegrationTest {
+
+ private String tablePath;
+
+ @BeforeEach
+ public void init() throws IOException {
+ String tableName = "test_table";
+ tablePath = basePath + Path.SEPARATOR + tableName;
+
+ // Create table and connect
+ new TableCommand().createTable(
+ tablePath, "test_table", HoodieTableType.COPY_ON_WRITE.name(),
+ "", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload");
+ }
+
+ /**
+ * Test case of command 'marker delete'.
+ */
+ @Test
+ public void testDeleteMarker() throws IOException {
+ // generate markers
+ String instantTime1 = "101";
+
+ FileCreateUtils.createMarkerFile(tablePath, "partA", instantTime1, "f0", IOType.APPEND);
+ FileCreateUtils.createMarkerFile(tablePath, "partA", instantTime1, "f1", IOType.APPEND);
+
+ assertEquals(2, FileCreateUtils.getTotalMarkerFileCount(tablePath, "partA", instantTime1, IOType.APPEND));
+
+ CommandResult cr = getShell().executeCommand(
+ String.format("marker delete --commit %s --sparkMaster %s", instantTime1, "local"));
+ assertTrue(cr.isSuccess());
+
+ assertEquals(0, FileCreateUtils.getTotalMarkerFileCount(tablePath, "partA", instantTime1, IOType.APPEND));
+ }
+}
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java
index 105a9f639c792..f59dca4e1ea9f 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java
@@ -73,9 +73,9 @@ public static void createCommitFileWithMetadata(String basePath, String commitTi
}
public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration,
- Option writes, Option updates, Map extraMetdata) throws Exception {
+ Option writes, Option updates, Map extraMetadata) throws Exception {
createCommitFileWithMetadata(basePath, commitTime, configuration, UUID.randomUUID().toString(),
- UUID.randomUUID().toString(), writes, updates, extraMetdata);
+ UUID.randomUUID().toString(), writes, updates, extraMetadata);
}
public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration,
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/SparkUtilTest.java b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/SparkUtilTest.java
new file mode 100644
index 0000000000000..4966438292949
--- /dev/null
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/SparkUtilTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.cli.testutils;
+
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.cli.utils.SparkUtil;
+import org.apache.spark.SparkConf;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class SparkUtilTest {
+ @Test
+ public void testGetDefaultSparkConf() {
+ SparkConf sparkConf = SparkUtil.getDefaultConf("test-spark-app", Option.of(""));
+ assertEquals(SparkUtil.DEFAULT_SPARK_MASTER, sparkConf.get("spark.master"));
+ }
+}
diff --git a/hudi-client/hudi-client-common/pom.xml b/hudi-client/hudi-client-common/pom.xml
index a9209f5534df8..a55a136652728 100644
--- a/hudi-client/hudi-client-common/pom.xml
+++ b/hudi-client/hudi-client-common/pom.xml
@@ -163,7 +163,6 @@
org.awaitility
awaitility
- 3.1.2
test
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncArchiveService.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncArchiveService.java
new file mode 100644
index 0000000000000..3fdc21dd21683
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncArchiveService.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.async;
+
+import org.apache.hudi.client.BaseHoodieWriteClient;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieException;
+
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * Async archive service to run concurrently with write operation.
+ */
+public class AsyncArchiveService extends HoodieAsyncTableService {
+
+ private static final Logger LOG = LogManager.getLogger(AsyncArchiveService.class);
+
+ private final BaseHoodieWriteClient writeClient;
+ private final transient ExecutorService executor = Executors.newSingleThreadExecutor();
+
+ protected AsyncArchiveService(BaseHoodieWriteClient writeClient) {
+ super(writeClient.getConfig());
+ this.writeClient = writeClient;
+ }
+
+ @Override
+ protected Pair startService() {
+ LOG.info("Starting async archive service...");
+ return Pair.of(CompletableFuture.supplyAsync(() -> {
+ writeClient.archive();
+ return true;
+ }, executor), executor);
+ }
+
+ public static AsyncArchiveService startAsyncArchiveIfEnabled(BaseHoodieWriteClient writeClient) {
+ HoodieWriteConfig config = writeClient.getConfig();
+ if (!config.isAutoArchive() || !config.isAsyncArchive()) {
+ LOG.info("The HoodieWriteClient is not configured to auto & async archive. Async archive service will not start.");
+ return null;
+ }
+ AsyncArchiveService asyncArchiveService = new AsyncArchiveService(writeClient);
+ asyncArchiveService.start(null);
+ return asyncArchiveService;
+ }
+
+ public static void waitForCompletion(AsyncArchiveService asyncArchiveService) {
+ if (asyncArchiveService != null) {
+ LOG.info("Waiting for async archive service to finish");
+ try {
+ asyncArchiveService.waitForShutdown();
+ } catch (Exception e) {
+ throw new HoodieException("Error waiting for async archive service to finish", e);
+ }
+ }
+ }
+
+ public static void forceShutdown(AsyncArchiveService asyncArchiveService) {
+ if (asyncArchiveService != null) {
+ LOG.info("Shutting down async archive service...");
+ asyncArchiveService.shutdown(true);
+ }
+ }
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AsyncCleanerService.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncCleanerService.java
similarity index 56%
rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AsyncCleanerService.java
rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncCleanerService.java
index a5a38f2cc5949..72907e6d3fbcd 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AsyncCleanerService.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncCleanerService.java
@@ -7,21 +7,24 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
-package org.apache.hudi.client;
+package org.apache.hudi.async;
-import org.apache.hudi.async.HoodieAsyncService;
+import org.apache.hudi.client.BaseHoodieWriteClient;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
+
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -30,54 +33,55 @@
import java.util.concurrent.Executors;
/**
- * Clean service running concurrently with write operation.
+ * Async clean service to run concurrently with write operation.
*/
-class AsyncCleanerService extends HoodieAsyncService {
+public class AsyncCleanerService extends HoodieAsyncTableService {
private static final Logger LOG = LogManager.getLogger(AsyncCleanerService.class);
- private final AbstractHoodieWriteClient writeClient;
+ private final BaseHoodieWriteClient writeClient;
private final transient ExecutorService executor = Executors.newSingleThreadExecutor();
- protected AsyncCleanerService(AbstractHoodieWriteClient writeClient) {
+ protected AsyncCleanerService(BaseHoodieWriteClient writeClient) {
+ super(writeClient.getConfig());
this.writeClient = writeClient;
}
@Override
protected Pair startService() {
String instantTime = HoodieActiveTimeline.createNewInstantTime();
- LOG.info("Auto cleaning is enabled. Running cleaner async to write operation at instant time " + instantTime);
+ LOG.info(String.format("Starting async clean service with instant time %s...", instantTime));
return Pair.of(CompletableFuture.supplyAsync(() -> {
writeClient.clean(instantTime);
return true;
}, executor), executor);
}
- public static AsyncCleanerService startAsyncCleaningIfEnabled(AbstractHoodieWriteClient writeClient) {
- AsyncCleanerService asyncCleanerService = null;
- if (writeClient.getConfig().isAutoClean() && writeClient.getConfig().isAsyncClean()) {
- asyncCleanerService = new AsyncCleanerService(writeClient);
- asyncCleanerService.start(null);
- } else {
- LOG.info("Async auto cleaning is not enabled. Not running cleaner now");
+ public static AsyncCleanerService startAsyncCleaningIfEnabled(BaseHoodieWriteClient writeClient) {
+ HoodieWriteConfig config = writeClient.getConfig();
+ if (!config.isAutoClean() || !config.isAsyncClean()) {
+ LOG.info("The HoodieWriteClient is not configured to auto & async clean. Async clean service will not start.");
+ return null;
}
+ AsyncCleanerService asyncCleanerService = new AsyncCleanerService(writeClient);
+ asyncCleanerService.start(null);
return asyncCleanerService;
}
public static void waitForCompletion(AsyncCleanerService asyncCleanerService) {
if (asyncCleanerService != null) {
- LOG.info("Waiting for async cleaner to finish");
+ LOG.info("Waiting for async clean service to finish");
try {
asyncCleanerService.waitForShutdown();
} catch (Exception e) {
- throw new HoodieException("Error waiting for async cleaning to finish", e);
+ throw new HoodieException("Error waiting for async clean service to finish", e);
}
}
}
public static void forceShutdown(AsyncCleanerService asyncCleanerService) {
if (asyncCleanerService != null) {
- LOG.info("Shutting down async cleaner");
+ LOG.info("Shutting down async clean service...");
asyncCleanerService.shutdown(true);
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncClusteringService.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncClusteringService.java
index b9707bb6d82a7..1c1cf2bb9f74b 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncClusteringService.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncClusteringService.java
@@ -19,8 +19,8 @@
package org.apache.hudi.async;
-import org.apache.hudi.client.AbstractClusteringClient;
-import org.apache.hudi.client.AbstractHoodieWriteClient;
+import org.apache.hudi.client.BaseClusterer;
+import org.apache.hudi.client.BaseHoodieWriteClient;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieIOException;
@@ -38,25 +38,25 @@
* Async clustering service that runs in a separate thread.
* Currently, only one clustering thread is allowed to run at any time.
*/
-public abstract class AsyncClusteringService extends HoodieAsyncService {
+public abstract class AsyncClusteringService extends HoodieAsyncTableService {
private static final long serialVersionUID = 1L;
private static final Logger LOG = LogManager.getLogger(AsyncClusteringService.class);
private final int maxConcurrentClustering;
- private transient AbstractClusteringClient clusteringClient;
+ private transient BaseClusterer clusteringClient;
- public AsyncClusteringService(AbstractHoodieWriteClient writeClient) {
+ public AsyncClusteringService(BaseHoodieWriteClient writeClient) {
this(writeClient, false);
}
- public AsyncClusteringService(AbstractHoodieWriteClient writeClient, boolean runInDaemonMode) {
- super(runInDaemonMode);
+ public AsyncClusteringService(BaseHoodieWriteClient writeClient, boolean runInDaemonMode) {
+ super(writeClient.getConfig(), runInDaemonMode);
this.clusteringClient = createClusteringClient(writeClient);
this.maxConcurrentClustering = 1;
}
- protected abstract AbstractClusteringClient createClusteringClient(AbstractHoodieWriteClient client);
+ protected abstract BaseClusterer createClusteringClient(BaseHoodieWriteClient client);
/**
* Start clustering service.
@@ -82,10 +82,16 @@ protected Pair startService() {
}
LOG.info("Clustering executor shutting down properly");
} catch (InterruptedException ie) {
+ hasError = true;
LOG.warn("Clustering executor got interrupted exception! Stopping", ie);
} catch (IOException e) {
- LOG.error("Clustering executor failed", e);
+ hasError = true;
+ LOG.error("Clustering executor failed due to IOException", e);
throw new HoodieIOException(e.getMessage(), e);
+ } catch (Exception e) {
+ hasError = true;
+ LOG.error("Clustering executor failed", e);
+ throw e;
}
return true;
}, executor)).toArray(CompletableFuture[]::new)), executor);
@@ -94,7 +100,7 @@ protected Pair startService() {
/**
* Update the write client to be used for clustering.
*/
- public synchronized void updateWriteClient(AbstractHoodieWriteClient writeClient) {
+ public synchronized void updateWriteClient(BaseHoodieWriteClient writeClient) {
this.clusteringClient.updateWriteClient(writeClient);
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncCompactService.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncCompactService.java
index 2f63297210e14..f1f7f416e466c 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncCompactService.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncCompactService.java
@@ -17,8 +17,8 @@
package org.apache.hudi.async;
-import org.apache.hudi.client.AbstractCompactor;
-import org.apache.hudi.client.AbstractHoodieWriteClient;
+import org.apache.hudi.client.BaseCompactor;
+import org.apache.hudi.client.BaseHoodieWriteClient;
import org.apache.hudi.common.engine.EngineProperty;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.table.timeline.HoodieInstant;
@@ -37,7 +37,7 @@
/**
* Async Compactor Service that runs in separate thread. Currently, only one compactor is allowed to run at any time.
*/
-public abstract class AsyncCompactService extends HoodieAsyncService {
+public abstract class AsyncCompactService extends HoodieAsyncTableService {
private static final long serialVersionUID = 1L;
private static final Logger LOG = LogManager.getLogger(AsyncCompactService.class);
@@ -48,21 +48,21 @@ public abstract class AsyncCompactService extends HoodieAsyncService {
public static final String COMPACT_POOL_NAME = "hoodiecompact";
private final int maxConcurrentCompaction;
- private transient AbstractCompactor compactor;
+ private transient BaseCompactor compactor;
protected transient HoodieEngineContext context;
- public AsyncCompactService(HoodieEngineContext context, AbstractHoodieWriteClient client) {
+ public AsyncCompactService(HoodieEngineContext context, BaseHoodieWriteClient client) {
this(context, client, false);
}
- public AsyncCompactService(HoodieEngineContext context, AbstractHoodieWriteClient client, boolean runInDaemonMode) {
- super(runInDaemonMode);
+ public AsyncCompactService(HoodieEngineContext context, BaseHoodieWriteClient client, boolean runInDaemonMode) {
+ super(client.getConfig(), runInDaemonMode);
this.context = context;
this.compactor = createCompactor(client);
this.maxConcurrentCompaction = 1;
}
- protected abstract AbstractCompactor createCompactor(AbstractHoodieWriteClient client);
+ protected abstract BaseCompactor createCompactor(BaseHoodieWriteClient client);
/**
* Start Compaction Service.
@@ -92,10 +92,16 @@ protected Pair startService() {
}
LOG.info("Compactor shutting down properly!!");
} catch (InterruptedException ie) {
+ hasError = true;
LOG.warn("Compactor executor thread got interrupted exception. Stopping", ie);
} catch (IOException e) {
- LOG.error("Compactor executor failed", e);
+ hasError = true;
+ LOG.error("Compactor executor failed due to IOException", e);
throw new HoodieIOException(e.getMessage(), e);
+ } catch (Exception e) {
+ hasError = true;
+ LOG.error("Compactor executor failed", e);
+ throw e;
}
return true;
}, executor)).toArray(CompletableFuture[]::new)), executor);
@@ -110,7 +116,7 @@ protected boolean shouldStopCompactor() {
return false;
}
- public synchronized void updateWriteClient(AbstractHoodieWriteClient writeClient) {
+ public synchronized void updateWriteClient(BaseHoodieWriteClient writeClient) {
this.compactor.updateWriteClient(writeClient);
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/HoodieAsyncService.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/HoodieAsyncService.java
index f57484d886c9b..1ce6dfb288d62 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/HoodieAsyncService.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/HoodieAsyncService.java
@@ -36,12 +36,15 @@
import java.util.function.Function;
/**
- * Base Class for running clean/delta-sync/compaction/clustering in separate thread and controlling their life-cycle.
+ * Base Class for running archive/clean/delta-sync/compaction/clustering in separate thread and controlling their life-cycles.
*/
public abstract class HoodieAsyncService implements Serializable {
private static final Logger LOG = LogManager.getLogger(HoodieAsyncService.class);
+ private static final long POLLING_SECONDS = 10;
+ // Flag indicating whether an error is incurred in the service
+ protected boolean hasError;
// Flag to track if the service is started.
private boolean started;
// Flag indicating shutdown is externally requested
@@ -70,21 +73,32 @@ protected HoodieAsyncService(boolean runInDaemonMode) {
this.runInDaemonMode = runInDaemonMode;
}
- protected boolean isShutdownRequested() {
+ public boolean isStarted() {
+ return started;
+ }
+
+ public boolean isShutdownRequested() {
return shutdownRequested;
}
- protected boolean isShutdown() {
+ public boolean isShutdown() {
return shutdown;
}
+ public boolean hasError() {
+ return hasError;
+ }
+
/**
* Wait till the service shutdown. If the service shutdown with exception, it will be thrown
- *
+ *
* @throws ExecutionException
* @throws InterruptedException
*/
public void waitForShutdown() throws ExecutionException, InterruptedException {
+ if (future == null) {
+ return;
+ }
try {
future.get();
} catch (ExecutionException ex) {
@@ -102,6 +116,7 @@ public void waitForShutdown() throws ExecutionException, InterruptedException {
public void shutdown(boolean force) {
if (!shutdownRequested || force) {
shutdownRequested = true;
+ shutdown = true;
if (executor != null) {
if (force) {
executor.shutdownNow();
@@ -125,6 +140,10 @@ public void shutdown(boolean force) {
* @param onShutdownCallback
*/
public void start(Function onShutdownCallback) {
+ if (started) {
+ LOG.warn("The async service already started.");
+ return;
+ }
Pair res = startService();
future = res.getKey();
executor = res.getValue();
@@ -134,8 +153,6 @@ public void start(Function onShutdownCallback) {
/**
* Service implementation.
- *
- * @return
*/
protected abstract Pair startService();
@@ -146,6 +163,9 @@ public void start(Function onShutdownCallback) {
*/
@SuppressWarnings("unchecked")
private void shutdownCallback(Function callback) {
+ if (future == null) {
+ return;
+ }
future.whenComplete((resp, error) -> {
if (null != callback) {
callback.apply(null != error);
@@ -166,8 +186,8 @@ public boolean isRunInDaemonMode() {
public void waitTillPendingAsyncServiceInstantsReducesTo(int numPending) throws InterruptedException {
try {
queueLock.lock();
- while (!isShutdown() && (pendingInstants.size() > numPending)) {
- consumed.await();
+ while (!isShutdown() && !hasError() && (pendingInstants.size() > numPending)) {
+ consumed.await(POLLING_SECONDS, TimeUnit.SECONDS);
}
} finally {
queueLock.unlock();
@@ -190,8 +210,8 @@ public void enqueuePendingAsyncServiceInstant(HoodieInstant instant) {
* @throws InterruptedException
*/
HoodieInstant fetchNextAsyncServiceInstant() throws InterruptedException {
- LOG.info("Waiting for next instant upto 10 seconds");
- HoodieInstant instant = pendingInstants.poll(10, TimeUnit.SECONDS);
+ LOG.info(String.format("Waiting for next instant up to %d seconds", POLLING_SECONDS));
+ HoodieInstant instant = pendingInstants.poll(POLLING_SECONDS, TimeUnit.SECONDS);
if (instant != null) {
try {
queueLock.lock();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/HoodieAsyncTableService.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/HoodieAsyncTableService.java
new file mode 100644
index 0000000000000..6a53d30063c1d
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/HoodieAsyncTableService.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.async;
+
+import org.apache.hudi.client.RunsTableService;
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import java.util.function.Function;
+
+public abstract class HoodieAsyncTableService extends HoodieAsyncService implements RunsTableService {
+
+ protected HoodieWriteConfig writeConfig;
+
+ protected HoodieAsyncTableService() {
+ }
+
+ protected HoodieAsyncTableService(HoodieWriteConfig writeConfig) {
+ this.writeConfig = writeConfig;
+ }
+
+ protected HoodieAsyncTableService(HoodieWriteConfig writeConfig, boolean runInDaemonMode) {
+ super(runInDaemonMode);
+ this.writeConfig = writeConfig;
+ }
+
+ @Override
+ public void start(Function onShutdownCallback) {
+ if (!tableServicesEnabled(writeConfig)) {
+ return;
+ }
+ super.start(onShutdownCallback);
+ }
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractClusteringClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseClusterer.java
similarity index 80%
rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractClusteringClient.java
rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseClusterer.java
index 34234f546ed19..648ce805b0825 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractClusteringClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseClusterer.java
@@ -28,13 +28,13 @@
/**
* Client will run one round of clustering.
*/
-public abstract class AbstractClusteringClient implements Serializable {
+public abstract class BaseClusterer implements Serializable {
private static final long serialVersionUID = 1L;
- protected transient AbstractHoodieWriteClient clusteringClient;
+ protected transient BaseHoodieWriteClient clusteringClient;
- public AbstractClusteringClient(AbstractHoodieWriteClient clusteringClient) {
+ public BaseClusterer(BaseHoodieWriteClient clusteringClient) {
this.clusteringClient = clusteringClient;
}
@@ -49,7 +49,7 @@ public AbstractClusteringClient(AbstractHoodieWriteClient clustering
* Update the write client used by async clustering.
* @param writeClient
*/
- public void updateWriteClient(AbstractHoodieWriteClient writeClient) {
+ public void updateWriteClient(BaseHoodieWriteClient writeClient) {
this.clusteringClient = writeClient;
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractCompactor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseCompactor.java
similarity index 78%
rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractCompactor.java
rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseCompactor.java
index c80b34a3ef656..88737dbcf1d7e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractCompactor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseCompactor.java
@@ -27,19 +27,19 @@
/**
* Run one round of compaction.
*/
-public abstract class AbstractCompactor implements Serializable {
+public abstract class BaseCompactor implements Serializable {
private static final long serialVersionUID = 1L;
- protected transient AbstractHoodieWriteClient compactionClient;
+ protected transient BaseHoodieWriteClient compactionClient;
- public AbstractCompactor(AbstractHoodieWriteClient compactionClient) {
+ public BaseCompactor(BaseHoodieWriteClient compactionClient) {
this.compactionClient = compactionClient;
}
public abstract void compact(HoodieInstant instant) throws IOException;
- public void updateWriteClient(AbstractHoodieWriteClient writeClient) {
+ public void updateWriteClient(BaseHoodieWriteClient writeClient) {
this.compactionClient = writeClient;
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java
similarity index 91%
rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java
rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java
index 350fe0c9bf7e0..3f208a0f86a09 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java
@@ -40,9 +40,9 @@
* Abstract class taking care of holding common member variables (FileSystem, SparkContext, HoodieConfigs) Also, manages
* embedded timeline-server if enabled.
*/
-public abstract class AbstractHoodieClient implements Serializable, AutoCloseable {
+public abstract class BaseHoodieClient implements Serializable, AutoCloseable {
- private static final Logger LOG = LogManager.getLogger(AbstractHoodieClient.class);
+ private static final Logger LOG = LogManager.getLogger(BaseHoodieClient.class);
protected final transient FileSystem fs;
protected final transient HoodieEngineContext context;
@@ -59,11 +59,11 @@ public abstract class AbstractHoodieClient implements Serializable, AutoCloseabl
private transient Option timelineServer;
private final boolean shouldStopTimelineServer;
- protected AbstractHoodieClient(HoodieEngineContext context, HoodieWriteConfig clientConfig) {
+ protected BaseHoodieClient(HoodieEngineContext context, HoodieWriteConfig clientConfig) {
this(context, clientConfig, Option.empty());
}
- protected AbstractHoodieClient(HoodieEngineContext context, HoodieWriteConfig clientConfig,
+ protected BaseHoodieClient(HoodieEngineContext context, HoodieWriteConfig clientConfig,
Option timelineServer) {
this.hadoopConf = context.getHadoopConf().get();
this.fs = FSUtils.getFs(clientConfig.getBasePath(), hadoopConf);
@@ -134,7 +134,8 @@ protected void initWrapperFSMetrics() {
protected HoodieTableMetaClient createMetaClient(boolean loadActiveTimelineOnLoad) {
return HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(config.getBasePath())
.setLoadActiveTimelineOnLoad(loadActiveTimelineOnLoad).setConsistencyGuardConfig(config.getConsistencyGuardConfig())
- .setLayoutVersion(Option.of(new TimelineLayoutVersion(config.getTimelineLayoutVersion()))).build();
+ .setLayoutVersion(Option.of(new TimelineLayoutVersion(config.getTimelineLayoutVersion())))
+ .setFileSystemRetryConfig(config.getFileSystemRetryConfig()).build();
}
public Option getTimelineServer() {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
similarity index 87%
rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieWriteClient.java
rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
index c9162de9e9ca1..7b67ff54a2aa5 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieWriteClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
@@ -18,11 +18,14 @@
package org.apache.hudi.client;
+import org.apache.hudi.async.AsyncArchiveService;
+import org.apache.hudi.async.AsyncCleanerService;
import org.apache.hudi.avro.model.HoodieCleanMetadata;
import org.apache.hudi.avro.model.HoodieCleanerPlan;
import org.apache.hudi.avro.model.HoodieClusteringPlan;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
import org.apache.hudi.avro.model.HoodieRestoreMetadata;
+import org.apache.hudi.avro.model.HoodieRestorePlan;
import org.apache.hudi.avro.model.HoodieRollbackMetadata;
import org.apache.hudi.avro.model.HoodieRollbackPlan;
import org.apache.hudi.callback.HoodieWriteCommitCallback;
@@ -66,7 +69,6 @@
import org.apache.hudi.metrics.HoodieMetrics;
import org.apache.hudi.table.BulkInsertPartitioner;
import org.apache.hudi.table.HoodieTable;
-import org.apache.hudi.table.HoodieTimelineArchiveLog;
import org.apache.hudi.table.action.HoodieWriteMetadata;
import org.apache.hudi.table.action.rollback.RollbackUtils;
import org.apache.hudi.table.action.savepoint.SavepointHelpers;
@@ -98,14 +100,15 @@
* @param Type of keys
* @param Type of outputs
*/
-public abstract class AbstractHoodieWriteClient extends AbstractHoodieClient {
+public abstract class BaseHoodieWriteClient extends BaseHoodieClient
+ implements RunsTableService {
protected static final String LOOKUP_STR = "lookup";
private static final long serialVersionUID = 1L;
- private static final Logger LOG = LogManager.getLogger(AbstractHoodieWriteClient.class);
+ private static final Logger LOG = LogManager.getLogger(BaseHoodieWriteClient.class);
protected final transient HoodieMetrics metrics;
- private final transient HoodieIndex index;
+ private final transient HoodieIndex index;
protected transient Timer.Context writeTimer = null;
protected transient Timer.Context compactionTimer;
@@ -114,6 +117,7 @@ public abstract class AbstractHoodieWriteClient>> lastCompletedTxnAndMetadata = Option.empty();
@@ -123,7 +127,7 @@ public abstract class AbstractHoodieWriteClient timelineService) {
super(context, writeConfig, timelineService);
this.metrics = new HoodieMetrics(config);
@@ -142,7 +146,7 @@ public AbstractHoodieWriteClient(HoodieEngineContext context, HoodieWriteConfig
this.txnManager = new TransactionManager(config, fs);
}
- protected abstract HoodieIndex createIndex(HoodieWriteConfig writeConfig);
+ protected abstract HoodieIndex createIndex(HoodieWriteConfig writeConfig);
public void setOperationType(WriteOperationType operationType) {
this.operationType = operationType;
@@ -359,7 +363,7 @@ public void rollbackFailedBootstrap() {
* table for the very first time (e.g: converting an existing table to Hoodie).
*
* This implementation uses sortBy (which does range partitioning based on reservoir sampling) and attempts to control
- * the numbers of files with less memory compared to the {@link AbstractHoodieWriteClient#insert(I, String)}
+ * the numbers of files with less memory compared to the {@link BaseHoodieWriteClient#insert(I, String)}
*
* @param records HoodieRecords to insert
* @param instantTime Instant time of the commit
@@ -372,7 +376,7 @@ public void rollbackFailedBootstrap() {
* table for the very first time (e.g: converting an existing table to Hoodie).
*
* This implementation uses sortBy (which does range partitioning based on reservoir sampling) and attempts to control
- * the numbers of files with less memory compared to the {@link AbstractHoodieWriteClient#insert(I, String)}. Optionally
+ * the numbers of files with less memory compared to the {@link BaseHoodieWriteClient#insert(I, String)}. Optionally
* it allows users to specify their own partitioner. If specified then it will be used for repartitioning records. See
* {@link BulkInsertPartitioner}.
*
@@ -392,7 +396,7 @@ public abstract O bulkInsert(I records, final String instantTime,
* duplicates if needed.
*
* This implementation uses sortBy (which does range partitioning based on reservoir sampling) and attempts to control
- * the numbers of files with less memory compared to the {@link AbstractHoodieWriteClient#insert(I, String)}. Optionally
+ * the numbers of files with less memory compared to the {@link BaseHoodieWriteClient#insert(I, String)}. Optionally
* it allows users to specify their own partitioner. If specified then it will be used for repartitioning records. See
* {@link BulkInsertPartitioner}.
*
@@ -430,6 +434,11 @@ protected void preWrite(String instantTime, WriteOperationType writeOperationTyp
} else {
this.asyncCleanerService.start(null);
}
+ if (null == this.asyncArchiveService) {
+ this.asyncArchiveService = AsyncArchiveService.startAsyncArchiveIfEnabled(this);
+ } else {
+ this.asyncArchiveService.start(null);
+ }
}
/**
@@ -455,16 +464,17 @@ protected void postCommit(HoodieTable table, HoodieCommitMetadata me
WriteMarkersFactory.get(config.getMarkersType(), table, instantTime)
.quietDeleteMarkerDir(context, config.getMarkersDeleteParallelism());
autoCleanOnCommit();
- if (config.isAutoArchive()) {
- archive(table);
- }
+ autoArchiveOnCommit(table);
} finally {
this.heartbeatClient.stop(instantTime);
}
}
protected void runTableServicesInline(HoodieTable table, HoodieCommitMetadata metadata, Option> extraMetadata) {
- if (config.areAnyTableServicesInline()) {
+ if (!tableServicesEnabled(config)) {
+ return;
+ }
+ if (config.areAnyTableServicesExecutedInline() || config.areAnyTableServicesScheduledInline()) {
if (config.isMetadataTableEnabled()) {
table.getHoodieView().sync();
}
@@ -472,19 +482,35 @@ protected void runTableServicesInline(HoodieTable table, HoodieCommi
if (config.inlineCompactionEnabled()) {
runAnyPendingCompactions(table);
metadata.addMetadata(HoodieCompactionConfig.INLINE_COMPACT.key(), "true");
- inlineCompact(extraMetadata);
+ inlineCompaction(extraMetadata);
} else {
metadata.addMetadata(HoodieCompactionConfig.INLINE_COMPACT.key(), "false");
}
+ // if just inline schedule is enabled
+ if (!config.inlineCompactionEnabled() && config.scheduleInlineCompaction()
+ && !table.getActiveTimeline().getWriteTimeline().filterPendingCompactionTimeline().getInstants().findAny().isPresent()) {
+ // proceed only if there are no pending compactions
+ metadata.addMetadata(HoodieCompactionConfig.SCHEDULE_INLINE_COMPACT.key(), "true");
+ inlineScheduleCompaction(extraMetadata);
+ }
+
// Do an inline clustering if enabled
if (config.inlineClusteringEnabled()) {
runAnyPendingClustering(table);
metadata.addMetadata(HoodieClusteringConfig.INLINE_CLUSTERING.key(), "true");
- inlineCluster(extraMetadata);
+ inlineClustering(extraMetadata);
} else {
metadata.addMetadata(HoodieClusteringConfig.INLINE_CLUSTERING.key(), "false");
}
+
+ // if just inline schedule is enabled
+ if (!config.inlineClusteringEnabled() && config.scheduleInlineClustering()
+ && !table.getActiveTimeline().filterPendingReplaceTimeline().getInstants().findAny().isPresent()) {
+ // proceed only if there are no pending clustering
+ metadata.addMetadata(HoodieClusteringConfig.SCHEDULE_INLINE_CLUSTERING.key(), "true");
+ inlineScheduleClustering(extraMetadata);
+ }
}
}
@@ -506,22 +532,34 @@ protected void runAnyPendingClustering(HoodieTable table) {
});
}
- /**
- * Handle auto clean during commit.
- *
- */
protected void autoCleanOnCommit() {
- if (config.isAutoClean()) {
- // Call clean to cleanup if there is anything to cleanup after the commit,
- if (config.isAsyncClean()) {
- LOG.info("Cleaner has been spawned already. Waiting for it to finish");
- AsyncCleanerService.waitForCompletion(asyncCleanerService);
- LOG.info("Cleaner has finished");
- } else {
- // Do not reuse instantTime for clean as metadata table requires all changes to have unique instant timestamps.
- LOG.info("Auto cleaning is enabled. Running cleaner now");
- clean(true);
- }
+ if (!config.isAutoClean()) {
+ return;
+ }
+
+ if (config.isAsyncClean()) {
+ LOG.info("Async cleaner has been spawned. Waiting for it to finish");
+ AsyncCleanerService.waitForCompletion(asyncCleanerService);
+ LOG.info("Async cleaner has finished");
+ } else {
+ LOG.info("Start to clean synchronously.");
+ // Do not reuse instantTime for clean as metadata table requires all changes to have unique instant timestamps.
+ clean(true);
+ }
+ }
+
+ protected void autoArchiveOnCommit(HoodieTable table) {
+ if (!config.isAutoArchive()) {
+ return;
+ }
+
+ if (config.isAsyncArchive()) {
+ LOG.info("Async archiver has been spawned. Waiting for it to finish");
+ AsyncArchiveService.waitForCompletion(asyncArchiveService);
+ LOG.info("Async archiver has finished");
+ } else {
+ LOG.info("Start to archive synchronously.");
+ archive(table);
}
}
@@ -606,7 +644,7 @@ public boolean rollback(final String commitInstantTime) throws HoodieRollbackExc
/**
* @Deprecated
* Rollback the inflight record changes with the given commit time. This
- * will be removed in future in favor of {@link AbstractHoodieWriteClient#restoreToInstant(String)}
+ * will be removed in future in favor of {@link BaseHoodieWriteClient#restoreToInstant(String)}
* Adding this api for backwards compatability.
* @param commitInstantTime Instant time of the commit
* @param skipLocking if this is triggered by another parent transaction, locking can be skipped.
@@ -620,7 +658,7 @@ public boolean rollback(final String commitInstantTime, boolean skipLocking) thr
/**
* @Deprecated
* Rollback the inflight record changes with the given commit time. This
- * will be removed in future in favor of {@link AbstractHoodieWriteClient#restoreToInstant(String)}
+ * will be removed in future in favor of {@link BaseHoodieWriteClient#restoreToInstant(String)}
*
* @param commitInstantTime Instant time of the commit
* @param pendingRollbackInfo pending rollback instant and plan if rollback failed from previous attempt.
@@ -639,8 +677,8 @@ public boolean rollback(final String commitInstantTime, Option rollbackPlanOption = pendingRollbackInfo.map(entry -> Option.of(entry.getRollbackPlan())).orElse(table.scheduleRollback(context, rollbackInstantTime,
- commitInstantOpt.get(), false, config.shouldRollbackUsingMarkers()));
+ Option rollbackPlanOption = pendingRollbackInfo.map(entry -> Option.of(entry.getRollbackPlan()))
+ .orElseGet(() -> table.scheduleRollback(context, rollbackInstantTime, commitInstantOpt.get(), false, config.shouldRollbackUsingMarkers()));
if (rollbackPlanOption.isPresent()) {
// execute rollback
HoodieRollbackMetadata rollbackMetadata = table.rollback(context, rollbackInstantTime, commitInstantOpt.get(), true,
@@ -674,16 +712,21 @@ public HoodieRestoreMetadata restoreToInstant(final String instantTime) throws H
Timer.Context timerContext = metrics.getRollbackCtx();
try {
HoodieTable table = createTable(config, hadoopConf, config.isMetadataTableEnabled());
- HoodieRestoreMetadata restoreMetadata = table.restore(context, restoreInstantTime, instantTime);
- if (timerContext != null) {
- final long durationInMs = metrics.getDurationInMs(timerContext.stop());
- final long totalFilesDeleted = restoreMetadata.getHoodieRestoreMetadata().values().stream()
- .flatMap(Collection::stream)
- .mapToLong(HoodieRollbackMetadata::getTotalFilesDeleted)
- .sum();
- metrics.updateRollbackMetrics(durationInMs, totalFilesDeleted);
+ Option restorePlanOption = table.scheduleRestore(context, restoreInstantTime, instantTime);
+ if (restorePlanOption.isPresent()) {
+ HoodieRestoreMetadata restoreMetadata = table.restore(context, restoreInstantTime, instantTime);
+ if (timerContext != null) {
+ final long durationInMs = metrics.getDurationInMs(timerContext.stop());
+ final long totalFilesDeleted = restoreMetadata.getHoodieRestoreMetadata().values().stream()
+ .flatMap(Collection::stream)
+ .mapToLong(HoodieRollbackMetadata::getTotalFilesDeleted)
+ .sum();
+ metrics.updateRollbackMetrics(durationInMs, totalFilesDeleted);
+ }
+ return restoreMetadata;
+ } else {
+ throw new HoodieRestoreException("Failed to restore " + config.getBasePath() + " to commit " + instantTime);
}
- return restoreMetadata;
} catch (Exception e) {
throw new HoodieRestoreException("Failed to restore to " + instantTime, e);
}
@@ -714,28 +757,38 @@ public HoodieCleanMetadata clean(String cleanInstantTime, boolean skipLocking) t
* Clean up any stale/old files/data lying around (either on file storage or index storage) based on the
* configurations and CleaningPolicy used. (typically files that no longer can be used by a running query can be
* cleaned). This API provides the flexibility to schedule clean instant asynchronously via
- * {@link AbstractHoodieWriteClient#scheduleTableService(String, Option, TableServiceType)} and disable inline scheduling
+ * {@link BaseHoodieWriteClient#scheduleTableService(String, Option, TableServiceType)} and disable inline scheduling
* of clean.
* @param cleanInstantTime instant time for clean.
* @param scheduleInline true if needs to be scheduled inline. false otherwise.
* @param skipLocking if this is triggered by another parent transaction, locking can be skipped.
*/
public HoodieCleanMetadata clean(String cleanInstantTime, boolean scheduleInline, boolean skipLocking) throws HoodieIOException {
- if (scheduleInline) {
- scheduleTableServiceInternal(cleanInstantTime, Option.empty(), TableServiceType.CLEAN);
+ if (!tableServicesEnabled(config)) {
+ return null;
}
- LOG.info("Cleaner started");
final Timer.Context timerContext = metrics.getCleanCtx();
- LOG.info("Cleaned failed attempts if any");
CleanerUtils.rollbackFailedWrites(config.getFailedWritesCleanPolicy(),
HoodieTimeline.CLEAN_ACTION, () -> rollbackFailedWrites(skipLocking));
- HoodieCleanMetadata metadata = createTable(config, hadoopConf).clean(context, cleanInstantTime, skipLocking);
- if (timerContext != null && metadata != null) {
- long durationMs = metrics.getDurationInMs(timerContext.stop());
- metrics.updateCleanMetrics(durationMs, metadata.getTotalFilesDeleted());
- LOG.info("Cleaned " + metadata.getTotalFilesDeleted() + " files"
- + " Earliest Retained Instant :" + metadata.getEarliestCommitToRetain()
- + " cleanerElapsedMs" + durationMs);
+
+ HoodieCleanMetadata metadata = null;
+ HoodieTable table = createTable(config, hadoopConf);
+ if (config.allowMultipleCleans() || !table.getActiveTimeline().getCleanerTimeline().filterInflightsAndRequested().firstInstant().isPresent()) {
+ LOG.info("Cleaner started");
+ // proceed only if multiple clean schedules are enabled or if there are no pending cleans.
+ if (scheduleInline) {
+ scheduleTableServiceInternal(cleanInstantTime, Option.empty(), TableServiceType.CLEAN);
+ table.getMetaClient().reloadActiveTimeline();
+ }
+
+ metadata = table.clean(context, cleanInstantTime, skipLocking);
+ if (timerContext != null && metadata != null) {
+ long durationMs = metrics.getDurationInMs(timerContext.stop());
+ metrics.updateCleanMetrics(durationMs, metadata.getTotalFilesDeleted());
+ LOG.info("Cleaned " + metadata.getTotalFilesDeleted() + " files"
+ + " Earliest Retained Instant :" + metadata.getEarliestCommitToRetain()
+ + " cleanerElapsedMs" + durationMs);
+ }
}
return metadata;
}
@@ -760,10 +813,13 @@ public HoodieCleanMetadata clean(boolean skipLocking) {
* @param table table to commit on.
*/
protected void archive(HoodieTable table) {
+ if (!tableServicesEnabled(config)) {
+ return;
+ }
try {
// We cannot have unbounded commit files. Archive commits if we have to archive
- HoodieTimelineArchiveLog archiveLog = new HoodieTimelineArchiveLog(config, table);
- archiveLog.archiveIfRequired(context);
+ HoodieTimelineArchiver archiver = new HoodieTimelineArchiver(config, table);
+ archiver.archiveIfRequired(context);
} catch (IOException ioe) {
throw new HoodieIOException("Failed to archive", ioe);
}
@@ -861,7 +917,7 @@ public boolean scheduleCompactionAtInstant(String instantTime, Option compact(String compactionInstantTime) {
return compact(compactionInstantTime, config.shouldAutoCommit());
}
@@ -869,17 +925,16 @@ public O compact(String compactionInstantTime) {
* Commit a compaction operation. Allow passing additional meta-data to be stored in commit instant file.
*
* @param compactionInstantTime Compaction Instant Time
- * @param writeStatuses Collection of WriteStatus to inspect errors and counts
+ * @param metadata All the metadata that gets stored along with a commit
* @param extraMetadata Extra Metadata to be stored
*/
- public abstract void commitCompaction(String compactionInstantTime, O writeStatuses,
- Option> extraMetadata) throws IOException;
+ public abstract void commitCompaction(String compactionInstantTime, HoodieCommitMetadata metadata,
+ Option> extraMetadata);
/**
* Commit Compaction and track metrics.
*/
- protected abstract void completeCompaction(HoodieCommitMetadata metadata, O writeStatuses,
- HoodieTable table, String compactionCommitTime);
+ protected abstract void completeCompaction(HoodieCommitMetadata metadata, HoodieTable table, String compactionCommitTime);
/**
* Get inflight time line exclude compaction and clustering.
@@ -1001,13 +1056,14 @@ protected List getInstantsToRollback(HoodieTableMetaClient metaClient, H
* @param compactionInstantTime Compaction Instant Time
* @return Collection of Write Status
*/
- protected abstract O compact(String compactionInstantTime, boolean shouldComplete);
+ protected abstract HoodieWriteMetadata compact(String compactionInstantTime, boolean shouldComplete);
/**
* Performs a compaction operation on a table, serially before or after an insert/upsert action.
+ * Scheduling and execution is done inline.
*/
- protected Option inlineCompact(Option> extraMetadata) {
- Option compactionInstantTimeOpt = scheduleCompaction(extraMetadata);
+ protected Option inlineCompaction(Option> extraMetadata) {
+ Option compactionInstantTimeOpt = inlineScheduleCompaction(extraMetadata);
compactionInstantTimeOpt.ifPresent(compactInstantTime -> {
// inline compaction should auto commit as the user is never given control
compact(compactInstantTime, true);
@@ -1015,6 +1071,15 @@ protected Option inlineCompact(Option> extraMetadata
return compactionInstantTimeOpt;
}
+ /***
+ * Schedules compaction inline.
+ * @param extraMetadata extrametada to be used.
+ * @return compaction instant if scheduled.
+ */
+ protected Option inlineScheduleCompaction(Option> extraMetadata) {
+ return scheduleCompaction(extraMetadata);
+ }
+
/**
* Schedules a new clustering instant.
* @param extraMetadata Extra Metadata to be stored
@@ -1093,7 +1158,13 @@ public Option scheduleTableService(String instantTime, Option scheduleTableServiceInternal(String instantTime, Option> extraMetadata,
TableServiceType tableServiceType) {
+ if (!tableServicesEnabled(config)) {
+ return Option.empty();
+ }
switch (tableServiceType) {
+ case ARCHIVE:
+ LOG.info("Scheduling archiving is not supported. Skipping.");
+ return Option.empty();
case CLUSTER:
LOG.info("Scheduling clustering at instant time :" + instantTime);
Option clusteringPlan = createTable(config, hadoopConf, config.isMetadataTableEnabled())
@@ -1116,9 +1187,10 @@ private Option scheduleTableServiceInternal(String instantTime, Option inlineCluster(Option> extraMetadata) {
- Option clusteringInstantOpt = scheduleClustering(extraMetadata);
+ protected Option inlineClustering(Option> extraMetadata) {
+ Option clusteringInstantOpt = inlineScheduleClustering(extraMetadata);
clusteringInstantOpt.ifPresent(clusteringInstant -> {
// inline cluster should auto commit as the user is never given control
cluster(clusteringInstant, true);
@@ -1126,6 +1198,15 @@ protected Option inlineCluster(Option> extraMetadata
return clusteringInstantOpt;
}
+ /**
+ * Schedules clustering inline.
+ * @param extraMetadata extrametadata to use.
+ * @return clustering instant if scheduled.
+ */
+ protected Option inlineScheduleClustering(Option> extraMetadata) {
+ return scheduleClustering(extraMetadata);
+ }
+
protected void rollbackInflightClustering(HoodieInstant inflightInstant, HoodieTable table) {
String commitTime = HoodieActiveTimeline.createNewInstantTime();
table.scheduleRollback(context, commitTime, inflightInstant, false, config.shouldRollbackUsingMarkers());
@@ -1160,7 +1241,7 @@ public HoodieMetrics getMetrics() {
return metrics;
}
- public HoodieIndex getIndex() {
+ public HoodieIndex getIndex() {
return index;
}
@@ -1208,7 +1289,8 @@ protected void releaseResources() {
@Override
public void close() {
- // release AsyncCleanerService
+ AsyncArchiveService.forceShutdown(asyncArchiveService);
+ asyncArchiveService = null;
AsyncCleanerService.forceShutdown(asyncCleanerService);
asyncCleanerService = null;
// Stop timeline-server if running
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java
index 1c869e46f1cbf..40e8f85a3ac70 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/CompactionAdminClient.java
@@ -61,7 +61,7 @@
/**
* Client to perform admin operations related to compaction.
*/
-public class CompactionAdminClient extends AbstractHoodieClient {
+public class CompactionAdminClient extends BaseHoodieClient {
private static final Logger LOG = LogManager.getLogger(CompactionAdminClient.class);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTimelineArchiveLog.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java
similarity index 97%
rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTimelineArchiveLog.java
rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java
index 138e40a90c6e0..15401c0292e14 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTimelineArchiveLog.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java
@@ -7,19 +7,18 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
-package org.apache.hudi.table;
+package org.apache.hudi.client;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hudi.avro.model.HoodieArchivedMetaEntry;
import org.apache.hudi.avro.model.HoodieMergeArchiveFilePlan;
import org.apache.hudi.client.utils.MetadataConversionUtils;
@@ -52,11 +51,14 @@
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.marker.WriteMarkers;
import org.apache.hudi.table.marker.WriteMarkersFactory;
import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -79,9 +81,9 @@
/**
* Archiver to bound the growth of files under .hoodie meta path.
*/
-public class HoodieTimelineArchiveLog {
+public class HoodieTimelineArchiver {
- private static final Logger LOG = LogManager.getLogger(HoodieTimelineArchiveLog.class);
+ private static final Logger LOG = LogManager.getLogger(HoodieTimelineArchiver.class);
private final Path archiveFilePath;
private final HoodieWriteConfig config;
@@ -91,7 +93,7 @@ public class HoodieTimelineArchiveLog {
private final HoodieTable table;
private final HoodieTableMetaClient metaClient;
- public HoodieTimelineArchiveLog(HoodieWriteConfig config, HoodieTable table) {
+ public HoodieTimelineArchiver(HoodieWriteConfig config, HoodieTable table) {
this.config = config;
this.table = table;
this.metaClient = table.getMetaClient();
@@ -319,8 +321,7 @@ public void mergeArchiveFiles(List compactCandidate) throws IOExcept
// Read the avro blocks
while (reader.hasNext()) {
HoodieAvroDataBlock blk = (HoodieAvroDataBlock) reader.next();
- List recordsPerFile = blk.getRecords();
- records.addAll(recordsPerFile);
+ blk.getRecordItr().forEachRemaining(records::add);
if (records.size() >= this.config.getCommitArchivalBatchSize()) {
writeToFile(wrapperSchema, records);
}
@@ -427,7 +428,7 @@ private Stream getInstantsToArchive() {
.collect(Collectors.groupingBy(i -> Pair.of(i.getTimestamp(),
HoodieInstant.getComparableAction(i.getAction()))));
- // If metadata table is enabled, do not archive instants which are more recent that the last compaction on the
+ // If metadata table is enabled, do not archive instants which are more recent than the last compaction on the
// metadata table.
if (config.isMetadataTableEnabled()) {
try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(table.getContext(), config.getMetadataConfig(),
@@ -445,7 +446,7 @@ private Stream getInstantsToArchive() {
throw new HoodieException("Error limiting instant archival based on metadata table", e);
}
}
-
+
return instants.flatMap(hoodieInstant ->
groupByTsAction.get(Pair.of(hoodieInstant.getTimestamp(),
HoodieInstant.getComparableAction(hoodieInstant.getAction()))).stream());
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/RunsTableService.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/RunsTableService.java
new file mode 100644
index 0000000000000..64e540568e8dc
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/RunsTableService.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.client;
+
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+public interface RunsTableService {
+
+ Logger LOG = LogManager.getLogger(RunsTableService.class);
+
+ default boolean tableServicesEnabled(HoodieWriteConfig config) {
+ boolean enabled = config.areTableServicesEnabled();
+ if (!enabled) {
+ LOG.warn(String.format("Table services are disabled. Set `%s` to enable.", HoodieWriteConfig.TABLE_SERVICES_ENABLED));
+ }
+ return enabled;
+ }
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
index 20f9b75a910a5..72f8e29c9fa8e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java
@@ -112,8 +112,12 @@ public FileSystemViewStorageConfig getRemoteFileSystemViewConfig() {
FileSystemViewStorageType viewStorageType = writeConfig.getClientSpecifiedViewStorageConfig()
.shouldEnableBackupForRemoteFileSystemView()
? FileSystemViewStorageType.REMOTE_FIRST : FileSystemViewStorageType.REMOTE_ONLY;
- return FileSystemViewStorageConfig.newBuilder().withStorageType(viewStorageType)
- .withRemoteServerHost(hostAddr).withRemoteServerPort(serverPort).build();
+ return FileSystemViewStorageConfig.newBuilder()
+ .withStorageType(viewStorageType)
+ .withRemoteServerHost(hostAddr)
+ .withRemoteServerPort(serverPort)
+ .withRemoteTimelineClientTimeoutSecs(writeConfig.getClientSpecifiedViewStorageConfig().getRemoteTimelineClientTimeoutSecs())
+ .build();
}
public FileSystemViewManager getViewManager() {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/InProcessLockProvider.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/InProcessLockProvider.java
index 426d1cfaf4020..9e5a2379c4c93 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/InProcessLockProvider.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/InProcessLockProvider.java
@@ -19,13 +19,14 @@
package org.apache.hudi.client.transaction.lock;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hudi.common.config.LockConfiguration;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.lock.LockProvider;
import org.apache.hudi.common.lock.LockState;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.exception.HoodieLockException;
+
+import org.apache.hadoop.conf.Configuration;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.jetbrains.annotations.NotNull;
@@ -92,7 +93,11 @@ public boolean tryLock(long time, @NotNull TimeUnit unit) {
public void unlock() {
LOG.info(getLogMessage(LockState.RELEASING));
try {
- LOCK.writeLock().unlock();
+ if (LOCK.isWriteLockedByCurrentThread()) {
+ LOCK.writeLock().unlock();
+ } else {
+ LOG.warn("Cannot unlock because the current thread does not hold the lock.");
+ }
} catch (Exception e) {
throw new HoodieLockException(getLogMessage(LockState.FAILED_TO_RELEASE), e);
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java
index 773685980af43..913736cad8a91 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockManager.java
@@ -83,6 +83,10 @@ public void lock() {
}
}
+ /**
+ * We need to take care of the scenarios that current thread may not be the holder of this lock
+ * and tries to call unlock()
+ */
public void unlock() {
if (writeConfig.getWriteConcurrencyMode().supportsOptimisticConcurrencyControl()) {
getLockProvider().unlock();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/TransactionUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/TransactionUtils.java
index ed2ea457764fb..9d7683128fc8c 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/TransactionUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/TransactionUtils.java
@@ -60,9 +60,31 @@ public static Option resolveWriteConflictIfAny(
final Option thisCommitMetadata,
final HoodieWriteConfig config,
Option lastCompletedTxnOwnerInstant) throws HoodieWriteConflictException {
+ return resolveWriteConflictIfAny(table, currentTxnOwnerInstant, thisCommitMetadata, config, lastCompletedTxnOwnerInstant, false);
+ }
+
+ /**
+ * Resolve any write conflicts when committing data.
+ *
+ * @param table
+ * @param currentTxnOwnerInstant
+ * @param thisCommitMetadata
+ * @param config
+ * @param lastCompletedTxnOwnerInstant
+ * @return
+ * @throws HoodieWriteConflictException
+ */
+ public static Option resolveWriteConflictIfAny(
+ final HoodieTable table,
+ final Option currentTxnOwnerInstant,
+ final Option thisCommitMetadata,
+ final HoodieWriteConfig config,
+ Option lastCompletedTxnOwnerInstant,
+ boolean reloadActiveTimeline) throws HoodieWriteConflictException {
if (config.getWriteConcurrencyMode().supportsOptimisticConcurrencyControl()) {
ConflictResolutionStrategy resolutionStrategy = config.getWriteConflictResolutionStrategy();
- Stream instantStream = resolutionStrategy.getCandidateInstants(table.getActiveTimeline(), currentTxnOwnerInstant.get(), lastCompletedTxnOwnerInstant);
+ Stream instantStream = resolutionStrategy.getCandidateInstants(reloadActiveTimeline
+ ? table.getMetaClient().reloadActiveTimeline() : table.getActiveTimeline(), currentTxnOwnerInstant.get(), lastCompletedTxnOwnerInstant);
final ConcurrentOperation thisOperation = new ConcurrentOperation(currentTxnOwnerInstant.get(), thisCommitMetadata.orElse(new HoodieCommitMetadata()));
instantStream.forEach(instant -> {
try {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieFileSliceReader.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/common/table/log/HoodieFileSliceReader.java
similarity index 85%
rename from hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieFileSliceReader.java
rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/common/table/log/HoodieFileSliceReader.java
index a786e8305bc27..a042255cdcb1a 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieFileSliceReader.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/common/table/log/HoodieFileSliceReader.java
@@ -7,13 +7,14 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
package org.apache.hudi.common.table.log;
@@ -23,6 +24,7 @@
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.SpillableMapUtils;
import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodiePayloadConfig;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.io.storage.HoodieFileReader;
@@ -53,10 +55,11 @@ public static HoodieFileSliceReader getFileSliceReader(
return new HoodieFileSliceReader(scanner.iterator());
} else {
Iterable> iterable = () -> scanner.iterator();
+ HoodiePayloadConfig payloadConfig = HoodiePayloadConfig.newBuilder().withPayloadOrderingField(preCombineField).build();
return new HoodieFileSliceReader(StreamSupport.stream(iterable.spliterator(), false)
.map(e -> {
try {
- GenericRecord record = (GenericRecord) e.getData().getInsertValue(schema).get();
+ GenericRecord record = (GenericRecord) e.getData().getInsertValue(schema, payloadConfig.getProps()).get();
return transform(record, scanner, payloadClass, preCombineField, simpleKeyGenFieldsOpt);
} catch (IOException io) {
throw new HoodieIOException("Error while creating reader for file slice with no base file.", io);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java
index 057b4a6f61299..41b1812c08151 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java
@@ -25,6 +25,7 @@
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.engine.EngineType;
import org.apache.hudi.common.util.TypeUtils;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieNotSupportedException;
import org.apache.hudi.table.action.cluster.ClusteringPlanPartitionFilterMode;
@@ -59,7 +60,7 @@ public class HoodieClusteringConfig extends HoodieConfig {
"hoodie.clustering.plan.partition.filter.mode";
// Any Space-filling curves optimize(z-order/hilbert) params can be saved with this prefix
- public static final String LAYOUT_OPTIMIZE_PARAM_PREFIX = "hoodie.layout.optimize.";
+ private static final String LAYOUT_OPTIMIZE_PARAM_PREFIX = "hoodie.layout.optimize.";
public static final ConfigProperty DAYBASED_LOOKBACK_PARTITIONS = ConfigProperty
.key(CLUSTERING_STRATEGY_PARAM_PREFIX + "daybased.lookback.partitions")
@@ -83,7 +84,7 @@ public class HoodieClusteringConfig extends HoodieConfig {
public static final ConfigProperty PLAN_STRATEGY_SMALL_FILE_LIMIT = ConfigProperty
.key(CLUSTERING_STRATEGY_PARAM_PREFIX + "small.file.limit")
- .defaultValue(String.valueOf(600 * 1024 * 1024L))
+ .defaultValue(String.valueOf(300 * 1024 * 1024L))
.sinceVersion("0.7.0")
.withDocumentation("Files smaller than the size specified here are candidates for clustering");
@@ -113,7 +114,8 @@ public class HoodieClusteringConfig extends HoodieConfig {
.key("hoodie.clustering.inline")
.defaultValue("false")
.sinceVersion("0.7.0")
- .withDocumentation("Turn on inline clustering - clustering will be run after each write operation is complete");
+ .withDocumentation("Turn on inline clustering - clustering will be run after each write operation is complete")
+ .withAlternatives("hoodie.datasource.clustering.inline.enable");
public static final ConfigProperty INLINE_CLUSTERING_MAX_COMMITS = ConfigProperty
.key("hoodie.clustering.inline.max.commits")
@@ -177,11 +179,22 @@ public class HoodieClusteringConfig extends HoodieConfig {
.withDocumentation("Determines how to handle updates, deletes to file groups that are under clustering."
+ " Default strategy just rejects the update");
+ public static final ConfigProperty SCHEDULE_INLINE_CLUSTERING = ConfigProperty
+ .key("hoodie.clustering.schedule.inline")
+ .defaultValue("false")
+ .withDocumentation("When set to true, clustering service will be attempted for inline scheduling after each write. Users have to ensure "
+ + "they have a separate job to run async clustering(execution) for the one scheduled by this writer. Users can choose to set both "
+ + "`hoodie.clustering.inline` and `hoodie.clustering.schedule.inline` to false and have both scheduling and execution triggered by any async process, on which "
+ + "case `hoodie.clustering.async.enabled` is expected to be set to true. But if `hoodie.clustering.inline` is set to false, and `hoodie.clustering.schedule.inline` "
+ + "is set to true, regular writers will schedule clustering inline, but users are expected to trigger async job for execution. If `hoodie.clustering.inline` is set "
+ + "to true, regular writers will do both scheduling and execution inline for clustering");
+
public static final ConfigProperty ASYNC_CLUSTERING_ENABLE = ConfigProperty
.key("hoodie.clustering.async.enabled")
.defaultValue("false")
.sinceVersion("0.7.0")
- .withDocumentation("Enable running of clustering service, asynchronously as inserts happen on the table.");
+ .withDocumentation("Enable running of clustering service, asynchronously as inserts happen on the table.")
+ .withAlternatives("hoodie.datasource.clustering.async.enable");
public static final ConfigProperty PRESERVE_COMMIT_METADATA = ConfigProperty
.key("hoodie.clustering.preserve.commit.metadata")
@@ -190,63 +203,88 @@ public class HoodieClusteringConfig extends HoodieConfig {
.withDocumentation("When rewriting data, preserves existing hoodie_commit_time");
/**
- * Using space-filling curves to optimize the layout of table to boost query performance.
- * The table data which sorted by space-filling curve has better aggregation;
- * combine with min-max filtering, it can achieve good performance improvement.
- *
- * Notice:
- * when we use this feature, we need specify the sort columns.
- * The more columns involved in sorting, the worse the aggregation, and the smaller the query performance improvement.
- * Choose the filter columns which commonly used in query sql as sort columns.
- * It is recommend that 2 ~ 4 columns participate in sorting.
+ * @deprecated this setting has no effect. Please refer to clustering configuration, as well as
+ * {@link #LAYOUT_OPTIMIZE_STRATEGY} config to enable advanced record layout optimization strategies
*/
public static final ConfigProperty LAYOUT_OPTIMIZE_ENABLE = ConfigProperty
.key(LAYOUT_OPTIMIZE_PARAM_PREFIX + "enable")
.defaultValue(false)
.sinceVersion("0.10.0")
- .withDocumentation("Enable use z-ordering/space-filling curves to optimize the layout of table to boost query performance. "
- + "This parameter takes precedence over clustering strategy set using " + EXECUTION_STRATEGY_CLASS_NAME.key());
+ .deprecatedAfter("0.11.0")
+ .withDocumentation("This setting has no effect. Please refer to clustering configuration, as well as "
+ + "LAYOUT_OPTIMIZE_STRATEGY config to enable advanced record layout optimization strategies");
- public static final ConfigProperty LAYOUT_OPTIMIZE_STRATEGY = ConfigProperty
+ /**
+ * Determines ordering strategy in for records layout optimization.
+ * Currently, following strategies are supported
+ *
+ * - Linear: simply orders records lexicographically
+ * - Z-order: orders records along Z-order spatial-curve
+ * - Hilbert: orders records along Hilbert's spatial-curve
+ *
+ *
+ * NOTE: "z-order", "hilbert" strategies may consume considerably more compute, than "linear".
+ * Make sure to perform small-scale local testing for your dataset before applying globally.
+ */
+ public static final ConfigProperty LAYOUT_OPTIMIZE_STRATEGY = ConfigProperty
.key(LAYOUT_OPTIMIZE_PARAM_PREFIX + "strategy")
- .defaultValue("z-order")
+ .defaultValue("linear")
.sinceVersion("0.10.0")
- .withDocumentation("Type of layout optimization to be applied, current only supports `z-order` and `hilbert` curves.");
+ .withDocumentation("Determines ordering strategy used in records layout optimization. "
+ + "Currently supported strategies are \"linear\", \"z-order\" and \"hilbert\" values are supported.");
/**
- * There exists two method to build z-curve.
- * one is directly mapping sort cols to z-value to build z-curve;
- * we can find this method in Amazon DynamoDB https://aws.amazon.com/cn/blogs/database/tag/z-order/
- * the other one is Boundary-based Interleaved Index method which we proposed. simply call it sample method.
- * Refer to rfc-28 for specific algorithm flow.
- * Boundary-based Interleaved Index method has better generalization, but the build speed is slower than direct method.
+ * NOTE: This setting only has effect if {@link #LAYOUT_OPTIMIZE_STRATEGY} value is set to
+ * either "z-order" or "hilbert" (ie leveraging space-filling curves)
+ *
+ * Currently, two methods to order records along the curve are supported "build" and "sample":
+ *
+ *
+ * - Direct: entails that spatial curve will be built in full, "filling in" all of the individual
+ * points corresponding to each individual record
+ * - Sample: leverages boundary-base interleaved index method (described in more details in
+ * Amazon DynamoDB blog [1])
+ *
+ *
+ * NOTE: Boundary-based interleaved Index method has better generalization,
+ * but is slower than direct method.
+ *
+ * Please refer to RFC-28 for specific elaboration on both flows.
+ *
+ * [1] https://aws.amazon.com/cn/blogs/database/tag/z-order/
*/
- public static final ConfigProperty LAYOUT_OPTIMIZE_CURVE_BUILD_METHOD = ConfigProperty
+ public static final ConfigProperty LAYOUT_OPTIMIZE_SPATIAL_CURVE_BUILD_METHOD = ConfigProperty
.key(LAYOUT_OPTIMIZE_PARAM_PREFIX + "curve.build.method")
.defaultValue("direct")
.sinceVersion("0.10.0")
- .withDocumentation("Controls how data is sampled to build the space filling curves. two methods: `direct`,`sample`."
- + "The direct method is faster than the sampling, however sample method would produce a better data layout.");
+ .withDocumentation("Controls how data is sampled to build the space-filling curves. "
+ + "Two methods: \"direct\", \"sample\". The direct method is faster than the sampling, "
+ + "however sample method would produce a better data layout.");
+
/**
- * Doing sample for table data is the first step in Boundary-based Interleaved Index method.
- * larger sample number means better optimize result, but more memory consumption
+ * NOTE: This setting only has effect if {@link #LAYOUT_OPTIMIZE_SPATIAL_CURVE_BUILD_METHOD} value
+ * is set to "sample"
+ *
+ * Determines target sample size used by the Boundary-based Interleaved Index method.
+ * Larger sample size entails better layout optimization outcomes, at the expense of higher memory
+ * footprint.
*/
- public static final ConfigProperty LAYOUT_OPTIMIZE_BUILD_CURVE_SAMPLE_SIZE = ConfigProperty
+ public static final ConfigProperty LAYOUT_OPTIMIZE_BUILD_CURVE_SAMPLE_SIZE = ConfigProperty
.key(LAYOUT_OPTIMIZE_PARAM_PREFIX + "build.curve.sample.size")
.defaultValue("200000")
.sinceVersion("0.10.0")
- .withDocumentation("when setting" + LAYOUT_OPTIMIZE_CURVE_BUILD_METHOD.key() + " to `sample`, the amount of sampling to be done."
- + "Large sample size leads to better results, at the expense of more memory usage.");
+ .withDocumentation("Determines target sample size used by the Boundary-based Interleaved Index method "
+ + "of building space-filling curve. Larger sample size entails better layout optimization outcomes, "
+ + "at the expense of higher memory footprint.");
/**
- * The best way to use Z-order/Space-filling curves is to cooperate with Data-Skipping
- * with data-skipping query engine can greatly reduce the number of table files to be read.
- * otherwise query engine can only do row-group skipping for files (parquet/orc)
+ * @deprecated this setting has no effect
*/
public static final ConfigProperty LAYOUT_OPTIMIZE_DATA_SKIPPING_ENABLE = ConfigProperty
.key(LAYOUT_OPTIMIZE_PARAM_PREFIX + "data.skipping.enable")
.defaultValue(true)
.sinceVersion("0.10.0")
+ .deprecatedAfter("0.11.0")
.withDocumentation("Enable data skipping by collecting statistics once layout optimization is complete.");
public static final ConfigProperty ROLLBACK_PENDING_CLUSTERING_ON_CONFLICT = ConfigProperty
@@ -480,6 +518,11 @@ public Builder withInlineClustering(Boolean inlineClustering) {
return this;
}
+ public Builder withScheduleInlineClustering(Boolean scheduleInlineClustering) {
+ clusteringConfig.setValue(SCHEDULE_INLINE_CLUSTERING, String.valueOf(scheduleInlineClustering));
+ return this;
+ }
+
public Builder withInlineClusteringNumCommits(int numCommits) {
clusteringConfig.setValue(INLINE_CLUSTERING_MAX_COMMITS, String.valueOf(numCommits));
return this;
@@ -516,18 +559,13 @@ public Builder withRollbackPendingClustering(Boolean rollbackPendingClustering)
return this;
}
- public Builder withSpaceFillingCurveDataOptimizeEnable(Boolean enable) {
- clusteringConfig.setValue(LAYOUT_OPTIMIZE_ENABLE, String.valueOf(enable));
- return this;
- }
-
public Builder withDataOptimizeStrategy(String strategy) {
clusteringConfig.setValue(LAYOUT_OPTIMIZE_STRATEGY, strategy);
return this;
}
public Builder withDataOptimizeBuildCurveStrategy(String method) {
- clusteringConfig.setValue(LAYOUT_OPTIMIZE_CURVE_BUILD_METHOD, method);
+ clusteringConfig.setValue(LAYOUT_OPTIMIZE_SPATIAL_CURVE_BUILD_METHOD, method);
return this;
}
@@ -536,17 +574,18 @@ public Builder withDataOptimizeBuildCurveSampleNumber(int sampleNumber) {
return this;
}
- public Builder withDataOptimizeDataSkippingEnable(boolean dataSkipping) {
- clusteringConfig.setValue(LAYOUT_OPTIMIZE_DATA_SKIPPING_ENABLE, String.valueOf(dataSkipping));
- return this;
- }
-
public HoodieClusteringConfig build() {
clusteringConfig.setDefaultValue(
PLAN_STRATEGY_CLASS_NAME, getDefaultPlanStrategyClassName(engineType));
clusteringConfig.setDefaultValue(
EXECUTION_STRATEGY_CLASS_NAME, getDefaultExecutionStrategyClassName(engineType));
clusteringConfig.setDefaults(HoodieClusteringConfig.class.getName());
+
+ boolean inlineCluster = clusteringConfig.getBoolean(HoodieClusteringConfig.INLINE_CLUSTERING);
+ boolean inlineClusterSchedule = clusteringConfig.getBoolean(HoodieClusteringConfig.SCHEDULE_INLINE_CLUSTERING);
+ ValidationUtils.checkArgument(!(inlineCluster && inlineClusterSchedule), String.format("Either of inline clustering (%s) or "
+ + "schedule inline clustering (%s) can be enabled. Both can't be set to true at the same time. %s,%s", HoodieClusteringConfig.INLINE_CLUSTERING.key(),
+ HoodieClusteringConfig.SCHEDULE_INLINE_CLUSTERING.key(), inlineCluster, inlineClusterSchedule));
return clusteringConfig;
}
@@ -578,21 +617,21 @@ private String getDefaultExecutionStrategyClassName(EngineType engineType) {
/**
* Type of a strategy for building Z-order/Hilbert space-filling curves.
*/
- public enum BuildCurveStrategyType {
+ public enum SpatialCurveCompositionStrategyType {
DIRECT("direct"),
SAMPLE("sample");
- private static final Map VALUE_TO_ENUM_MAP =
- TypeUtils.getValueToEnumMap(BuildCurveStrategyType.class, e -> e.value);
+ private static final Map VALUE_TO_ENUM_MAP =
+ TypeUtils.getValueToEnumMap(SpatialCurveCompositionStrategyType.class, e -> e.value);
private final String value;
- BuildCurveStrategyType(String value) {
+ SpatialCurveCompositionStrategyType(String value) {
this.value = value;
}
- public static BuildCurveStrategyType fromValue(String value) {
- BuildCurveStrategyType enumValue = VALUE_TO_ENUM_MAP.get(value);
+ public static SpatialCurveCompositionStrategyType fromValue(String value) {
+ SpatialCurveCompositionStrategyType enumValue = VALUE_TO_ENUM_MAP.get(value);
if (enumValue == null) {
throw new HoodieException(String.format("Invalid value (%s)", value));
}
@@ -605,6 +644,7 @@ public static BuildCurveStrategyType fromValue(String value) {
* Layout optimization strategies such as Z-order/Hilbert space-curves, etc
*/
public enum LayoutOptimizationStrategy {
+ LINEAR("linear"),
ZORDER("z-order"),
HILBERT("hilbert");
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java
index 4d1e197cf8a1e..0aac9308da439 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java
@@ -50,13 +50,6 @@
+ "cleaning (reclamation of older/unused file groups/slices).")
public class HoodieCompactionConfig extends HoodieConfig {
- public static final ConfigProperty AUTO_CLEAN = ConfigProperty
- .key("hoodie.clean.automatic")
- .defaultValue("true")
- .withDocumentation("When enabled, the cleaner table service is invoked immediately after each commit,"
- + " to delete older file slices. It's recommended to enable this, to ensure metadata and data storage"
- + " growth is bounded.");
-
public static final ConfigProperty AUTO_ARCHIVE = ConfigProperty
.key("hoodie.archive.automatic")
.defaultValue("true")
@@ -64,6 +57,20 @@ public class HoodieCompactionConfig extends HoodieConfig {
+ " to archive commits if we cross a maximum value of commits."
+ " It's recommended to enable this, to ensure number of active commits is bounded.");
+ public static final ConfigProperty ASYNC_ARCHIVE = ConfigProperty
+ .key("hoodie.archive.async")
+ .defaultValue("false")
+ .sinceVersion("0.11.0")
+ .withDocumentation("Only applies when " + AUTO_ARCHIVE.key() + " is turned on. "
+ + "When turned on runs archiver async with writing, which can speed up overall write performance.");
+
+ public static final ConfigProperty AUTO_CLEAN = ConfigProperty
+ .key("hoodie.clean.automatic")
+ .defaultValue("true")
+ .withDocumentation("When enabled, the cleaner table service is invoked immediately after each commit,"
+ + " to delete older file slices. It's recommended to enable this, to ensure metadata and data storage"
+ + " growth is bounded.");
+
public static final ConfigProperty ASYNC_CLEAN = ConfigProperty
.key("hoodie.clean.async")
.defaultValue("false")
@@ -76,6 +83,12 @@ public class HoodieCompactionConfig extends HoodieConfig {
.withDocumentation("Number of commits to retain, without cleaning. This will be retained for num_of_commits * time_between_commits "
+ "(scheduled). This also directly translates into how much data retention the table supports for incremental queries.");
+ public static final ConfigProperty CLEANER_HOURS_RETAINED = ConfigProperty.key("hoodie.cleaner.hours.retained")
+ .defaultValue("24")
+ .withDocumentation("Number of hours for which commits need to be retained. This config provides a more flexible option as"
+ + "compared to number of commits retained for cleaning service. Setting this property ensures all the files, but the latest in a file group,"
+ + " corresponding to commits with commit times older than the configured number of hours to be retained are cleaned.");
+
public static final ConfigProperty CLEANER_POLICY = ConfigProperty
.key("hoodie.cleaner.policy")
.defaultValue(HoodieCleaningPolicy.KEEP_LATEST_COMMITS.name())
@@ -90,6 +103,16 @@ public class HoodieCompactionConfig extends HoodieConfig {
.withDocumentation("When set to true, compaction service is triggered after each write. While being "
+ " simpler operationally, this adds extra latency on the write path.");
+ public static final ConfigProperty SCHEDULE_INLINE_COMPACT = ConfigProperty
+ .key("hoodie.compact.schedule.inline")
+ .defaultValue("false")
+ .withDocumentation("When set to true, compaction service will be attempted for inline scheduling after each write. Users have to ensure "
+ + "they have a separate job to run async compaction(execution) for the one scheduled by this writer. Users can choose to set both "
+ + "`hoodie.compact.inline` and `hoodie.compact.schedule.inline` to false and have both scheduling and execution triggered by any async process. "
+ + "But if `hoodie.compact.inline` is set to false, and `hoodie.compact.schedule.inline` is set to true, regular writers will schedule compaction inline, "
+ + "but users are expected to trigger async job for execution. If `hoodie.compact.inline` is set to true, regular writers will do both scheduling and "
+ + "execution inline for compaction");
+
public static final ConfigProperty INLINE_COMPACT_NUM_DELTA_COMMITS = ConfigProperty
.key("hoodie.compact.inline.max.delta.commits")
.defaultValue("5")
@@ -156,7 +179,8 @@ public class HoodieCompactionConfig extends HoodieConfig {
.defaultValue(String.valueOf(104857600))
.withDocumentation("During upsert operation, we opportunistically expand existing small files on storage, instead of writing"
+ " new files, to keep number of files to an optimum. This config sets the file size limit below which a file on storage "
- + " becomes a candidate to be selected as such a `small file`. By default, treat any file <= 100MB as a small file.");
+ + " becomes a candidate to be selected as such a `small file`. By default, treat any file <= 100MB as a small file."
+ + " Also note that if this set <= 0, will not try to get small files and directly write new files");
public static final ConfigProperty RECORD_SIZE_ESTIMATION_THRESHOLD = ConfigProperty
.key("hoodie.record.size.estimation.threshold")
@@ -200,7 +224,7 @@ public class HoodieCompactionConfig extends HoodieConfig {
public static final ConfigProperty COMPACTION_LAZY_BLOCK_READ_ENABLE = ConfigProperty
.key("hoodie.compaction.lazy.block.read")
- .defaultValue("false")
+ .defaultValue("true")
.withDocumentation("When merging the delta log files, this config helps to choose whether the log blocks "
+ "should be read lazily or not. Choose true to use lazy block reading (low memory usage, but incurs seeks to each block"
+ " header) or false for immediate block read (higher memory usage)");
@@ -254,6 +278,13 @@ public class HoodieCompactionConfig extends HoodieConfig {
.withDocumentation("The average record size. If not explicitly specified, hudi will compute the "
+ "record size estimate compute dynamically based on commit metadata. "
+ " This is critical in computing the insert parallelism and bin-packing inserts into small files.");
+
+ public static final ConfigProperty ALLOW_MULTIPLE_CLEANS = ConfigProperty
+ .key("hoodie.clean.allow.multiple")
+ .defaultValue(true)
+ .sinceVersion("0.11.0")
+ .withDocumentation("Allows scheduling/executing multiple cleans by enabling this config. If users prefer to strictly ensure clean requests should be mutually exclusive, "
+ + ".i.e. a 2nd clean will not be scheduled if another clean is not yet completed to avoid repeat cleaning of same files, they might want to disable this config.");
public static final ConfigProperty ARCHIVE_MERGE_FILES_BATCH_SIZE = ConfigProperty
.key("hoodie.archive.merge.files.batch.size")
@@ -512,6 +543,16 @@ public Builder fromProperties(Properties props) {
return this;
}
+ public Builder withAutoArchive(Boolean autoArchive) {
+ compactionConfig.setValue(AUTO_ARCHIVE, String.valueOf(autoArchive));
+ return this;
+ }
+
+ public Builder withAsyncArchive(Boolean asyncArchive) {
+ compactionConfig.setValue(ASYNC_ARCHIVE, String.valueOf(asyncArchive));
+ return this;
+ }
+
public Builder withAutoClean(Boolean autoClean) {
compactionConfig.setValue(AUTO_CLEAN, String.valueOf(autoClean));
return this;
@@ -522,11 +563,6 @@ public Builder withAsyncClean(Boolean asyncClean) {
return this;
}
- public Builder withAutoArchive(Boolean autoArchive) {
- compactionConfig.setValue(AUTO_ARCHIVE, String.valueOf(autoArchive));
- return this;
- }
-
public Builder withIncrementalCleaningMode(Boolean incrementalCleaningMode) {
compactionConfig.setValue(CLEANER_INCREMENTAL_MODE_ENABLE, String.valueOf(incrementalCleaningMode));
return this;
@@ -537,6 +573,11 @@ public Builder withInlineCompaction(Boolean inlineCompaction) {
return this;
}
+ public Builder withScheduleInlineCompaction(Boolean scheduleAsyncCompaction) {
+ compactionConfig.setValue(SCHEDULE_INLINE_COMPACT, String.valueOf(scheduleAsyncCompaction));
+ return this;
+ }
+
public Builder withInlineCompactionTriggerStrategy(CompactionTriggerStrategy compactionTriggerStrategy) {
compactionConfig.setValue(INLINE_COMPACT_TRIGGER_STRATEGY, compactionTriggerStrategy.name());
return this;
@@ -557,6 +598,11 @@ public Builder retainCommits(int commitsRetained) {
return this;
}
+ public Builder cleanerNumHoursRetained(int cleanerHoursRetained) {
+ compactionConfig.setValue(CLEANER_HOURS_RETAINED, String.valueOf(cleanerHoursRetained));
+ return this;
+ }
+
public Builder archiveCommitsWith(int minToKeep, int maxToKeep) {
compactionConfig.setValue(MIN_COMMITS_TO_KEEP, String.valueOf(minToKeep));
compactionConfig.setValue(MAX_COMMITS_TO_KEEP, String.valueOf(maxToKeep));
@@ -603,6 +649,11 @@ public Builder approxRecordSize(int recordSizeEstimate) {
return this;
}
+ public Builder allowMultipleCleans(boolean allowMultipleCleanSchedules) {
+ compactionConfig.setValue(ALLOW_MULTIPLE_CLEANS, String.valueOf(allowMultipleCleanSchedules));
+ return this;
+ }
+
public Builder withCleanerParallelism(int cleanerParallelism) {
compactionConfig.setValue(CLEANER_PARALLELISM_VALUE, String.valueOf(cleanerParallelism));
return this;
@@ -700,6 +751,12 @@ public HoodieCompactionConfig build() {
+ "missing data from few instants.",
HoodieCompactionConfig.MIN_COMMITS_TO_KEEP.key(), minInstantsToKeep,
HoodieCompactionConfig.CLEANER_COMMITS_RETAINED.key(), cleanerCommitsRetained));
+
+ boolean inlineCompact = compactionConfig.getBoolean(HoodieCompactionConfig.INLINE_COMPACT);
+ boolean inlineCompactSchedule = compactionConfig.getBoolean(HoodieCompactionConfig.SCHEDULE_INLINE_COMPACT);
+ ValidationUtils.checkArgument(!(inlineCompact && inlineCompactSchedule), String.format("Either of inline compaction (%s) or "
+ + "schedule inline compaction (%s) can be enabled. Both can't be set to true at the same time. %s, %s", HoodieCompactionConfig.INLINE_COMPACT.key(),
+ HoodieCompactionConfig.SCHEDULE_INLINE_COMPACT.key(), inlineCompact, inlineCompactSchedule));
return compactionConfig;
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java
index e156310c736b0..f82f14d5a9c64 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java
@@ -152,7 +152,7 @@ public class HoodieIndexConfig extends HoodieConfig {
public static final ConfigProperty SIMPLE_INDEX_PARALLELISM = ConfigProperty
.key("hoodie.simple.index.parallelism")
- .defaultValue("50")
+ .defaultValue("100")
.withDocumentation("Only applies if index type is SIMPLE. "
+ "This is the amount of parallelism for index lookup, which involves a Spark Shuffle");
@@ -568,7 +568,7 @@ public HoodieIndexConfig build() {
private String getDefaultIndexType(EngineType engineType) {
switch (engineType) {
case SPARK:
- return HoodieIndex.IndexType.BLOOM.name();
+ return HoodieIndex.IndexType.SIMPLE.name();
case FLINK:
case JAVA:
return HoodieIndex.IndexType.INMEMORY.name();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java
index 42689ec18e948..6447a039cc069 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java
@@ -83,14 +83,17 @@ public class HoodieStorageConfig extends HoodieConfig {
.withDocumentation("Lower values increase the size of metadata tracked within HFile, but can offer potentially "
+ "faster lookup times.");
- // used to size log files
+ public static final ConfigProperty LOGFILE_DATA_BLOCK_FORMAT = ConfigProperty
+ .key("hoodie.logfile.data.block.format")
+ .noDefaultValue()
+ .withDocumentation("Format of the data block within delta logs. Following formats are currently supported \"avro\", \"hfile\", \"parquet\"");
+
public static final ConfigProperty LOGFILE_MAX_SIZE = ConfigProperty
.key("hoodie.logfile.max.size")
.defaultValue(String.valueOf(1024 * 1024 * 1024)) // 1 GB
.withDocumentation("LogFile max size. This is the maximum size allowed for a log file "
+ "before it is rolled over to the next version.");
- // used to size data blocks in log file
public static final ConfigProperty LOGFILE_DATA_BLOCK_MAX_SIZE = ConfigProperty
.key("hoodie.logfile.data.block.max.size")
.defaultValue(String.valueOf(256 * 1024 * 1024))
@@ -124,7 +127,7 @@ public class HoodieStorageConfig extends HoodieConfig {
public static final ConfigProperty PARQUET_OUTPUT_TIMESTAMP_TYPE = ConfigProperty
.key("hoodie.parquet.outputtimestamptype")
- .defaultValue("TIMESTAMP_MILLIS")
+ .defaultValue("TIMESTAMP_MICROS")
.withDocumentation("Sets spark.sql.parquet.outputTimestampType. Parquet timestamp type to use when Spark writes data to Parquet files.");
public static final ConfigProperty HFILE_COMPRESSION_ALGORITHM_NAME = ConfigProperty
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index 5896ac49ea69d..b7b410817b2fd 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -31,15 +31,18 @@
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.engine.EngineType;
import org.apache.hudi.common.fs.ConsistencyGuardConfig;
+import org.apache.hudi.common.fs.FileSystemRetryConfig;
import org.apache.hudi.common.model.HoodieCleaningPolicy;
import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy;
import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload;
import org.apache.hudi.common.model.WriteConcurrencyMode;
import org.apache.hudi.common.table.HoodieTableConfig;
+import org.apache.hudi.common.table.log.block.HoodieLogBlock;
import org.apache.hudi.common.table.marker.MarkerType;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
+import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
@@ -59,9 +62,9 @@
import org.apache.hudi.table.action.cluster.ClusteringPlanPartitionFilterMode;
import org.apache.hudi.table.action.compact.CompactionTriggerStrategy;
import org.apache.hudi.table.action.compact.strategy.CompactionStrategy;
+import org.apache.hudi.table.storage.HoodieStorageLayout;
import org.apache.hadoop.hbase.io.compress.Compression;
-import org.apache.hudi.table.storage.HoodieStorageLayout;
import org.apache.orc.CompressionKind;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
@@ -438,7 +441,20 @@ public class HoodieWriteConfig extends HoodieConfig {
.sinceVersion("0.10.0")
.withDocumentation("File Id Prefix provider class, that implements `org.apache.hudi.fileid.FileIdPrefixProvider`");
+ public static final ConfigProperty TABLE_SERVICES_ENABLED = ConfigProperty
+ .key("hoodie.table.services.enabled")
+ .defaultValue(true)
+ .sinceVersion("0.11.0")
+ .withDocumentation("Master control to disable all table services including archive, clean, compact, cluster, etc.");
+
+ public static final ConfigProperty RELEASE_RESOURCE_ENABLE = ConfigProperty
+ .key("hoodie.release.resource.on.completion.enable")
+ .defaultValue(true)
+ .sinceVersion("0.11.0")
+ .withDocumentation("Control to enable release all persist rdds when the spark job finish.");
+
private ConsistencyGuardConfig consistencyGuardConfig;
+ private FileSystemRetryConfig fileSystemRetryConfig;
// Hoodie Write Client transparently rewrites File System View config when embedded mode is enabled
// We keep track of original config and rewritten config
@@ -832,6 +848,7 @@ protected HoodieWriteConfig(EngineType engineType, Properties props) {
newProps.putAll(props);
this.engineType = engineType;
this.consistencyGuardConfig = ConsistencyGuardConfig.newBuilder().fromProperties(newProps).build();
+ this.fileSystemRetryConfig = FileSystemRetryConfig.newBuilder().fromProperties(newProps).build();
this.clientSpecifiedViewStorageConfig = FileSystemViewStorageConfig.newBuilder().fromProperties(newProps).build();
this.viewStorageConfig = clientSpecifiedViewStorageConfig;
this.hoodiePayloadConfig = HoodiePayloadConfig.newBuilder().fromProperties(newProps).build();
@@ -1074,6 +1091,10 @@ public int getCleanerCommitsRetained() {
return getInt(HoodieCompactionConfig.CLEANER_COMMITS_RETAINED);
}
+ public int getCleanerHoursRetained() {
+ return getInt(HoodieCompactionConfig.CLEANER_HOURS_RETAINED);
+ }
+
public int getMaxCommitsToKeep() {
return getInt(HoodieCompactionConfig.MAX_COMMITS_TO_KEEP);
}
@@ -1102,6 +1123,10 @@ public int getCopyOnWriteRecordSizeEstimate() {
return getInt(HoodieCompactionConfig.COPY_ON_WRITE_RECORD_SIZE_ESTIMATE);
}
+ public boolean allowMultipleCleans() {
+ return getBoolean(HoodieCompactionConfig.ALLOW_MULTIPLE_CLEANS);
+ }
+
public boolean shouldAutoTuneInsertSplits() {
return getBoolean(HoodieCompactionConfig.COPY_ON_WRITE_AUTO_SPLIT_INSERTS);
}
@@ -1110,10 +1135,6 @@ public int getCleanerParallelism() {
return getInt(HoodieCompactionConfig.CLEANER_PARALLELISM_VALUE);
}
- public boolean isAutoClean() {
- return getBoolean(HoodieCompactionConfig.AUTO_CLEAN);
- }
-
public boolean getArchiveMergeEnable() {
return getBoolean(HoodieCompactionConfig.ARCHIVE_MERGE_ENABLE);
}
@@ -1126,6 +1147,14 @@ public boolean isAutoArchive() {
return getBoolean(HoodieCompactionConfig.AUTO_ARCHIVE);
}
+ public boolean isAsyncArchive() {
+ return getBoolean(HoodieCompactionConfig.ASYNC_ARCHIVE);
+ }
+
+ public boolean isAutoClean() {
+ return getBoolean(HoodieCompactionConfig.AUTO_CLEAN);
+ }
+
public boolean isAsyncClean() {
return getBoolean(HoodieCompactionConfig.ASYNC_CLEAN);
}
@@ -1138,6 +1167,10 @@ public boolean inlineCompactionEnabled() {
return getBoolean(HoodieCompactionConfig.INLINE_COMPACT);
}
+ public boolean scheduleInlineCompaction() {
+ return getBoolean(HoodieCompactionConfig.SCHEDULE_INLINE_COMPACT);
+ }
+
public CompactionTriggerStrategy getInlineCompactTriggerStrategy() {
return CompactionTriggerStrategy.valueOf(getString(HoodieCompactionConfig.INLINE_COMPACT_TRIGGER_STRATEGY));
}
@@ -1178,6 +1211,10 @@ public boolean inlineClusteringEnabled() {
return getBoolean(HoodieClusteringConfig.INLINE_CLUSTERING);
}
+ public boolean scheduleInlineClustering() {
+ return getBoolean(HoodieClusteringConfig.SCHEDULE_INLINE_CLUSTERING);
+ }
+
public boolean isAsyncClusteringEnabled() {
return getBoolean(HoodieClusteringConfig.ASYNC_CLUSTERING_ENABLE);
}
@@ -1288,30 +1325,21 @@ public String getClusteringSortColumns() {
return getString(HoodieClusteringConfig.PLAN_STRATEGY_SORT_COLUMNS);
}
- /**
- * Data layout optimize properties.
- */
- public boolean isLayoutOptimizationEnabled() {
- return getBoolean(HoodieClusteringConfig.LAYOUT_OPTIMIZE_ENABLE);
+ public HoodieClusteringConfig.LayoutOptimizationStrategy getLayoutOptimizationStrategy() {
+ return HoodieClusteringConfig.LayoutOptimizationStrategy.fromValue(
+ getStringOrDefault(HoodieClusteringConfig.LAYOUT_OPTIMIZE_STRATEGY)
+ );
}
- public String getLayoutOptimizationStrategy() {
- return getString(HoodieClusteringConfig.LAYOUT_OPTIMIZE_STRATEGY);
- }
-
- public HoodieClusteringConfig.BuildCurveStrategyType getLayoutOptimizationCurveBuildMethod() {
- return HoodieClusteringConfig.BuildCurveStrategyType.fromValue(
- getString(HoodieClusteringConfig.LAYOUT_OPTIMIZE_CURVE_BUILD_METHOD));
+ public HoodieClusteringConfig.SpatialCurveCompositionStrategyType getLayoutOptimizationCurveBuildMethod() {
+ return HoodieClusteringConfig.SpatialCurveCompositionStrategyType.fromValue(
+ getString(HoodieClusteringConfig.LAYOUT_OPTIMIZE_SPATIAL_CURVE_BUILD_METHOD));
}
public int getLayoutOptimizationSampleSize() {
return getInt(HoodieClusteringConfig.LAYOUT_OPTIMIZE_BUILD_CURVE_SAMPLE_SIZE);
}
- public boolean isDataSkippingEnabled() {
- return getBoolean(HoodieClusteringConfig.LAYOUT_OPTIMIZE_DATA_SKIPPING_ENABLE);
- }
-
/**
* index properties.
*/
@@ -1444,6 +1472,14 @@ public boolean useBloomIndexBucketizedChecking() {
return getBoolean(HoodieIndexConfig.BLOOM_INDEX_BUCKETIZED_CHECKING);
}
+ public boolean isMetadataBloomFilterIndexEnabled() {
+ return isMetadataTableEnabled() && getMetadataConfig().isBloomFilterIndexEnabled();
+ }
+
+ public boolean isMetadataIndexColumnStatsForAllColumnsEnabled() {
+ return isMetadataTableEnabled() && getMetadataConfig().isMetadataColumnStatsIndexForAllColumnsEnabled();
+ }
+
public int getBloomIndexKeysPerBucket() {
return getInt(HoodieIndexConfig.BLOOM_INDEX_KEYS_PER_BUCKET);
}
@@ -1515,6 +1551,11 @@ public String parquetOutputTimestampType() {
return getString(HoodieStorageConfig.PARQUET_OUTPUT_TIMESTAMP_TYPE);
}
+ public Option getLogDataBlockFormat() {
+ return Option.ofNullable(getString(HoodieStorageConfig.LOGFILE_DATA_BLOCK_FORMAT))
+ .map(HoodieLogBlock.HoodieLogBlockType::fromId);
+ }
+
public long getLogFileMaxSize() {
return getLong(HoodieStorageConfig.LOGFILE_MAX_SIZE);
}
@@ -1681,7 +1722,7 @@ public boolean getPushGatewayRandomJobNameSuffix() {
public String getMetricReporterMetricsNamePrefix() {
return getStringOrDefault(HoodieMetricsConfig.METRICS_REPORTER_PREFIX);
}
-
+
/**
* memory configs.
*/
@@ -1701,6 +1742,10 @@ public ConsistencyGuardConfig getConsistencyGuardConfig() {
return consistencyGuardConfig;
}
+ public FileSystemRetryConfig getFileSystemRetryConfig() {
+ return fileSystemRetryConfig;
+ }
+
public void setConsistencyGuardConfig(ConsistencyGuardConfig consistencyGuardConfig) {
this.consistencyGuardConfig = consistencyGuardConfig;
}
@@ -1853,12 +1898,12 @@ public WriteConcurrencyMode getWriteConcurrencyMode() {
}
/**
- * Are any table services configured to run inline?
+ * Are any table services configured to run inline for both scheduling and execution?
*
* @return True if any table services are configured to run inline, false otherwise.
*/
- public Boolean areAnyTableServicesInline() {
- return inlineClusteringEnabled() || inlineCompactionEnabled() || isAutoClean();
+ public Boolean areAnyTableServicesExecutedInline() {
+ return inlineClusteringEnabled() || inlineCompactionEnabled() || isAutoClean() || isAutoArchive();
}
/**
@@ -1867,7 +1912,11 @@ public Boolean areAnyTableServicesInline() {
* @return True if any table services are configured to run async, false otherwise.
*/
public Boolean areAnyTableServicesAsync() {
- return isAsyncClusteringEnabled() || !inlineCompactionEnabled() || isAsyncClean();
+ return isAsyncClusteringEnabled() || !inlineCompactionEnabled() || isAsyncClean() || isAsyncArchive();
+ }
+
+ public Boolean areAnyTableServicesScheduledInline() {
+ return scheduleInlineCompaction() || scheduleInlineClustering();
}
public String getPreCommitValidators() {
@@ -1898,6 +1947,14 @@ public String getFileIdPrefixProviderClassName() {
return getString(FILEID_PREFIX_PROVIDER_CLASS);
}
+ public boolean areTableServicesEnabled() {
+ return getBooleanOrDefault(TABLE_SERVICES_ENABLED);
+ }
+
+ public boolean areReleaseResourceEnabled() {
+ return getBooleanOrDefault(RELEASE_RESOURCE_ENABLE);
+ }
+
/**
* Layout configs.
*/
@@ -2263,6 +2320,16 @@ public Builder withFileIdPrefixProviderClassName(String fileIdPrefixProviderClas
return this;
}
+ public Builder withTableServicesEnabled(boolean enabled) {
+ writeConfig.setValue(TABLE_SERVICES_ENABLED, Boolean.toString(enabled));
+ return this;
+ }
+
+ public Builder withReleaseResourceEnabled(boolean enabled) {
+ writeConfig.setValue(RELEASE_RESOURCE_ENABLE, Boolean.toString(enabled));
+ return this;
+ }
+
public Builder withProperties(Properties properties) {
this.writeConfig.getProps().putAll(properties);
return this;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/exception/HoodieRestoreException.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/exception/HoodieRestoreException.java
index c6c9076f51bae..baad53aba5941 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/exception/HoodieRestoreException.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/exception/HoodieRestoreException.java
@@ -23,4 +23,8 @@ public class HoodieRestoreException extends HoodieException {
public HoodieRestoreException(String msg, Throwable e) {
super(msg, e);
}
+
+ public HoodieRestoreException(String msg) {
+ super(msg);
+ }
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/HoodieLazyInsertIterable.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/HoodieLazyInsertIterable.java
index 0d28c74e13f9b..b078076b864f5 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/HoodieLazyInsertIterable.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/HoodieLazyInsertIterable.java
@@ -18,7 +18,6 @@
package org.apache.hudi.execution;
-import java.util.Properties;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.client.utils.LazyIterableIterator;
import org.apache.hudi.common.engine.TaskContextSupplier;
@@ -36,6 +35,7 @@
import java.util.Iterator;
import java.util.List;
+import java.util.Properties;
import java.util.function.Function;
/**
@@ -87,7 +87,7 @@ public static class HoodieInsertValueGenResult {
public HoodieInsertValueGenResult(T record, Schema schema, Properties properties) {
this.record = record;
try {
- this.insertValue = record.getData().getInsertValue(schema, properties);
+ this.insertValue = ((HoodieRecordPayload) record.getData()).getInsertValue(schema, properties);
} catch (Exception e) {
this.exception = Option.of(e);
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndex.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndex.java
index 0428d12c40306..922371c4a0f45 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndex.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndex.java
@@ -27,7 +27,6 @@
import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
-import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.model.WriteOperationType;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieIndexException;
@@ -39,13 +38,11 @@
/**
* Base class for different types of indexes to determine the mapping from uuid.
*
- * @param Sub type of HoodieRecordPayload
* @param Type of inputs for deprecated APIs
- * @param Type of keys for deprecated APIs
* @param Type of outputs for deprecated APIs
*/
@PublicAPIClass(maturity = ApiMaturityLevel.EVOLVING)
-public abstract class HoodieIndex implements Serializable {
+public abstract class HoodieIndex implements Serializable {
protected final HoodieWriteConfig config;
@@ -60,7 +57,7 @@ protected HoodieIndex(HoodieWriteConfig config) {
@Deprecated
@PublicAPIMethod(maturity = ApiMaturityLevel.DEPRECATED)
public I tagLocation(I records, HoodieEngineContext context,
- HoodieTable hoodieTable) throws HoodieIndexException {
+ HoodieTable hoodieTable) throws HoodieIndexException {
throw new HoodieNotSupportedException("Deprecated API should not be called");
}
@@ -70,7 +67,7 @@ public I tagLocation(I records, HoodieEngineContext context,
@Deprecated
@PublicAPIMethod(maturity = ApiMaturityLevel.DEPRECATED)
public O updateLocation(O writeStatuses, HoodieEngineContext context,
- HoodieTable hoodieTable) throws HoodieIndexException {
+ HoodieTable hoodieTable) throws HoodieIndexException {
throw new HoodieNotSupportedException("Deprecated API should not be called");
}
@@ -79,8 +76,8 @@ public O updateLocation(O writeStatuses, HoodieEngineContext context,
* the row (if it is actually present).
*/
@PublicAPIMethod(maturity = ApiMaturityLevel.EVOLVING)
- public abstract HoodieData> tagLocation(
- HoodieData> records, HoodieEngineContext context,
+ public abstract HoodieData> tagLocation(
+ HoodieData> records, HoodieEngineContext context,
HoodieTable hoodieTable) throws HoodieIndexException;
/**
@@ -144,6 +141,6 @@ public void close() {
}
public enum IndexType {
- HBASE, INMEMORY, BLOOM, GLOBAL_BLOOM, SIMPLE, GLOBAL_SIMPLE, BUCKET
+ HBASE, INMEMORY, BLOOM, GLOBAL_BLOOM, SIMPLE, GLOBAL_SIMPLE, BUCKET, FLINK_STATE
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java
index e5426ca1161f9..b714c50334b4f 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java
@@ -18,17 +18,30 @@
package org.apache.hudi.index;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordLocation;
import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.util.HoodieTimer;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.exception.HoodieIndexException;
+import org.apache.hudi.io.storage.HoodieFileReader;
+import org.apache.hudi.io.storage.HoodieFileReaderFactory;
import org.apache.hudi.table.HoodieTable;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
import static java.util.stream.Collectors.toList;
@@ -37,6 +50,8 @@
*/
public class HoodieIndexUtils {
+ private static final Logger LOG = LogManager.getLogger(HoodieIndexUtils.class);
+
/**
* Fetches Pair of partition path and {@link HoodieBaseFile}s for interested partitions.
*
@@ -87,18 +102,48 @@ public static List> getLatestBaseFilesForAllPartiti
* @return the tagged {@link HoodieRecord}
*/
public static HoodieRecord getTaggedRecord(HoodieRecord inputRecord, Option location) {
- HoodieRecord record = inputRecord;
+ HoodieRecord record = inputRecord;
if (location.isPresent()) {
// When you have a record in multiple files in the same partition, then collection
// will have 2 entries with the same exact in memory copy of the HoodieRecord and the 2
// separate filenames that the record is found in. This will result in setting
// currentLocation 2 times and it will fail the second time. So creating a new in memory
// copy of the hoodie record.
- record = new HoodieRecord<>(inputRecord);
+ record = inputRecord.newInstance();
record.unseal();
record.setCurrentLocation(location.get());
record.seal();
}
return record;
}
+
+ /**
+ * Given a list of row keys and one file, return only row keys existing in that file.
+ *
+ * @param filePath - File to filter keys from
+ * @param candidateRecordKeys - Candidate keys to filter
+ * @return List of candidate keys that are available in the file
+ */
+ public static List filterKeysFromFile(Path filePath, List candidateRecordKeys,
+ Configuration configuration) throws HoodieIndexException {
+ ValidationUtils.checkArgument(FSUtils.isBaseFile(filePath));
+ List foundRecordKeys = new ArrayList<>();
+ try {
+ // Load all rowKeys from the file, to double-confirm
+ if (!candidateRecordKeys.isEmpty()) {
+ HoodieTimer timer = new HoodieTimer().startTimer();
+ HoodieFileReader fileReader = HoodieFileReaderFactory.getFileReader(configuration, filePath);
+ Set fileRowKeys = fileReader.filterRowKeys(new TreeSet<>(candidateRecordKeys));
+ foundRecordKeys.addAll(fileRowKeys);
+ LOG.info(String.format("Checked keys against file %s, in %d ms. #candidates (%d) #found (%d)", filePath,
+ timer.endTimer(), candidateRecordKeys.size(), foundRecordKeys.size()));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Keys matching for file " + filePath + " => " + foundRecordKeys);
+ }
+ }
+ } catch (Exception e) {
+ throw new HoodieIndexException("Error checking candidate keys against file.", e);
+ }
+ return foundRecordKeys;
+ }
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/BaseHoodieBloomIndexHelper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/BaseHoodieBloomIndexHelper.java
index 9f0e815632f38..9430d9bb5e50b 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/BaseHoodieBloomIndexHelper.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/BaseHoodieBloomIndexHelper.java
@@ -24,7 +24,7 @@
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecordLocation;
-import org.apache.hudi.common.util.collection.ImmutablePair;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
@@ -51,7 +51,7 @@ public abstract class BaseHoodieBloomIndexHelper implements Serializable {
public abstract HoodiePairData findMatchingFilesForRecordKeys(
HoodieWriteConfig config, HoodieEngineContext context, HoodieTable hoodieTable,
HoodiePairData partitionRecordKeyPairs,
- HoodieData> fileComparisonPairs,
+ HoodieData> fileComparisonPairs,
Map> partitionToFileInfo,
Map recordsPerPartition);
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/HoodieBaseBloomIndexCheckFunction.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/HoodieBaseBloomIndexCheckFunction.java
index 441a212c59f40..80031f4e8f025 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/HoodieBaseBloomIndexCheckFunction.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/HoodieBaseBloomIndexCheckFunction.java
@@ -25,7 +25,7 @@
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIndexException;
import org.apache.hudi.io.HoodieKeyLookupHandle;
-import org.apache.hudi.io.HoodieKeyLookupHandle.KeyLookupResult;
+import org.apache.hudi.io.HoodieKeyLookupResult;
import org.apache.hudi.table.HoodieTable;
import java.util.function.Function;
@@ -37,7 +37,7 @@
* Function performing actual checking of list containing (fileId, hoodieKeys) against the actual files.
*/
public class HoodieBaseBloomIndexCheckFunction
- implements Function>, Iterator>> {
+ implements Function>, Iterator>> {
private final HoodieTable hoodieTable;
@@ -49,11 +49,11 @@ public HoodieBaseBloomIndexCheckFunction(HoodieTable hoodieTable, HoodieWriteCon
}
@Override
- public Iterator> apply(Iterator> filePartitionRecordKeyTripletItr) {
+ public Iterator> apply(Iterator> filePartitionRecordKeyTripletItr) {
return new LazyKeyCheckIterator(filePartitionRecordKeyTripletItr);
}
- class LazyKeyCheckIterator extends LazyIterableIterator, List> {
+ class LazyKeyCheckIterator extends LazyIterableIterator, List> {
private HoodieKeyLookupHandle keyLookupHandle;
@@ -66,8 +66,8 @@ protected void start() {
}
@Override
- protected List computeNext() {
- List ret = new ArrayList<>();
+ protected List computeNext() {
+ List ret = new ArrayList<>();
try {
// process one file in each go.
while (inputItr.hasNext()) {
@@ -83,7 +83,7 @@ protected List computeNext() {
}
// if continue on current file
- if (keyLookupHandle.getPartitionPathFilePair().equals(partitionPathFilePair)) {
+ if (keyLookupHandle.getPartitionPathFileIDPair().equals(partitionPathFilePair)) {
keyLookupHandle.addKey(recordKey);
} else {
// do the actual checking of file & break out
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java
index a223d695cc03a..d3e73c058cc56 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java
@@ -19,20 +19,22 @@
package org.apache.hudi.index.bloom;
+import org.apache.hudi.avro.model.HoodieMetadataColumnStats;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.common.data.HoodieData;
import org.apache.hudi.common.data.HoodiePairData;
import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordLocation;
-import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ImmutablePair;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieIndexConfig;
import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.exception.MetadataNotFoundException;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.index.HoodieIndexUtils;
@@ -46,6 +48,7 @@
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.mapping;
@@ -55,8 +58,7 @@
/**
* Indexing mechanism based on bloom filter. Each parquet file includes its row_key bloom filter in its metadata.
*/
-public class HoodieBloomIndex>
- extends HoodieIndex {
+public class HoodieBloomIndex extends HoodieIndex {
private static final Logger LOG = LogManager.getLogger(HoodieBloomIndex.class);
private final BaseHoodieBloomIndexHelper bloomIndexHelper;
@@ -67,8 +69,8 @@ public HoodieBloomIndex(HoodieWriteConfig config, BaseHoodieBloomIndexHelper blo
}
@Override
- public HoodieData> tagLocation(
- HoodieData> records, HoodieEngineContext context,
+ public HoodieData> tagLocation(
+ HoodieData> records, HoodieEngineContext context,
HoodieTable hoodieTable) {
// Step 0: cache the input records if needed
if (config.getBloomIndexUseCaching()) {
@@ -94,7 +96,7 @@ record -> new ImmutablePair<>(record.getPartitionPath(), record.getRecordKey()))
}
// Step 3: Tag the incoming records, as inserts or updates, by joining with existing record keys
- HoodieData> taggedRecords = tagLocationBacktoRecords(keyFilenamePairs, records);
+ HoodieData> taggedRecords = tagLocationBacktoRecords(keyFilenamePairs, records);
if (config.getBloomIndexUseCaching()) {
records.unpersist();
@@ -111,19 +113,25 @@ record -> new ImmutablePair<>(record.getPartitionPath(), record.getRecordKey()))
private HoodiePairData lookupIndex(
HoodiePairData partitionRecordKeyPairs, final HoodieEngineContext context,
final HoodieTable hoodieTable) {
- // Obtain records per partition, in the incoming records
+ // Step 1: Obtain records per partition, in the incoming records
Map recordsPerPartition = partitionRecordKeyPairs.countByKey();
List affectedPartitionPathList = new ArrayList<>(recordsPerPartition.keySet());
// Step 2: Load all involved files as pairs
- List> fileInfoList =
- loadInvolvedFiles(affectedPartitionPathList, context, hoodieTable);
+ List> fileInfoList;
+ if (config.getBloomIndexPruneByRanges()) {
+ fileInfoList = (config.getMetadataConfig().isColumnStatsIndexEnabled()
+ ? loadColumnRangesFromMetaIndex(affectedPartitionPathList, context, hoodieTable)
+ : loadColumnRangesFromFiles(affectedPartitionPathList, context, hoodieTable));
+ } else {
+ fileInfoList = getFileInfoForLatestBaseFiles(affectedPartitionPathList, context, hoodieTable);
+ }
final Map> partitionToFileInfo =
fileInfoList.stream().collect(groupingBy(Pair::getLeft, mapping(Pair::getRight, toList())));
// Step 3: Obtain a HoodieData, for each incoming record, that already exists, with the file id,
// that contains it.
- HoodieData> fileComparisonPairs =
+ HoodieData> fileComparisonPairs =
explodeRecordsWithFileComparisons(partitionToFileInfo, partitionRecordKeyPairs);
return bloomIndexHelper.findMatchingFilesForRecordKeys(config, context, hoodieTable,
@@ -133,30 +141,84 @@ private HoodiePairData