> instantsToRollback, boolean skipLocking) {
@@ -1425,17 +1438,38 @@ public HoodieMetrics getMetrics() {
}
/**
- * Instantiates engine-specific instance of {@link HoodieTable} as well as performs necessary
- * bootstrapping operations (for ex, validating whether Metadata Table has to be bootstrapped)
+ * Performs necessary bootstrapping operations (for ex, validating whether Metadata Table has to be bootstrapped).
*
- * NOTE: THIS OPERATION IS EXECUTED UNDER LOCK, THEREFORE SHOULD AVOID ANY OPERATIONS
- * NOT REQUIRING EXTERNAL SYNCHRONIZATION
+ * NOTE: THIS OPERATION IS EXECUTED UNDER LOCK, THEREFORE SHOULD AVOID ANY OPERATIONS
+ * NOT REQUIRING EXTERNAL SYNCHRONIZATION
*
* @param metaClient instance of {@link HoodieTableMetaClient}
* @param instantTime current inflight instant time
- * @return instantiated {@link HoodieTable}
*/
- protected abstract HoodieTable doInitTable(HoodieTableMetaClient metaClient, Option instantTime, boolean initialMetadataTableIfNecessary);
+ protected void doInitTable(HoodieTableMetaClient metaClient, Option instantTime, boolean initialMetadataTableIfNecessary) {
+ Option ownerInstant = Option.empty();
+ if (instantTime.isPresent()) {
+ ownerInstant = Option.of(new HoodieInstant(true, CommitUtils.getCommitActionType(operationType, metaClient.getTableType()), instantTime.get()));
+ }
+ this.txnManager.beginTransaction(ownerInstant, Option.empty());
+ try {
+ tryUpgrade(metaClient, instantTime);
+ if (initialMetadataTableIfNecessary) {
+ initMetadataTable(instantTime);
+ }
+ } finally {
+ this.txnManager.endTransaction(ownerInstant);
+ }
+ }
+
+ /**
+ * Bootstrap the metadata table.
+ *
+ * @param instantTime current inflight instant time
+ */
+ protected void initMetadataTable(Option instantTime) {
+ // by default do nothing.
+ }
/**
* Instantiates and initializes instance of {@link HoodieTable}, performing crucial bootstrapping
@@ -1457,18 +1491,8 @@ protected final HoodieTable initTable(WriteOperationType operationType, Option ownerInstant = Option.empty();
- if (instantTime.isPresent()) {
- ownerInstant = Option.of(new HoodieInstant(true, CommitUtils.getCommitActionType(operationType, metaClient.getTableType()), instantTime.get()));
- }
- this.txnManager.beginTransaction(ownerInstant, Option.empty());
- try {
- tryUpgrade(metaClient, instantTime);
- table = doInitTable(metaClient, instantTime, initialMetadataTableIfNecessary);
- } finally {
- this.txnManager.endTransaction(ownerInstant);
- }
+ doInitTable(metaClient, instantTime, initialMetadataTableIfNecessary);
+ HoodieTable table = createTable(config, hadoopConf, metaClient);
// Validate table properties
metaClient.validateTableProperties(config.getProps());
@@ -1513,7 +1537,8 @@ protected void setWriteSchemaForDeletes(HoodieTableMetaClient metaClient) {
if (lastInstant.isPresent()) {
HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes(
activeTimeline.getInstantDetails(lastInstant.get()).get(), HoodieCommitMetadata.class);
- if (commitMetadata.getExtraMetadata().containsKey(SCHEMA_KEY)) {
+ String extraSchema = commitMetadata.getExtraMetadata().get(SCHEMA_KEY);
+ if (!StringUtils.isNullOrEmpty(extraSchema)) {
config.setSchema(commitMetadata.getExtraMetadata().get(SCHEMA_KEY));
} else {
throw new HoodieIOException("Latest commit does not have any schema in commit metadata");
@@ -1557,6 +1582,12 @@ private void setWriteTimer(HoodieTable table) {
}
}
+ /**
+ * Upgrades the hoodie table if need be when moving to a new Hudi version.
+ * This method is called within a lock. Try to avoid double locking from within this method.
+ * @param metaClient instance of {@link HoodieTableMetaClient} to use.
+ * @param instantTime instant time of interest if we have one.
+ */
protected void tryUpgrade(HoodieTableMetaClient metaClient, Option instantTime) {
UpgradeDowngrade upgradeDowngrade =
new UpgradeDowngrade(metaClient, config, context, upgradeDowngradeHelper);
@@ -1569,7 +1600,6 @@ protected void tryUpgrade(HoodieTableMetaClient metaClient, Option insta
if (!instantsToRollback.isEmpty()) {
Map> pendingRollbacks = getPendingRollbackInfos(metaClient);
instantsToRollback.forEach(entry -> pendingRollbacks.putIfAbsent(entry, Option.empty()));
-
rollbackFailedWrites(pendingRollbacks, true);
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java
index 2992f4abd4c9e..eaa0d8d167fbb 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java
@@ -43,7 +43,9 @@
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
+import org.apache.hudi.common.table.timeline.TimelineUtils;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
+import org.apache.hudi.common.util.ClusteringUtils;
import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.CompactionUtils;
import org.apache.hudi.common.util.FileIOUtils;
@@ -397,7 +399,7 @@ private Stream getCleanInstantsToArchive() {
}).flatMap(Collection::stream);
}
- private Stream getCommitInstantsToArchive() {
+ private Stream getCommitInstantsToArchive() throws IOException {
// TODO (na) : Add a way to return actions associated with a timeline and then merge/unify
// with logic above to avoid Stream.concat
HoodieTimeline commitTimeline = table.getCompletedCommitsTimeline();
@@ -430,6 +432,13 @@ private Stream getCommitInstantsToArchive() {
table.getActiveTimeline(), config.getInlineCompactDeltaCommitMax())
: Option.empty();
+ // The clustering commit instant can not be archived unless we ensure that the replaced files have been cleaned,
+ // without the replaced files metadata on the timeline, the fs view would expose duplicates for readers.
+ // Meanwhile, when inline or async clustering is enabled, we need to ensure that there is a commit in the active timeline
+ // to check whether the file slice generated in pending clustering after archive isn't committed.
+ Option oldestInstantToRetainForClustering =
+ ClusteringUtils.getOldestInstantToRetainForClustering(table.getActiveTimeline(), table.getMetaClient());
+
// Actually do the commits
Stream instantToArchiveStream = commitTimeline.getInstants()
.filter(s -> {
@@ -442,7 +451,7 @@ private Stream getCommitInstantsToArchive() {
return !(firstSavepoint.isPresent() && compareTimestamps(firstSavepoint.get().getTimestamp(), LESSER_THAN_OR_EQUALS, s.getTimestamp()));
}
}).filter(s -> {
- // Ensure commits >= oldest pending compaction commit is retained
+ // Ensure commits >= the oldest pending compaction/replace commit is retained
return oldestPendingCompactionAndReplaceInstant
.map(instant -> compareTimestamps(instant.getTimestamp(), GREATER_THAN, s.getTimestamp()))
.orElse(true);
@@ -459,6 +468,10 @@ private Stream getCommitInstantsToArchive() {
oldestInstantToRetainForCompaction.map(instantToRetain ->
compareTimestamps(s.getTimestamp(), LESSER_THAN, instantToRetain.getTimestamp()))
.orElse(true)
+ ).filter(s ->
+ oldestInstantToRetainForClustering.map(instantToRetain ->
+ HoodieTimeline.compareTimestamps(s.getTimestamp(), LESSER_THAN, instantToRetain.getTimestamp()))
+ .orElse(true)
);
return instantToArchiveStream.limit(commitTimeline.countInstants() - minInstantsToKeep);
} else {
@@ -466,7 +479,7 @@ private Stream getCommitInstantsToArchive() {
}
}
- private Stream getInstantsToArchive() {
+ private Stream getInstantsToArchive() throws IOException {
Stream instants = Stream.concat(getCleanInstantsToArchive(), getCommitInstantsToArchive());
if (config.isMetastoreEnabled()) {
return Stream.empty();
@@ -502,24 +515,27 @@ private Stream getInstantsToArchive() {
.setBasePath(HoodieTableMetadata.getDatasetBasePath(config.getBasePath()))
.setConf(metaClient.getHadoopConf())
.build();
- Option earliestActiveDatasetCommit = dataMetaClient.getActiveTimeline().firstInstant();
-
- if (config.shouldArchiveBeyondSavepoint()) {
- // There are chances that there could be holes in the timeline due to archival and savepoint interplay.
- // So, the first non-savepoint commit in the data timeline is considered as beginning of the active timeline.
- Option firstNonSavepointCommit = dataMetaClient.getActiveTimeline().getFirstNonSavepointCommit();
- if (firstNonSavepointCommit.isPresent()) {
- String firstNonSavepointCommitTime = firstNonSavepointCommit.get().getTimestamp();
- instants = instants.filter(instant ->
- compareTimestamps(instant.getTimestamp(), LESSER_THAN, firstNonSavepointCommitTime));
- }
- } else {
- // Do not archive the commits that live in data set active timeline.
- // This is required by metadata table, see HoodieTableMetadataUtil#processRollbackMetadata for details.
- if (earliestActiveDatasetCommit.isPresent()) {
- instants = instants.filter(instant ->
- compareTimestamps(instant.getTimestamp(), HoodieTimeline.LESSER_THAN, earliestActiveDatasetCommit.get().getTimestamp()));
- }
+ Option qualifiedEarliestInstant =
+ TimelineUtils.getEarliestInstantForMetadataArchival(
+ dataMetaClient.getActiveTimeline(), config.shouldArchiveBeyondSavepoint());
+
+ // Do not archive the instants after the earliest commit (COMMIT, DELTA_COMMIT, and
+ // REPLACE_COMMIT only, considering non-savepoint commit only if enabling archive
+ // beyond savepoint) and the earliest inflight instant (all actions).
+ // This is required by metadata table, see HoodieTableMetadataUtil#processRollbackMetadata
+ // for details.
+ // Note that we cannot blindly use the earliest instant of all actions, because CLEAN and
+ // ROLLBACK instants are archived separately apart from commits (check
+ // HoodieTimelineArchiver#getCleanInstantsToArchive). If we do so, a very old completed
+ // CLEAN or ROLLBACK instant can block the archive of metadata table timeline and causes
+ // the active timeline of metadata table to be extremely long, leading to performance issues
+ // for loading the timeline.
+ if (qualifiedEarliestInstant.isPresent()) {
+ instants = instants.filter(instant ->
+ compareTimestamps(
+ instant.getTimestamp(),
+ HoodieTimeline.LESSER_THAN,
+ qualifiedEarliestInstant.get().getTimestamp()));
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/DeletePartitionUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/DeletePartitionUtils.java
new file mode 100644
index 0000000000000..28f308a32f907
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/DeletePartitionUtils.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.client.utils;
+
+import org.apache.hudi.common.table.view.SyncableFileSystemView;
+import org.apache.hudi.exception.HoodieDeletePartitionException;
+import org.apache.hudi.table.HoodieTable;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * A utility class for helper functions when performing a delete partition operation.
+ */
+public class DeletePartitionUtils {
+
+ /**
+ * Check if there are any pending table service actions (requested + inflight) on a table affecting the partitions to
+ * be dropped.
+ *
+ * This check is to prevent a drop-partition from proceeding should a partition have a table service action in
+ * the pending stage. If this is allowed to happen, the filegroup that is an input for a table service action, might
+ * also be a candidate for being replaced. As such, when the table service action and drop-partition commits are
+ * committed, there will be two commits replacing a single filegroup.
+ *
+ * For example, a timeline might have an execution order as such:
+ * 000.replacecommit.requested (clustering filegroup_1 + filegroup_2 -> filegroup_3)
+ * 001.replacecommit.requested, 001.replacecommit.inflight, 0001.replacecommit (drop_partition to replace filegroup_1)
+ * 000.replacecommit.inflight (clustering is executed now)
+ * 000.replacecommit (clustering completed)
+ * For an execution order as shown above, 000.replacecommit and 001.replacecommit will both flag filegroup_1 to be replaced.
+ * This will cause downstream duplicate key errors when a map is being constructed.
+ *
+ * @param table Table to perform validation on
+ * @param partitionsToDrop List of partitions to drop
+ */
+ public static void checkForPendingTableServiceActions(HoodieTable table, List partitionsToDrop) {
+ List instantsOfOffendingPendingTableServiceAction = new ArrayList<>();
+ // ensure that there are no pending inflight clustering/compaction operations involving this partition
+ SyncableFileSystemView fileSystemView = (SyncableFileSystemView) table.getSliceView();
+
+ // separating the iteration of pending compaction operations from clustering as they return different stream types
+ fileSystemView.getPendingCompactionOperations()
+ .filter(op -> partitionsToDrop.contains(op.getRight().getPartitionPath()))
+ .forEach(op -> instantsOfOffendingPendingTableServiceAction.add(op.getLeft()));
+
+ fileSystemView.getFileGroupsInPendingClustering()
+ .filter(fgIdInstantPair -> partitionsToDrop.contains(fgIdInstantPair.getLeft().getPartitionPath()))
+ .forEach(x -> instantsOfOffendingPendingTableServiceAction.add(x.getRight().getTimestamp()));
+
+ if (instantsOfOffendingPendingTableServiceAction.size() > 0) {
+ throw new HoodieDeletePartitionException("Failed to drop partitions. "
+ + "Please ensure that there are no pending table service actions (clustering/compaction) for the partitions to be deleted: " + partitionsToDrop + ". "
+ + "Instant(s) of offending pending table service action: "
+ + instantsOfOffendingPendingTableServiceAction.stream().distinct().collect(Collectors.toList()));
+ }
+ }
+
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java
index 1180845a6ed8a..b3b6177f119bf 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java
@@ -153,7 +153,9 @@ public class HoodieClusteringConfig extends HoodieConfig {
+ "RECENT_DAYS: keep a continuous range of partitions, worked together with configs '" + DAYBASED_LOOKBACK_PARTITIONS.key() + "' and '"
+ PLAN_STRATEGY_SKIP_PARTITIONS_FROM_LATEST.key() + "."
+ "SELECTED_PARTITIONS: keep partitions that are in the specified range ['" + PARTITION_FILTER_BEGIN_PARTITION.key() + "', '"
- + PARTITION_FILTER_END_PARTITION.key() + "'].");
+ + PARTITION_FILTER_END_PARTITION.key() + "']."
+ + "DAY_ROLLING: clustering partitions on a rolling basis by the hour to avoid clustering all partitions each time, "
+ + "which strategy sorts the partitions asc and chooses the partition of which index is divided by 24 and the remainder is equal to the current hour.");
public static final ConfigProperty PLAN_STRATEGY_MAX_BYTES_PER_OUTPUT_FILEGROUP = ConfigProperty
.key(CLUSTERING_STRATEGY_PARAM_PREFIX + "max.bytes.per.group")
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
index 738e2d6b48d13..855d1e4936aa4 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
@@ -55,7 +55,7 @@ public class HoodieCreateHandle extends
private static final Logger LOG = LogManager.getLogger(HoodieCreateHandle.class);
- protected final HoodieFileWriter fileWriter;
+ protected HoodieFileWriter fileWriter;
protected final Path path;
protected long recordsWritten = 0;
protected long insertRecordsWritten = 0;
@@ -208,7 +208,10 @@ public List close() {
LOG.info("Closing the file " + writeStatus.getFileId() + " as we are done with all the records " + recordsWritten);
try {
- fileWriter.close();
+ if (fileWriter != null) {
+ fileWriter.close();
+ fileWriter = null;
+ }
setupWriteStatus();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index 9607080d1e74e..408443f0fb553 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -112,6 +112,8 @@ public abstract class HoodieBackedTableMetadataWriter implements HoodieTableMeta
private static final Logger LOG = LogManager.getLogger(HoodieBackedTableMetadataWriter.class);
+ public static final String METADATA_COMPACTION_TIME_SUFFIX = "001";
+
// Virtual keys support for metadata table. This Field is
// from the metadata payload schema.
private static final String RECORD_KEY_FIELD_NAME = HoodieMetadataPayload.KEY_FIELD_NAME;
@@ -242,6 +244,7 @@ private HoodieWriteConfig createMetadataWriteConfig(HoodieWriteConfig writeConfi
// Create the write config for the metadata table by borrowing options from the main write config.
HoodieWriteConfig.Builder builder = HoodieWriteConfig.newBuilder()
+ .withEngineType(writeConfig.getEngineType())
.withTimelineLayoutVersion(TimelineLayoutVersion.CURR_VERSION)
.withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder()
.withConsistencyCheckEnabled(writeConfig.getConsistencyGuardConfig().isConsistencyCheckEnabled())
@@ -265,7 +268,7 @@ private HoodieWriteConfig createMetadataWriteConfig(HoodieWriteConfig writeConfi
.withAutoClean(false)
.withCleanerParallelism(parallelism)
.withCleanerPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS)
- .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY)
+ .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.EAGER)
.retainCommits(writeConfig.getMetadataCleanerCommitsRetained())
.build())
// we will trigger archive manually, to ensure only regular writer invokes it
@@ -1016,22 +1019,38 @@ protected void compactIfNecessary(BaseHoodieWriteClient writeClient, String inst
// finish off any pending compactions if any from previous attempt.
writeClient.runAnyPendingCompactions();
- String latestDeltaCommitTime = metadataMetaClient.reloadActiveTimeline().getDeltaCommitTimeline().filterCompletedInstants().lastInstant()
- .get().getTimestamp();
+ String latestDeltaCommitTimeInMetadataTable = metadataMetaClient.reloadActiveTimeline()
+ .getDeltaCommitTimeline()
+ .filterCompletedInstants()
+ .lastInstant().orElseThrow(() -> new HoodieMetadataException("No completed deltacommit in metadata table"))
+ .getTimestamp();
+ // we need to find if there are any inflights in data table timeline before or equal to the latest delta commit in metadata table.
+ // Whenever you want to change this logic, please ensure all below scenarios are considered.
+ // a. There could be a chance that latest delta commit in MDT is committed in MDT, but failed in DT. And so findInstantsBeforeOrEquals() should be employed
+ // b. There could be DT inflights after latest delta commit in MDT and we are ok with it. bcoz, the contract is, latest compaction instant time in MDT represents
+ // any instants before that is already synced with metadata table.
+ // c. Do consider out of order commits. For eg, c4 from DT could complete before c3. and we can't trigger compaction in MDT with c4 as base instant time, until every
+ // instant before c4 is synced with metadata table.
List pendingInstants = dataMetaClient.reloadActiveTimeline().filterInflightsAndRequested()
- .findInstantsBefore(instantTime).getInstants().collect(Collectors.toList());
+ .findInstantsBeforeOrEquals(latestDeltaCommitTimeInMetadataTable).getInstants().collect(Collectors.toList());
if (!pendingInstants.isEmpty()) {
- LOG.info(String.format("Cannot compact metadata table as there are %d inflight instants before latest deltacommit %s: %s",
- pendingInstants.size(), latestDeltaCommitTime, Arrays.toString(pendingInstants.toArray())));
+ LOG.info(String.format(
+ "Cannot compact metadata table as there are %d inflight instants in data table before latest deltacommit in metadata table: %s. Inflight instants in data table: %s",
+ pendingInstants.size(), latestDeltaCommitTimeInMetadataTable, Arrays.toString(pendingInstants.toArray())));
return;
}
// Trigger compaction with suffixes based on the same instant time. This ensures that any future
// delta commits synced over will not have an instant time lesser than the last completed instant on the
// metadata table.
- final String compactionInstantTime = latestDeltaCommitTime + "001";
- if (writeClient.scheduleCompactionAtInstant(compactionInstantTime, Option.empty())) {
+ final String compactionInstantTime = latestDeltaCommitTimeInMetadataTable + METADATA_COMPACTION_TIME_SUFFIX;
+ // we need to avoid checking compaction w/ same instant again.
+ // lets say we trigger compaction after C5 in MDT and so compaction completes with C4001. but C5 crashed before completing in MDT.
+ // and again w/ C6, we will re-attempt compaction at which point latest delta commit is C4 in MDT.
+ // and so we try compaction w/ instant C4001. So, we can avoid compaction if we already have compaction w/ same instant time.
+ if (!metadataMetaClient.getActiveTimeline().filterCompletedInstants().containsInstant(compactionInstantTime)
+ && writeClient.scheduleCompactionAtInstant(compactionInstantTime, Option.empty())) {
writeClient.compact(compactionInstantTime);
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
index 35eb0edfbfc61..fac1db6d54526 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
@@ -470,7 +470,17 @@ public abstract Option scheduleCleaning(HoodieEngineContext c
*
* @return information on cleaned file slices
*/
- public abstract HoodieCleanMetadata clean(HoodieEngineContext context, String cleanInstantTime, boolean skipLocking);
+ @Deprecated
+ public HoodieCleanMetadata clean(HoodieEngineContext context, String cleanInstantTime, boolean skipLocking) {
+ return clean(context, cleanInstantTime);
+ }
+
+ /**
+ * Executes a new clean action.
+ *
+ * @return information on cleaned file slices
+ */
+ public abstract HoodieCleanMetadata clean(HoodieEngineContext context, String cleanInstantTime);
/**
* Schedule rollback for the instant time.
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
index 64e69b1d2a9bd..737388645b4b0 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
@@ -465,7 +465,24 @@ public Option getEarliestCommitToRetain() {
int hoursRetained = config.getCleanerHoursRetained();
if (config.getCleanerPolicy() == HoodieCleaningPolicy.KEEP_LATEST_COMMITS
&& commitTimeline.countInstants() > commitsRetained) {
- earliestCommitToRetain = commitTimeline.nthInstant(commitTimeline.countInstants() - commitsRetained); //15 instants total, 10 commits to retain, this gives 6th instant in the list
+ Option earliestPendingCommits = hoodieTable.getMetaClient()
+ .getActiveTimeline()
+ .getCommitsTimeline()
+ .filter(s -> !s.isCompleted()).firstInstant();
+ if (earliestPendingCommits.isPresent()) {
+ // Earliest commit to retain must not be later than the earliest pending commit
+ earliestCommitToRetain =
+ commitTimeline.nthInstant(commitTimeline.countInstants() - commitsRetained).map(nthInstant -> {
+ if (nthInstant.compareTo(earliestPendingCommits.get()) <= 0) {
+ return Option.of(nthInstant);
+ } else {
+ return commitTimeline.findInstantsBefore(earliestPendingCommits.get().getTimestamp()).lastInstant();
+ }
+ }).orElse(Option.empty());
+ } else {
+ earliestCommitToRetain = commitTimeline.nthInstant(commitTimeline.countInstants()
+ - commitsRetained); //15 instants total, 10 commits to retain, this gives 6th instant in the list
+ }
} else if (config.getCleanerPolicy() == HoodieCleaningPolicy.KEEP_LATEST_BY_HOURS) {
Instant instant = Instant.now();
ZonedDateTime currentDateTime = ZonedDateTime.ofInstant(instant, ZoneId.systemDefault());
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java
index 5d62ef390233f..8aafa6d28c425 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java
@@ -71,11 +71,18 @@ public Option generateClusteringPlan() {
HoodieTableMetaClient metaClient = getHoodieTable().getMetaClient();
LOG.info("Scheduling clustering for " + metaClient.getBasePath());
HoodieWriteConfig config = getWriteConfig();
- List partitionPaths = FSUtils.getAllPartitionPaths(getEngineContext(), config.getMetadataConfig(), metaClient.getBasePath());
- // get matched partitions if set
- partitionPaths = getMatchedPartitions(config, partitionPaths);
- // filter the partition paths if needed to reduce list status
+ String partitionSelected = config.getClusteringPartitionSelected();
+ List partitionPaths;
+
+ if (StringUtils.isNullOrEmpty(partitionSelected)) {
+ // get matched partitions if set
+ partitionPaths = getRegexPatternMatchedPartitions(config, FSUtils.getAllPartitionPaths(getEngineContext(), config.getMetadataConfig(), metaClient.getBasePath()));
+ // filter the partition paths if needed to reduce list status
+ } else {
+ partitionPaths = Arrays.asList(partitionSelected.split(","));
+ }
+
partitionPaths = filterPartitionPaths(partitionPaths);
if (partitionPaths.isEmpty()) {
@@ -114,15 +121,6 @@ public Option generateClusteringPlan() {
.build());
}
- public List getMatchedPartitions(HoodieWriteConfig config, List partitionPaths) {
- String partitionSelected = config.getClusteringPartitionSelected();
- if (!StringUtils.isNullOrEmpty(partitionSelected)) {
- return Arrays.asList(partitionSelected.split(","));
- } else {
- return getRegexPatternMatchedPartitions(config, partitionPaths);
- }
- }
-
public List getRegexPatternMatchedPartitions(HoodieWriteConfig config, List partitionPaths) {
String pattern = config.getClusteringPartitionFilterRegexPattern();
if (!StringUtils.isNullOrEmpty(pattern)) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
index 4add51886fe3a..e8acd54c5aa49 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
@@ -37,6 +37,7 @@
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieRollbackException;
+import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.BaseActionExecutor;
import org.apache.hudi.table.marker.WriteMarkersFactory;
@@ -156,12 +157,13 @@ private void validateRollbackCommitSequence() {
// since with LAZY rollback we support parallel writing which can allow a new inflight while rollback is ongoing
// Remove this once we support LAZY rollback of failed writes by default as parallel writing becomes the default
// writer mode.
- if (config.getFailedWritesCleanPolicy().isEager()) {
+ if (config.getFailedWritesCleanPolicy().isEager() && !HoodieTableMetadata.isMetadataTable(config.getBasePath())) {
final String instantTimeToRollback = instantToRollback.getTimestamp();
HoodieTimeline commitTimeline = table.getCompletedCommitsTimeline();
HoodieTimeline inflightAndRequestedCommitTimeline = table.getPendingCommitTimeline();
// Make sure only the last n commits are being rolled back
// If there is a commit in-between or after that is not rolled back, then abort
+ // this condition may not hold good for metadata table. since the order of commits applied to MDT is data table commits and the ordering could be different.
if ((instantTimeToRollback != null) && !commitTimeline.empty()
&& !commitTimeline.findInstantsAfter(instantTimeToRollback, Integer.MAX_VALUE).empty()) {
// check if remnants are from a previous LAZY rollback config, if yes, let out of order rollback continue
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/savepoint/SavepointActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/savepoint/SavepointActionExecutor.java
index 7f408c1b8d24a..310ebba3c14c7 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/savepoint/SavepointActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/savepoint/SavepointActionExecutor.java
@@ -27,6 +27,7 @@
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
+import org.apache.hudi.common.table.view.FileSystemViewStorageType;
import org.apache.hudi.common.table.view.TableFileSystemView;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
@@ -85,15 +86,33 @@ public HoodieSavepointMetadata execute() {
"Could not savepoint commit " + instantTime + " as this is beyond the lookup window " + lastCommitRetained);
context.setJobStatus(this.getClass().getSimpleName(), "Collecting latest files for savepoint " + instantTime + " " + table.getConfig().getTableName());
- List partitions = FSUtils.getAllPartitionPaths(context, config.getMetadataConfig(), table.getMetaClient().getBasePath());
- Map> latestFilesMap = context.mapToPair(partitions, partitionPath -> {
- // Scan all partitions files with this commit time
- LOG.info("Collecting latest files in partition path " + partitionPath);
- TableFileSystemView.BaseFileOnlyView view = table.getBaseFileOnlyView();
- List latestFiles = view.getLatestBaseFilesBeforeOrOn(partitionPath, instantTime)
- .map(HoodieBaseFile::getFileName).collect(Collectors.toList());
- return new ImmutablePair<>(partitionPath, latestFiles);
- }, null);
+ TableFileSystemView.BaseFileOnlyView view = table.getBaseFileOnlyView();
+
+ Map> latestFilesMap;
+ // NOTE: for performance, we have to use different logic here for listing the latest files
+ // before or on the given instant:
+ // (1) using metadata-table-based file listing: instead of parallelizing the partition
+ // listing which incurs unnecessary metadata table reads, we directly read the metadata
+ // table once in a batch manner through the timeline server;
+ // (2) using direct file system listing: we parallelize the partition listing so that
+ // each partition can be listed on the file system concurrently through Spark.
+ // Note that
+ if (shouldUseBatchLookup(config)) {
+ latestFilesMap = view.getAllLatestBaseFilesBeforeOrOn(instantTime).entrySet().stream()
+ .collect(Collectors.toMap(
+ Map.Entry::getKey,
+ entry -> entry.getValue().map(HoodieBaseFile::getFileName).collect(Collectors.toList())));
+ } else {
+ List partitions = FSUtils.getAllPartitionPaths(context, config.getMetadataConfig(), table.getMetaClient().getBasePath());
+ latestFilesMap = context.mapToPair(partitions, partitionPath -> {
+ // Scan all partitions files with this commit time
+ LOG.info("Collecting latest files in partition path " + partitionPath);
+ List latestFiles = view.getLatestBaseFilesBeforeOrOn(partitionPath, instantTime)
+ .map(HoodieBaseFile::getFileName).collect(Collectors.toList());
+ return new ImmutablePair<>(partitionPath, latestFiles);
+ }, null);
+ }
+
HoodieSavepointMetadata metadata = TimelineMetadataUtils.convertSavepointMetadata(user, comment, latestFilesMap);
// Nothing to save in the savepoint
table.getActiveTimeline().createNewInstant(
@@ -107,4 +126,22 @@ public HoodieSavepointMetadata execute() {
throw new HoodieSavepointException("Failed to savepoint " + instantTime, e);
}
}
+
+ /**
+ * Whether to use batch lookup for listing the latest base files in metadata table.
+ *
+ * Note that metadata table has to be enabled, and the storage type of the file system view
+ * cannot be EMBEDDED_KV_STORE or SPILLABLE_DISK (these two types are not integrated with
+ * metadata table, see HUDI-5612).
+ *
+ * @param config Write configs.
+ * @return {@code true} if using batch lookup; {@code false} otherwise.
+ */
+ private boolean shouldUseBatchLookup(HoodieWriteConfig config) {
+ FileSystemViewStorageType storageType =
+ config.getClientSpecifiedViewStorageConfig().getStorageType();
+ return config.getMetadataConfig().enabled()
+ && !FileSystemViewStorageType.EMBEDDED_KV_STORE.equals(storageType)
+ && !FileSystemViewStorageType.SPILLABLE_DISK.equals(storageType);
+ }
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkersFactory.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkersFactory.java
index dfd55f2958125..625ed18227853 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkersFactory.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkersFactory.java
@@ -24,7 +24,6 @@
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.table.HoodieTable;
-import com.esotericsoftware.minlog.Log;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -47,14 +46,14 @@ public static WriteMarkers get(MarkerType markerType, HoodieTable table, String
return new DirectWriteMarkers(table, instantTime);
case TIMELINE_SERVER_BASED:
if (!table.getConfig().isEmbeddedTimelineServerEnabled()) {
- Log.warn("Timeline-server-based markers are configured as the marker type "
+ LOG.warn("Timeline-server-based markers are configured as the marker type "
+ "but embedded timeline server is not enabled. Falling back to direct markers.");
return new DirectWriteMarkers(table, instantTime);
}
String basePath = table.getMetaClient().getBasePath();
if (StorageSchemes.HDFS.getScheme().equals(
FSUtils.getFs(basePath, table.getContext().getHadoopConf().newCopy()).getScheme())) {
- Log.warn("Timeline-server-based markers are not supported for HDFS: "
+ LOG.warn("Timeline-server-based markers are not supported for HDFS: "
+ "base path " + basePath + ". Falling back to direct markers.");
return new DirectWriteMarkers(table, instantTime);
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestDeletePartitionUtils.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestDeletePartitionUtils.java
new file mode 100644
index 0000000000000..929540b8dcaf7
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestDeletePartitionUtils.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.client.utils;
+
+import org.apache.hudi.common.model.CompactionOperation;
+import org.apache.hudi.common.model.HoodieFileGroupId;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.view.SyncableFileSystemView;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.exception.HoodieDeletePartitionException;
+import org.apache.hudi.table.HoodieTable;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class TestDeletePartitionUtils {
+
+ private static final String PARTITION_IN_PENDING_SERVICE_ACTION = "partition_with_pending_table_service_action";
+ private static final String HARDCODED_INSTANT_TIME = "0";
+
+ private final HoodieTable table = Mockito.mock(HoodieTable.class);
+
+ private final SyncableFileSystemView fileSystemView = Mockito.mock(SyncableFileSystemView.class);
+
+ public static Stream generateTruthValues() {
+ int noOfVariables = 2;
+ int noOfRows = 1 << noOfVariables;
+ Object[][] truthValues = new Object[noOfRows][noOfVariables];
+ for (int i = 0; i < noOfRows; i++) {
+ for (int j = noOfVariables - 1; j >= 0; j--) {
+ boolean out = (i / (1 << j)) % 2 != 0;
+ truthValues[i][j] = out;
+ }
+ }
+ return Stream.of(truthValues).map(Arguments::of);
+ }
+
+ @ParameterizedTest
+ @MethodSource("generateTruthValues")
+ public void testDeletePartitionUtils(
+ boolean hasPendingCompactionOperations,
+ boolean hasFileGroupsInPendingClustering) {
+ System.out.printf("hasPendingCompactionOperations: %s, hasFileGroupsInPendingClustering: %s%n",
+ hasPendingCompactionOperations, hasFileGroupsInPendingClustering);
+ Mockito.when(table.getSliceView()).thenReturn(fileSystemView);
+ Mockito.when(fileSystemView.getPendingCompactionOperations()).thenReturn(createPendingCompactionOperations(hasPendingCompactionOperations));
+ Mockito.when(fileSystemView.getFileGroupsInPendingClustering()).thenReturn(createFileGroupsInPendingClustering(hasFileGroupsInPendingClustering));
+
+ boolean shouldThrowException = hasPendingCompactionOperations || hasFileGroupsInPendingClustering;
+
+ if (shouldThrowException) {
+ assertThrows(HoodieDeletePartitionException.class,
+ () -> DeletePartitionUtils.checkForPendingTableServiceActions(table,
+ Collections.singletonList(PARTITION_IN_PENDING_SERVICE_ACTION)));
+ } else {
+ assertDoesNotThrow(() -> DeletePartitionUtils.checkForPendingTableServiceActions(table,
+ Collections.singletonList(PARTITION_IN_PENDING_SERVICE_ACTION)));
+ }
+ }
+
+ private static Stream> createPendingCompactionOperations(boolean hasPendingCompactionOperations) {
+ return Stream.of(Pair.of(HARDCODED_INSTANT_TIME, getCompactionOperation(hasPendingCompactionOperations)));
+ }
+
+ private static CompactionOperation getCompactionOperation(boolean hasPendingJobInPartition) {
+ return new CompactionOperation(
+ "fileId", getPartitionName(hasPendingJobInPartition), HARDCODED_INSTANT_TIME, Option.empty(),
+ new ArrayList<>(), Option.empty(), Option.empty(), new HashMap<>());
+ }
+
+ private static Stream> createFileGroupsInPendingClustering(boolean hasFileGroupsInPendingClustering) {
+ HoodieFileGroupId hoodieFileGroupId = new HoodieFileGroupId(getPartitionName(hasFileGroupsInPendingClustering), "fileId");
+ HoodieInstant hoodieInstant = new HoodieInstant(true, "replacecommit", HARDCODED_INSTANT_TIME);
+ return Stream.of(Pair.of(hoodieFileGroupId, hoodieInstant));
+ }
+
+ private static String getPartitionName(boolean hasPendingTableServiceAction) {
+ return hasPendingTableServiceAction ? PARTITION_IN_PENDING_SERVICE_ACTION : "unaffected_partition";
+ }
+
+}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestPartitionAwareClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestPartitionAwareClusteringPlanStrategy.java
index 440bc95615391..a053a9611050c 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestPartitionAwareClusteringPlanStrategy.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestPartitionAwareClusteringPlanStrategy.java
@@ -71,7 +71,7 @@ public void testFilterPartitionPaths() {
fakeTimeBasedPartitionsPath.add("20210719");
fakeTimeBasedPartitionsPath.add("20210721");
- List list = strategyTestRegexPattern.getMatchedPartitions(hoodieWriteConfig, fakeTimeBasedPartitionsPath);
+ List list = strategyTestRegexPattern.getRegexPatternMatchedPartitions(hoodieWriteConfig, fakeTimeBasedPartitionsPath);
assertEquals(2, list.size());
assertTrue(list.contains("20210721"));
assertTrue(list.contains("20210723"));
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java
index 551b412ccbc6c..9a9037c17cc07 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java
@@ -69,6 +69,7 @@
import com.codahale.metrics.Timer;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -100,7 +101,7 @@ public class HoodieFlinkWriteClient extends
* FileID to write handle mapping in order to record the write handles for each file group,
* so that we can append the mini-batch data buffer incrementally.
*/
- private final Map> bucketToHandles;
+ private final Map bucketToHandles;
public HoodieFlinkWriteClient(HoodieEngineContext context, HoodieWriteConfig writeConfig) {
super(context, writeConfig, FlinkUpgradeDowngradeHelper.getInstance());
@@ -126,6 +127,11 @@ protected HoodieTable createTable(HoodieWriteConfig config, Configuration hadoop
return HoodieFlinkTable.create(config, (HoodieFlinkEngineContext) context);
}
+ @Override
+ protected HoodieTable createTable(HoodieWriteConfig config, Configuration hadoopConf, HoodieTableMetaClient metaClient) {
+ return HoodieFlinkTable.create(config, (HoodieFlinkEngineContext) context, metaClient);
+ }
+
@Override
public List> filterExists(List> hoodieRecords) {
// Create a Hoodie table which encapsulated the commits and files visible
@@ -147,9 +153,10 @@ public List upsert(List> records, String instantTim
initTable(WriteOperationType.UPSERT, Option.ofNullable(instantTime));
table.validateUpsertSchema();
preWrite(instantTime, WriteOperationType.UPSERT, table.getMetaClient());
- final HoodieWriteHandle writeHandle = getOrCreateWriteHandle(records.get(0), getConfig(),
- instantTime, table, records.listIterator());
- HoodieWriteMetadata> result = ((HoodieFlinkTable) table).upsert(context, writeHandle, instantTime, records);
+ HoodieWriteMetadata> result;
+ try (AutoCloseableWriteHandle closeableHandle = new AutoCloseableWriteHandle(records, instantTime, table)) {
+ result = ((HoodieFlinkTable) table).upsert(context, closeableHandle.getWriteHandle(), instantTime, records);
+ }
if (result.getIndexLookupDuration().isPresent()) {
metrics.updateIndexMetrics(LOOKUP_STR, result.getIndexLookupDuration().get().toMillis());
}
@@ -166,9 +173,10 @@ public List upsertPreppedRecords(List> preppedRecor
Map>> preppedRecordsByFileId = preppedRecords.stream().parallel()
.collect(Collectors.groupingBy(r -> r.getCurrentLocation().getFileId()));
return preppedRecordsByFileId.values().stream().parallel().map(records -> {
- final HoodieWriteHandle writeHandle = getOrCreateWriteHandle(records.get(0), getConfig(),
- instantTime, table, records.listIterator());
- HoodieWriteMetadata> result = ((HoodieFlinkTable) table).upsertPrepped(context, writeHandle, instantTime, records);
+ HoodieWriteMetadata> result;
+ try (AutoCloseableWriteHandle closeableHandle = new AutoCloseableWriteHandle(records, instantTime, table)) {
+ result = ((HoodieFlinkTable) table).upsertPrepped(context, closeableHandle.getWriteHandle(), instantTime, records);
+ }
return postWrite(result, instantTime, table);
}).flatMap(Collection::stream).collect(Collectors.toList());
}
@@ -180,9 +188,10 @@ public List insert(List> records, String instantTim
table.validateInsertSchema();
preWrite(instantTime, WriteOperationType.INSERT, table.getMetaClient());
// create the write handle if not exists
- final HoodieWriteHandle writeHandle = getOrCreateWriteHandle(records.get(0), getConfig(),
- instantTime, table, records.listIterator());
- HoodieWriteMetadata> result = ((HoodieFlinkTable) table).insert(context, writeHandle, instantTime, records);
+ HoodieWriteMetadata> result;
+ try (AutoCloseableWriteHandle closeableHandle = new AutoCloseableWriteHandle(records, instantTime, table)) {
+ result = ((HoodieFlinkTable) table).insert(context, closeableHandle.getWriteHandle(), instantTime, records);
+ }
if (result.getIndexLookupDuration().isPresent()) {
metrics.updateIndexMetrics(LOOKUP_STR, result.getIndexLookupDuration().get().toMillis());
}
@@ -203,9 +212,10 @@ public List insertOverwrite(
table.validateInsertSchema();
preWrite(instantTime, WriteOperationType.INSERT_OVERWRITE, table.getMetaClient());
// create the write handle if not exists
- final HoodieWriteHandle writeHandle = getOrCreateWriteHandle(records.get(0), getConfig(),
- instantTime, table, records.listIterator());
- HoodieWriteMetadata result = ((HoodieFlinkTable) table).insertOverwrite(context, writeHandle, instantTime, records);
+ HoodieWriteMetadata> result;
+ try (AutoCloseableWriteHandle closeableHandle = new AutoCloseableWriteHandle(records, instantTime, table)) {
+ result = ((HoodieFlinkTable) table).insertOverwrite(context, closeableHandle.getWriteHandle(), instantTime, records);
+ }
return postWrite(result, instantTime, table);
}
@@ -222,9 +232,10 @@ public List insertOverwriteTable(
table.validateInsertSchema();
preWrite(instantTime, WriteOperationType.INSERT_OVERWRITE_TABLE, table.getMetaClient());
// create the write handle if not exists
- final HoodieWriteHandle writeHandle = getOrCreateWriteHandle(records.get(0), getConfig(),
- instantTime, table, records.listIterator());
- HoodieWriteMetadata result = ((HoodieFlinkTable) table).insertOverwriteTable(context, writeHandle, instantTime, records);
+ HoodieWriteMetadata> result;
+ try (AutoCloseableWriteHandle closeableHandle = new AutoCloseableWriteHandle(records, instantTime, table)) {
+ result = ((HoodieFlinkTable) table).insertOverwriteTable(context, closeableHandle.getWriteHandle(), instantTime, records);
+ }
return postWrite(result, instantTime, table);
}
@@ -291,10 +302,14 @@ public void initMetadataTable() {
HoodieFlinkTable table = getHoodieTable();
if (config.isMetadataTableEnabled()) {
// initialize the metadata table path
- try (HoodieBackedTableMetadataWriter metadataWriter = initMetadataWriter()) {
- // do nothing
+ // guard the metadata writer with concurrent lock
+ try {
+ this.txnManager.getLockManager().lock();
+ initMetadataWriter().close();
} catch (Exception e) {
throw new HoodieException("Failed to initialize metadata table", e);
+ } finally {
+ this.txnManager.getLockManager().unlock();
}
// clean the obsolete index stats
table.deleteMetadataIndexIfNecessary();
@@ -360,18 +375,21 @@ protected List postWrite(HoodieWriteMetadata> res
protected void postCommit(HoodieTable table,
HoodieCommitMetadata metadata,
String instantTime,
- Option> extraMetadata,
- boolean acquireLockForArchival) {
+ Option> extraMetadata) {
try {
// Delete the marker directory for the instant.
WriteMarkersFactory.get(config.getMarkersType(), createTable(config, hadoopConf), instantTime)
.quietDeleteMarkerDir(context, config.getMarkersDeleteParallelism());
- autoArchiveOnCommit(table, acquireLockForArchival);
} finally {
this.heartbeatClient.stop(instantTime);
}
}
+ @Override
+ protected void mayBeCleanAndArchive(HoodieTable table) {
+ autoArchiveOnCommit(table);
+ }
+
@Override
public void commitCompaction(
String compactionInstantTime,
@@ -478,16 +496,13 @@ private void completeClustering(
}
@Override
- protected HoodieTable doInitTable(HoodieTableMetaClient metaClient, Option instantTime, boolean initialMetadataTableIfNecessary) {
- // Create a Hoodie table which encapsulated the commits and files visible
- return getHoodieTable();
- }
-
- @Override
- protected void tryUpgrade(HoodieTableMetaClient metaClient, Option instantTime) {
+ protected void doInitTable(HoodieTableMetaClient metaClient, Option instantTime, boolean initialMetadataTableIfNecessary) {
// do nothing.
+
// flink executes the upgrade/downgrade once when initializing the first instant on start up,
// no need to execute the upgrade/downgrade on each write in streaming.
+
+ // flink performs metadata table bootstrap on the coordinator when it starts up.
}
public void completeTableService(
@@ -526,15 +541,10 @@ public void cleanHandles() {
this.bucketToHandles.clear();
}
- /**
- * Clean the write handles within a checkpoint interval, this operation
- * would close the underneath file handles, if any error happens, clean the
- * corrupted data file.
- */
- public void cleanHandlesGracefully() {
- this.bucketToHandles.values()
- .forEach(handle -> ((MiniBatchHandle) handle).closeGracefully());
- this.bucketToHandles.clear();
+ @Override
+ public void close() {
+ super.close();
+ cleanHandles();
}
/**
@@ -558,24 +568,20 @@ public void cleanHandlesGracefully() {
final String partitionPath = record.getPartitionPath();
final boolean insertClustering = config.allowDuplicateInserts();
- if (bucketToHandles.containsKey(fileID)) {
- MiniBatchHandle lastHandle = (MiniBatchHandle) bucketToHandles.get(fileID);
- if (lastHandle.shouldReplace()) {
- HoodieWriteHandle writeHandle = insertClustering
- ? new FlinkConcatAndReplaceHandle<>(config, instantTime, table, recordItr, partitionPath, fileID,
- table.getTaskContextSupplier(), lastHandle.getWritePath())
- : new FlinkMergeAndReplaceHandle<>(config, instantTime, table, recordItr, partitionPath, fileID,
- table.getTaskContextSupplier(), lastHandle.getWritePath());
- this.bucketToHandles.put(fileID, writeHandle); // override with new replace handle
- return writeHandle;
- }
- }
-
final boolean isDelta = table.getMetaClient().getTableType().equals(HoodieTableType.MERGE_ON_READ);
- final HoodieWriteHandle writeHandle;
if (isDelta) {
- writeHandle = new FlinkAppendHandle<>(config, instantTime, table, partitionPath, fileID, recordItr,
+ return new FlinkAppendHandle<>(config, instantTime, table, partitionPath, fileID, recordItr,
table.getTaskContextSupplier());
+ }
+
+ final HoodieWriteHandle writeHandle;
+ if (bucketToHandles.containsKey(fileID)) {
+ Path lastPath = bucketToHandles.get(fileID);
+ writeHandle = insertClustering
+ ? new FlinkConcatAndReplaceHandle<>(config, instantTime, table, recordItr, partitionPath, fileID,
+ table.getTaskContextSupplier(), lastPath)
+ : new FlinkMergeAndReplaceHandle<>(config, instantTime, table, recordItr, partitionPath, fileID,
+ table.getTaskContextSupplier(), lastPath);
} else if (loc.getInstantTime().equals("I")) {
writeHandle = new FlinkCreateHandle<>(config, instantTime, table, partitionPath,
fileID, table.getTaskContextSupplier());
@@ -586,7 +592,7 @@ public void cleanHandlesGracefully() {
: new FlinkMergeHandle<>(config, instantTime, table, recordItr, partitionPath,
fileID, table.getTaskContextSupplier());
}
- this.bucketToHandles.put(fileID, writeHandle);
+ this.bucketToHandles.put(fileID, ((MiniBatchHandle) writeHandle).getWritePath());
return writeHandle;
}
@@ -627,4 +633,25 @@ private List getAllExistingFileIds(HoodieFlinkTable table, String par
// because new commit is not complete. it is safe to mark all existing file Ids as old files
return table.getSliceView().getLatestFileSlices(partitionPath).map(FileSlice::getFileId).distinct().collect(Collectors.toList());
}
+
+ private final class AutoCloseableWriteHandle implements AutoCloseable {
+ private final HoodieWriteHandle writeHandle;
+
+ AutoCloseableWriteHandle(
+ List> records,
+ String instantTime,
+ HoodieTable>, List, List> table
+ ) {
+ this.writeHandle = getOrCreateWriteHandle(records.get(0), getConfig(), instantTime, table, records.listIterator());
+ }
+
+ HoodieWriteHandle getWriteHandle() {
+ return writeHandle;
+ }
+
+ @Override
+ public void close() {
+ ((MiniBatchHandle) writeHandle).closeGracefully();
+ }
+ }
}
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/HoodieFlinkCopyOnWriteTable.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/HoodieFlinkCopyOnWriteTable.java
index 543751a041078..6a0262321a6f0 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/HoodieFlinkCopyOnWriteTable.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/HoodieFlinkCopyOnWriteTable.java
@@ -319,7 +319,7 @@ public Option scheduleRollback(HoodieEngineContext context,
}
@Override
- public HoodieCleanMetadata clean(HoodieEngineContext context, String cleanInstantTime, boolean skipLocking) {
+ public HoodieCleanMetadata clean(HoodieEngineContext context, String cleanInstantTime) {
return new CleanActionExecutor(context, config, this, cleanInstantTime).execute();
}
diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/HoodieJavaWriteClient.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/HoodieJavaWriteClient.java
index b6951bc6b7874..83bbafc4e84fe 100644
--- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/HoodieJavaWriteClient.java
+++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/HoodieJavaWriteClient.java
@@ -92,6 +92,11 @@ protected HoodieTable createTable(HoodieWriteConfig config, Configuration hadoop
return HoodieJavaTable.create(config, context);
}
+ @Override
+ protected HoodieTable createTable(HoodieWriteConfig config, Configuration hadoopConf, HoodieTableMetaClient metaClient) {
+ return HoodieJavaTable.create(config, context, metaClient);
+ }
+
@Override
public List upsert(List> records,
String instantTime) {
@@ -197,7 +202,8 @@ protected List postWrite(HoodieWriteMetadata> res
result.getWriteStats().get().size());
}
- postCommit(hoodieTable, result.getCommitMetadata().get(), instantTime, Option.empty(), true);
+ postCommit(hoodieTable, result.getCommitMetadata().get(), instantTime, Option.empty());
+ mayBeCleanAndArchive(hoodieTable);
emitCommitMetrics(instantTime, result.getCommitMetadata().get(), hoodieTable.getMetaClient().getCommitActionType());
}
@@ -228,13 +234,4 @@ protected HoodieWriteMetadata> compact(String compactionInstan
public HoodieWriteMetadata> cluster(final String clusteringInstant, final boolean shouldComplete) {
throw new HoodieNotSupportedException("Cluster is not supported in HoodieJavaClient");
}
-
- @Override
- protected HoodieTable doInitTable(HoodieTableMetaClient metaClient, Option instantTime, boolean initialMetadataTableIfNecessary) {
- // new JavaUpgradeDowngrade(metaClient, config, context).run(metaClient, HoodieTableVersion.current(), config, context, instantTime);
-
- // Create a Hoodie table which encapsulated the commits and files visible
- return HoodieJavaTable.create(config, (HoodieJavaEngineContext) context, metaClient);
- }
-
}
diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaCopyOnWriteTable.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaCopyOnWriteTable.java
index 342c018e5a269..a1646dae47356 100644
--- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaCopyOnWriteTable.java
+++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaCopyOnWriteTable.java
@@ -217,7 +217,7 @@ public Option scheduleCleaning(HoodieEngineContext context, S
@Override
public HoodieCleanMetadata clean(HoodieEngineContext context,
- String cleanInstantTime, boolean skipLocking) {
+ String cleanInstantTime) {
return new CleanActionExecutor(context, config, this, cleanInstantTime).execute();
}
diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaTable.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaTable.java
index 3c878cbc14cf8..960b6baef8eb0 100644
--- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaTable.java
+++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaTable.java
@@ -19,7 +19,6 @@
package org.apache.hudi.table;
import org.apache.hudi.client.WriteStatus;
-import org.apache.hudi.client.common.HoodieJavaEngineContext;
import org.apache.hudi.common.data.HoodieData;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieKey;
@@ -47,11 +46,11 @@ public static HoodieJavaTable create(HoodieWr
HoodieTableMetaClient.builder().setConf(context.getHadoopConf().get()).setBasePath(config.getBasePath())
.setLoadActiveTimelineOnLoad(true).setConsistencyGuardConfig(config.getConsistencyGuardConfig())
.setLayoutVersion(Option.of(new TimelineLayoutVersion(config.getTimelineLayoutVersion()))).build();
- return HoodieJavaTable.create(config, (HoodieJavaEngineContext) context, metaClient);
+ return HoodieJavaTable.create(config, context, metaClient);
}
public static HoodieJavaTable create(HoodieWriteConfig config,
- HoodieJavaEngineContext context,
+ HoodieEngineContext context,
HoodieTableMetaClient metaClient) {
switch (metaClient.getTableType()) {
case COPY_ON_WRITE:
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDWriteClient.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDWriteClient.java
index 1f9fcf3ef9c29..1b1754501c66f 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDWriteClient.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDWriteClient.java
@@ -131,6 +131,11 @@ protected HoodieTable createTable(HoodieWriteConfig config, Configuration hadoop
return HoodieSparkTable.create(config, context);
}
+ @Override
+ protected HoodieTable createTable(HoodieWriteConfig config, Configuration hadoopConf, HoodieTableMetaClient metaClient) {
+ return HoodieSparkTable.create(config, context, metaClient);
+ }
+
@Override
public JavaRDD> filterExists(JavaRDD> hoodieRecords) {
// Create a Hoodie table which encapsulated the commits and files visible
@@ -287,7 +292,8 @@ protected JavaRDD postWrite(HoodieWriteMetadata instantTime, boolean initialMetadataTableIfNecessary) {
- if (initialMetadataTableIfNecessary) {
- // Initialize Metadata Table to make sure it's bootstrapped _before_ the operation,
- // if it didn't exist before
- // See https://issues.apache.org/jira/browse/HUDI-3343 for more details
- initializeMetadataTable(instantTime);
- }
-
- // Create a Hoodie table which encapsulated the commits and files visible
- return HoodieSparkTable.create(config, (HoodieSparkEngineContext) context, metaClient);
+ protected void initMetadataTable(Option instantTime) {
+ // Initialize Metadata Table to make sure it's bootstrapped _before_ the operation,
+ // if it didn't exist before
+ // See https://issues.apache.org/jira/browse/HUDI-3343 for more details
+ initializeMetadataTable(instantTime);
}
/**
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java
index 272d3d479852a..81526c25bccde 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java
@@ -27,6 +27,7 @@
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.model.WriteOperationType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.CommitUtils;
@@ -132,7 +133,11 @@ protected void commit(String instantTime, Map preppedRecords = prepRecords(partitionRecordsMap);
JavaRDD preppedRecordRDD = HoodieJavaRDD.getJavaRDD(preppedRecords);
- try (SparkRDDWriteClient writeClient = new SparkRDDWriteClient(engineContext, metadataWriteConfig, true)) {
+ try (SparkRDDWriteClient writeClient = new SparkRDDWriteClient(engineContext, metadataWriteConfig)) {
+ // rollback partially failed writes if any.
+ if (writeClient.rollbackFailedWrites()) {
+ metadataMetaClient = HoodieTableMetaClient.reload(metadataMetaClient);
+ }
if (canTriggerTableService) {
// trigger compaction before doing the delta commit. this is to ensure, if this delta commit succeeds in metadata table, but failed in data table,
// we would have compacted metadata table and so could have included uncommitted data which will never be ignored while reading from metadata
@@ -162,7 +167,7 @@ protected void commit(String instantTime, Map statuses = writeClient.upsertPreppedRecords(preppedRecordRDD, instantTime).collect();
statuses.forEach(writeStatus -> {
if (writeStatus.hasErrors()) {
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkCopyOnWriteTable.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkCopyOnWriteTable.java
index 743aff51a1254..fc159f5ea8e38 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkCopyOnWriteTable.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkCopyOnWriteTable.java
@@ -265,8 +265,8 @@ public Iterator> handleInsert(
}
@Override
- public HoodieCleanMetadata clean(HoodieEngineContext context, String cleanInstantTime, boolean skipLocking) {
- return new CleanActionExecutor<>(context, config, this, cleanInstantTime, skipLocking).execute();
+ public HoodieCleanMetadata clean(HoodieEngineContext context, String cleanInstantTime) {
+ return new CleanActionExecutor<>(context, config, this, cleanInstantTime, false).execute();
}
@Override
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkTable.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkTable.java
index 66d51c91283f3..3719ed742bcb3 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkTable.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkTable.java
@@ -19,7 +19,6 @@
package org.apache.hudi.table;
import org.apache.hudi.client.WriteStatus;
-import org.apache.hudi.client.common.HoodieSparkEngineContext;
import org.apache.hudi.common.data.HoodieData;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieKey;
@@ -60,11 +59,11 @@ public static HoodieSparkTable create(HoodieW
.setLayoutVersion(Option.of(new TimelineLayoutVersion(config.getTimelineLayoutVersion())))
.setFileSystemRetryConfig(config.getFileSystemRetryConfig())
.setProperties(config.getProps()).build();
- return HoodieSparkTable.create(config, (HoodieSparkEngineContext) context, metaClient);
+ return HoodieSparkTable.create(config, context, metaClient);
}
public static HoodieSparkTable create(HoodieWriteConfig config,
- HoodieSparkEngineContext context,
+ HoodieEngineContext context,
HoodieTableMetaClient metaClient) {
HoodieSparkTable hoodieSparkTable;
switch (metaClient.getTableType()) {
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java
index f8e4b31ff687e..06127b8aa4fbf 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java
@@ -379,8 +379,7 @@ protected HoodieMergeHandle getUpdateHandle(String partitionPath, String fileId,
}
@Override
- public Iterator> handleInsert(String idPfx, Iterator> recordItr)
- throws Exception {
+ public Iterator> handleInsert(String idPfx, Iterator> recordItr) {
// This is needed since sometimes some buckets are never picked in getPartition() and end up with 0 records
if (!recordItr.hasNext()) {
LOG.info("Empty partition");
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBucketIndexPartitioner.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBucketIndexPartitioner.java
index 65a45e1c6a047..f4572bad6db88 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBucketIndexPartitioner.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBucketIndexPartitioner.java
@@ -27,6 +27,7 @@
import java.util.Map.Entry;
import java.util.Set;
+import org.apache.hudi.common.model.WriteOperationType;
import org.apache.hudi.index.bucket.BucketIdentifier;
import scala.Tuple2;
@@ -43,6 +44,9 @@
import org.apache.hudi.table.WorkloadProfile;
import org.apache.hudi.table.WorkloadStat;
+import static org.apache.hudi.common.model.WriteOperationType.INSERT_OVERWRITE;
+import static org.apache.hudi.common.model.WriteOperationType.INSERT_OVERWRITE_TABLE;
+
/**
* Packs incoming records to be inserted into buckets (1 bucket = 1 RDD partition).
*/
@@ -58,6 +62,7 @@ public class SparkBucketIndexPartitioner> exten
* The partition offset is a multiple of the bucket num.
*/
private final Map partitionPathOffset;
+ private final boolean isOverwrite;
/**
* Partition path and file groups in it pair. Decide the file group an incoming update should go to.
@@ -85,6 +90,8 @@ public SparkBucketIndexPartitioner(WorkloadProfile profile,
i += numBuckets;
}
assignUpdates(profile);
+ WriteOperationType operationType = profile.getOperationType();
+ this.isOverwrite = INSERT_OVERWRITE.equals(operationType) || INSERT_OVERWRITE_TABLE.equals(operationType);
}
private void assignUpdates(WorkloadProfile profile) {
@@ -107,6 +114,10 @@ private void assignUpdates(WorkloadProfile profile) {
public BucketInfo getBucketInfo(int bucketNumber) {
String partitionPath = partitionPaths.get(bucketNumber / numBuckets);
String bucketId = BucketIdentifier.bucketIdStr(bucketNumber % numBuckets);
+ // Insert overwrite always generates new bucket file id
+ if (isOverwrite) {
+ return new BucketInfo(BucketType.INSERT, BucketIdentifier.newBucketFileIdPrefix(bucketId), partitionPath);
+ }
Option fileIdOption = Option.fromJavaOptional(updatePartitionPathFileIds
.getOrDefault(partitionPath, Collections.emptySet()).stream()
.filter(e -> e.startsWith(bucketId))
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeletePartitionCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeletePartitionCommitActionExecutor.java
index 149aef03e238a..8923b47fb8757 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeletePartitionCommitActionExecutor.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeletePartitionCommitActionExecutor.java
@@ -26,6 +26,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata;
import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.client.utils.DeletePartitionUtils;
import org.apache.hudi.common.data.HoodieData;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieRecordPayload;
@@ -58,6 +59,8 @@ public SparkDeletePartitionCommitActionExecutor(HoodieEngineContext context,
@Override
public HoodieWriteMetadata> execute() {
+ DeletePartitionUtils.checkForPendingTableServiceActions(table, partitions);
+
try {
HoodieTimer timer = new HoodieTimer().startTimer();
context.setJobStatus(this.getClass().getSimpleName(), "Gather all file ids from all deleting partitions.");
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteCommitActionExecutor.java
index 518063ed34186..380470fc25524 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteCommitActionExecutor.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteCommitActionExecutor.java
@@ -35,6 +35,7 @@
import org.apache.spark.Partitioner;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -86,4 +87,17 @@ protected List getAllExistingFileIds(String partitionPath) {
// because new commit is not complete. it is safe to mark all existing file Ids as old files
return table.getSliceView().getLatestFileSlices(partitionPath).map(FileSlice::getFileId).distinct().collect(Collectors.toList());
}
+
+ @Override
+ protected Iterator> handleInsertPartition(String instantTime, Integer partition, Iterator recordItr, Partitioner partitioner) {
+ SparkHoodiePartitioner upsertPartitioner = (SparkHoodiePartitioner) partitioner;
+ BucketInfo binfo = upsertPartitioner.getBucketInfo(partition);
+ BucketType btype = binfo.bucketType;
+ switch (btype) {
+ case INSERT:
+ return handleInsert(binfo.fileIdPrefix, recordItr);
+ default:
+ throw new AssertionError("Expect INSERT bucketType for insert overwrite, please correct the logical of " + partitioner.getClass().getName());
+ }
+ }
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwritePartitioner.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwritePartitioner.java
index dd545d5262846..056b3beec842b 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwritePartitioner.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwritePartitioner.java
@@ -19,6 +19,7 @@
package org.apache.hudi.table.action.commit;
import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.WorkloadProfile;
@@ -41,6 +42,20 @@ public SparkInsertOverwritePartitioner(WorkloadProfile profile, HoodieEngineCont
super(profile, context, table, config);
}
+ @Override
+ public BucketInfo getBucketInfo(int bucketNumber) {
+ BucketInfo bucketInfo = super.getBucketInfo(bucketNumber);
+ switch (bucketInfo.bucketType) {
+ case INSERT:
+ return bucketInfo;
+ case UPDATE:
+ // Insert overwrite always generates new bucket file id
+ return new BucketInfo(BucketType.INSERT, FSUtils.createNewFileIdPfx(), bucketInfo.partitionPath);
+ default:
+ throw new AssertionError();
+ }
+ }
+
/**
* Returns a list of small files in the given partition path.
*/
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/BaseSparkDeltaCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/BaseSparkDeltaCommitActionExecutor.java
index 61e6f25af9429..fe467ddbffbeb 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/BaseSparkDeltaCommitActionExecutor.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/BaseSparkDeltaCommitActionExecutor.java
@@ -87,8 +87,7 @@ public Iterator> handleUpdate(String partitionPath, String fil
}
@Override
- public Iterator> handleInsert(String idPfx, Iterator> recordItr)
- throws Exception {
+ public Iterator> handleInsert(String idPfx, Iterator> recordItr) {
// If canIndexLogFiles, write inserts to log files else write inserts to base files
if (table.getIndex().canIndexLogFiles()) {
return new SparkLazyInsertIterable<>(recordItr, true, config, instantTime, table,
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSavepoint.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSavepoint.java
new file mode 100644
index 0000000000000..7c578ad3cdee0
--- /dev/null
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSavepoint.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.client;
+
+import org.apache.hudi.avro.model.HoodieSavepointPartitionMetadata;
+import org.apache.hudi.common.config.HoodieMetadataConfig;
+import org.apache.hudi.common.fs.ConsistencyGuardConfig;
+import org.apache.hudi.common.model.HoodieCommitMetadata;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieWriteStat;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
+import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
+import org.apache.hudi.common.table.view.FileSystemViewStorageType;
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
+import org.apache.hudi.config.HoodieCompactionConfig;
+import org.apache.hudi.config.HoodieStorageConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.table.HoodieSparkTable;
+import org.apache.hudi.testutils.HoodieClientTestBase;
+import org.apache.hudi.testutils.MetadataMergeWriteStatus;
+
+import org.apache.spark.api.java.JavaRDD;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.apache.hudi.common.table.view.FileSystemViewStorageType.EMBEDDED_KV_STORE;
+import static org.apache.hudi.common.table.view.FileSystemViewStorageType.MEMORY;
+import static org.apache.hudi.common.testutils.HoodieTestUtils.RAW_TRIPS_TEST_NAME;
+import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Test cases for savepoint operation.
+ */
+public class TestSavepoint extends HoodieClientTestBase {
+
+ private static Stream testSavepointParams() {
+ return Arrays.stream(new Object[][] {
+ {true, MEMORY}, {true, EMBEDDED_KV_STORE},
+ {false, MEMORY}, {false, EMBEDDED_KV_STORE}
+ }).map(Arguments::of);
+ }
+
+ @ParameterizedTest
+ @MethodSource("testSavepointParams")
+ public void testSavepoint(boolean enableMetadataTable,
+ FileSystemViewStorageType storageType) throws IOException {
+ HoodieWriteConfig cfg = getWriteConfig(enableMetadataTable, storageType);
+ HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator(0x17AB);
+ try (SparkRDDWriteClient client = getHoodieWriteClient(cfg)) {
+ String newCommitTime = "001";
+ client.startCommitWithTime(newCommitTime);
+
+ List records = dataGen.generateInserts(newCommitTime, 200);
+ JavaRDD writeRecords = jsc.parallelize(records, 1);
+
+ List statuses = client.upsert(writeRecords, newCommitTime).collect();
+ assertNoWriteErrors(statuses);
+ client.savepoint("user", "hoodie-savepoint-unit-test");
+
+ metaClient = HoodieTableMetaClient.reload(metaClient);
+ HoodieSparkTable table = HoodieSparkTable.create(getConfig(), context, metaClient);
+ HoodieTimeline savepointTimeline = table.getActiveTimeline().getSavePointTimeline();
+ assertEquals(1, savepointTimeline.countInstants());
+
+ Map savepointPartitionMetadataMap =
+ TimelineMetadataUtils.deserializeHoodieSavepointMetadata(
+ savepointTimeline.getInstantDetails(savepointTimeline.firstInstant().get()).get())
+ .getPartitionMetadata();
+
+ HoodieTimeline commitsTimeline = table.getActiveTimeline().getCommitsTimeline();
+ Map> partitionToWriteStats = HoodieCommitMetadata.fromBytes(
+ commitsTimeline.getInstantDetails(commitsTimeline.firstInstant().get()).get(),
+ HoodieCommitMetadata.class)
+ .getPartitionToWriteStats();
+
+ assertEquals(partitionToWriteStats.size(), savepointPartitionMetadataMap.size());
+ for (Map.Entry> entry : partitionToWriteStats.entrySet()) {
+ String partition = entry.getKey();
+ assertTrue(savepointPartitionMetadataMap.containsKey(partition));
+ assertEquals(
+ entry.getValue().stream().map(path -> getFileNameFromPath(path.getPath()))
+ .sorted().collect(Collectors.toList()),
+ savepointPartitionMetadataMap.get(partition).getSavepointDataFile()
+ .stream().sorted().collect(Collectors.toList())
+ );
+ }
+ }
+ }
+
+ private HoodieWriteConfig getWriteConfig(boolean enableMetadataTable,
+ FileSystemViewStorageType storageType) {
+ return HoodieWriteConfig.newBuilder()
+ .withPath(basePath)
+ .withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA)
+ .withParallelism(2, 2)
+ .withBulkInsertParallelism(2)
+ .withFinalizeWriteParallelism(2)
+ .withDeleteParallelism(2)
+ .withWriteStatusClass(MetadataMergeWriteStatus.class)
+ .withConsistencyGuardConfig(
+ ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true).build())
+ .withCompactionConfig(
+ HoodieCompactionConfig.newBuilder().compactionSmallFileSize(1024 * 1024).build())
+ .withStorageConfig(HoodieStorageConfig.newBuilder()
+ .hfileMaxFileSize(1024 * 1024).parquetMaxFileSize(1024 * 1024)
+ .orcMaxFileSize(1024 * 1024).build())
+ .forTable(RAW_TRIPS_TEST_NAME)
+ .withEmbeddedTimelineServerEnabled(true)
+ .withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder()
+ .withEnableBackupForRemoteFileSystemView(false) // Fail test if problem connecting to timeline-server
+ .withRemoteServerPort(timelineServicePort)
+ .withStorageType(storageType)
+ .build())
+ .withMetadataConfig(
+ HoodieMetadataConfig.newBuilder().enable(enableMetadataTable).build())
+ .build();
+ }
+
+ private String getFileNameFromPath(String path) {
+ String[] parts = path.split("/");
+ return parts[parts.length - 1];
+ }
+}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
index 8ea6c2adf895f..8f4da37fbda6f 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
@@ -152,6 +152,7 @@
import static org.apache.hudi.common.model.WriteOperationType.INSERT;
import static org.apache.hudi.common.model.WriteOperationType.UPSERT;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA;
+import static org.apache.hudi.metadata.HoodieBackedTableMetadataWriter.METADATA_COMPACTION_TIME_SUFFIX;
import static org.apache.hudi.metadata.MetadataPartitionType.BLOOM_FILTERS;
import static org.apache.hudi.metadata.MetadataPartitionType.COLUMN_STATS;
import static org.apache.hudi.metadata.MetadataPartitionType.FILES;
@@ -580,8 +581,7 @@ public void testUpdationOfPopulateMetaFieldsForMetadataTable() throws Exception
@Test
public void testMetadataInsertUpsertCleanNonPartitioned() throws Exception {
- HoodieTableType tableType = COPY_ON_WRITE;
- init(tableType);
+ init(COPY_ON_WRITE);
doWriteOperationNonPartitioned(testTable, "0000001", INSERT);
doWriteOperationNonPartitioned(testTable, "0000002", UPSERT);
testTable.doCleanBasedOnCommits("0000003", Arrays.asList("0000001"));
@@ -609,8 +609,7 @@ public void testInsertUpsertCluster(HoodieTableType tableType) throws Exception
*/
@Test
public void testMetadataTableServices() throws Exception {
- HoodieTableType tableType = COPY_ON_WRITE;
- init(tableType, false);
+ init(COPY_ON_WRITE, false);
writeConfig = getWriteConfigBuilder(true, true, false)
.withMetadataConfig(HoodieMetadataConfig.newBuilder()
.enable(true)
@@ -735,8 +734,7 @@ public void testMetadataTableDeletePartition(HoodieTableType tableType) throws I
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testVirtualKeysInBaseFiles(boolean populateMetaFields) throws Exception {
- HoodieTableType tableType = MERGE_ON_READ;
- init(tableType, false);
+ init(MERGE_ON_READ, false);
writeConfig = getWriteConfigBuilder(true, true, false)
.withMetadataConfig(HoodieMetadataConfig.newBuilder()
.enable(true)
@@ -776,14 +774,61 @@ public void testVirtualKeysInBaseFiles(boolean populateMetaFields) throws Except
});
}
+ @Test
+ public void testMetadataTableCompactionWithPendingInstants() throws Exception {
+ init(COPY_ON_WRITE, false);
+ writeConfig = getWriteConfigBuilder(true, true, false)
+ .withMetadataConfig(HoodieMetadataConfig.newBuilder()
+ .enable(true)
+ .enableFullScan(true)
+ .enableMetrics(false)
+ .withMaxNumDeltaCommitsBeforeCompaction(4)
+ .build()).build();
+ initWriteConfigAndMetatableWriter(writeConfig, true);
+ doWriteOperation(testTable, "0000001", INSERT);
+ String commitInstant = "0000002";
+ doWriteOperation(testTable, commitInstant, INSERT);
+
+ // test multi-writer scenario. lets add 1,2,3,4 where 1,2,4 succeeded, but 3 is still inflight. so latest delta commit in MDT is 4, while 3 is still pending
+ // in DT and not seen by MDT yet. compaction should not trigger until 3 goes to completion.
+
+ // create an inflight commit for 3
+ HoodieCommitMetadata inflightCommitMeta = testTable.doWriteOperation("0000003", UPSERT, emptyList(),
+ asList("p1", "p2"), 2, false, true);
+ doWriteOperation(testTable, "0000004");
+ HoodieTableMetadata tableMetadata = metadata(writeConfig, context);
+ // verify that compaction of metadata table does not kick in.
+ assertFalse(tableMetadata.getLatestCompactionTime().isPresent());
+ doWriteOperation(testTable, "0000005", INSERT);
+ doWriteOperation(testTable, "0000006", INSERT);
+ doWriteOperation(testTable, "0000007", INSERT);
+
+ tableMetadata = metadata(writeConfig, context);
+ // verify that compaction of metadata table does not kick in.
+ assertFalse(tableMetadata.getLatestCompactionTime().isPresent());
+
+ // move inflight to completed
+ testTable.moveInflightCommitToComplete("0000003", inflightCommitMeta);
+
+ // we have to add another commit for compaction to trigger. if not, latest delta commit in MDT is 7, but the new incoming i.e 3 is still inflight in DT while "3"
+ // is getting applied to MDT.
+ doWriteOperation(testTable, "0000008", INSERT);
+ // verify compaction kicked in now
+ String metadataCompactionInstant = "0000007" + METADATA_COMPACTION_TIME_SUFFIX;
+ tableMetadata = metadata(writeConfig, context);
+ assertTrue(tableMetadata.getLatestCompactionTime().isPresent());
+ assertEquals(tableMetadata.getLatestCompactionTime().get(), metadataCompactionInstant);
+ // do full metadata validation
+ validateMetadata(testTable, true);
+ }
+
/**
* Tests that virtual key configs are honored in base files after compaction in metadata table.
*/
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testMetadataTableWithPendingCompaction(boolean simulateFailedCompaction) throws Exception {
- HoodieTableType tableType = COPY_ON_WRITE;
- init(tableType, false);
+ init(COPY_ON_WRITE, false);
writeConfig = getWriteConfigBuilder(true, true, false)
.withMetadataConfig(HoodieMetadataConfig.newBuilder()
.enable(true)
@@ -802,7 +847,7 @@ public void testMetadataTableWithPendingCompaction(boolean simulateFailedCompact
doWriteOperation(testTable, "0000003", INSERT);
HoodieTableMetadata tableMetadata = metadata(writeConfig, context);
- String metadataCompactionInstant = commitInstant + "001";
+ String metadataCompactionInstant = commitInstant + METADATA_COMPACTION_TIME_SUFFIX;
assertTrue(tableMetadata.getLatestCompactionTime().isPresent());
assertEquals(tableMetadata.getLatestCompactionTime().get(), metadataCompactionInstant);
@@ -860,6 +905,65 @@ public void testMetadataTableWithPendingCompaction(boolean simulateFailedCompact
}
}
+ @Test
+ public void testMetadataRollbackWithCompaction() throws Exception {
+ HoodieTableType tableType = COPY_ON_WRITE;
+ init(tableType, false);
+ writeConfig = getWriteConfigBuilder(false, true, false)
+ .withMetadataConfig(HoodieMetadataConfig.newBuilder()
+ .enable(true)
+ .withPopulateMetaFields(true)
+ .build())
+ .build();
+
+ HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc);
+
+ try (SparkRDDWriteClient client = new SparkRDDWriteClient(engineContext, writeConfig)) {
+ // Write 1 (Bulk insert)
+ String newCommitTime1 = "0000001";
+ List records = dataGen.generateInserts(newCommitTime1, 100);
+ client.startCommitWithTime(newCommitTime1);
+ JavaRDD writeStatuses = client.insert(jsc.parallelize(records, 1), newCommitTime1);
+ client.commit(newCommitTime1, writeStatuses);
+
+ String newCommitTime2 = "0000002";
+ records = dataGen.generateUniqueUpdates(newCommitTime2, 20);
+ client.startCommitWithTime(newCommitTime2);
+ writeStatuses = client.upsert(jsc.parallelize(records, 1), newCommitTime2);
+ client.commit(newCommitTime2, writeStatuses);
+
+ String newCommitTime3 = "0000003";
+ records = dataGen.generateUniqueUpdates(newCommitTime3, 20);
+ client.startCommitWithTime(newCommitTime3);
+ writeStatuses = client.upsert(jsc.parallelize(records, 1), newCommitTime3);
+ client.commit(newCommitTime3, writeStatuses);
+
+ // rollback "3" so that there is no "3" in data table timeline, but there exists a DC "3" in metadata timeline.
+ client.rollback(newCommitTime3);
+
+ // mimicing crash or making an inflight in metadata table.
+ Path toDelete = new Path(metaClient.getMetaPath() + "/metadata/.hoodie/" + newCommitTime2 + "." + HoodieTimeline.DELTA_COMMIT_ACTION);
+ metaClient.getFs().delete(toDelete);
+
+ // re-ingest w/ same commit time.
+ records = dataGen.generateUniqueUpdates(newCommitTime3, 20);
+ client.startCommitWithTime(newCommitTime3);
+ writeStatuses = client.upsert(jsc.parallelize(records, 1), newCommitTime3);
+ client.commit(newCommitTime3, writeStatuses);
+
+ // collect all commit meta files from metadata table.
+ FileStatus[] metaFiles = metaClient.getFs().listStatus(new Path(metaClient.getMetaPath() + "/metadata/.hoodie"));
+ List commit3Files = Arrays.stream(metaFiles).filter(fileStatus ->
+ fileStatus.getPath().getName().equals(newCommitTime3 + "." + HoodieTimeline.DELTA_COMMIT_ACTION)).collect(Collectors.toList());
+ List rollbackFiles = Arrays.stream(metaFiles).filter(fileStatus ->
+ fileStatus.getPath().getName().endsWith("." + HoodieTimeline.ROLLBACK_ACTION)).collect(Collectors.toList());
+
+ // ensure commit2's delta commit in MDT has last mod time > the actual rollback for previous failed commit i.e. commit2.
+ // if rollback wasn't eager, rollback's last mod time will be lower than the commit3'd delta commit last mod time.
+ assertTrue(commit3Files.get(0).getModificationTime() > rollbackFiles.get(0).getModificationTime());
+ }
+ }
+
/**
* Test arguments - Table type, populate meta fields, exclude key from payload.
*/
@@ -1163,8 +1267,7 @@ public void testRollbackOperations(HoodieTableType tableType) throws Exception {
@Test
public void testRollbackOperationsNonPartitioned() throws Exception {
- HoodieTableType tableType = COPY_ON_WRITE;
- init(tableType);
+ init(COPY_ON_WRITE);
doWriteInsertAndUpsertNonPartitioned(testTable);
// trigger an upsert
@@ -1189,8 +1292,7 @@ public void testRollbackOperationsNonPartitioned() throws Exception {
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testManualRollbacks(final boolean populateMateFields) throws Exception {
- HoodieTableType tableType = COPY_ON_WRITE;
- init(tableType, false);
+ init(COPY_ON_WRITE, false);
// Setting to archive more aggressively on the Metadata Table than the Dataset
final int maxDeltaCommitsBeforeCompaction = 4;
final int minArchiveCommitsMetadata = 2;
@@ -1555,6 +1657,54 @@ public void testColStatsPrefixLookup() throws IOException {
}
}
+ @Test
+ public void testEagerRollbackinMDT() throws IOException {
+ tableType = MERGE_ON_READ;
+ initPath();
+ init(tableType);
+ HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc);
+ SparkRDDWriteClient client = new SparkRDDWriteClient(engineContext, writeConfig);
+ // Write 1 (Bulk insert)
+ String commit1 = HoodieActiveTimeline.createNewInstantTime();
+ List records = dataGen.generateInserts(commit1, 20);
+ client.startCommitWithTime(commit1);
+ List writeStatuses = client.bulkInsert(jsc.parallelize(records, 1), commit1).collect();
+ assertNoWriteErrors(writeStatuses);
+
+ // Write 2 (inserts)
+ String commit2 = HoodieActiveTimeline.createNewInstantTime();
+ client.startCommitWithTime(commit2);
+ records = dataGen.generateInserts(commit2, 20);
+ writeStatuses = client.insert(jsc.parallelize(records, 1), commit2).collect();
+ assertNoWriteErrors(writeStatuses);
+ // remove latest completed delta commit from MDT.
+ Path toDelete = new Path(metaClient.getMetaPath() + "/metadata/.hoodie/" + commit2 + "." + HoodieTimeline.DELTA_COMMIT_ACTION);
+ metaClient.getFs().delete(toDelete);
+
+ // Write 3 (updates)
+ client = new SparkRDDWriteClient(engineContext, writeConfig);
+ String commit3 = HoodieActiveTimeline.createNewInstantTime();
+ client.startCommitWithTime(commit3);
+ records = dataGen.generateUniqueUpdates(commit3, 10);
+ writeStatuses = client.upsert(jsc.parallelize(records, 1), commit3).collect();
+ assertNoWriteErrors(writeStatuses);
+
+ // ensure that 000003 is after rollback of the partially failed 2nd commit.
+ HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setBasePath(metaClient.getMetaPath() + "/metadata/").setConf(metaClient.getHadoopConf()).build();
+ HoodieInstant rollbackInstant = metadataMetaClient.getActiveTimeline().getRollbackTimeline().getInstants().collect(Collectors.toList()).get(0);
+
+ // collect all commit meta files from metadata table.
+ FileStatus[] metaFiles = metaClient.getFs().listStatus(new Path(metaClient.getMetaPath() + "/metadata/.hoodie"));
+ List commit3Files = Arrays.stream(metaFiles).filter(fileStatus ->
+ fileStatus.getPath().getName().equals(commit3 + "." + HoodieTimeline.DELTA_COMMIT_ACTION)).collect(Collectors.toList());
+ List rollbackFiles = Arrays.stream(metaFiles).filter(fileStatus ->
+ fileStatus.getPath().getName().equals(rollbackInstant.getTimestamp() + "." + HoodieTimeline.ROLLBACK_ACTION)).collect(Collectors.toList());
+
+ // ensure commit3's delta commit in MDT has last mod time > the actual rollback for previous failed commit i.e. commit2.
+ // if rollback wasn't eager, rollback's last mod time will be lower than the commit3'd delta commit last mod time.
+ assertTrue(commit3Files.get(0).getModificationTime() > rollbackFiles.get(0).getModificationTime());
+ }
+
/**
* Test all major table operations with the given table, config and context.
*
@@ -1818,6 +1968,69 @@ public void testReattemptOfFailedClusteringCommit() throws Exception {
validateMetadata(client);
}
+ /**
+ * Validates that if an instant is completed in MDT, but crashed before commiting to DT, MDT compaction should not kick in based on the instant time
+ * since its not complete in DT yet.
+ * @throws Exception
+ */
+ @Test
+ public void testMDTCompactionWithFailedCommits() throws Exception {
+ tableType = HoodieTableType.COPY_ON_WRITE;
+ init(tableType);
+ context = new HoodieSparkEngineContext(jsc);
+ HoodieWriteConfig initialConfig = getSmallInsertWriteConfig(2000, TRIP_EXAMPLE_SCHEMA, 10, false);
+ HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withProperties(initialConfig.getProps())
+ .withMetadataConfig(HoodieMetadataConfig.newBuilder().withMaxNumDeltaCommitsBeforeCompaction(4).build()).build();
+ SparkRDDWriteClient client = getHoodieWriteClient(config);
+
+ // Write 1 (Bulk insert)
+ String newCommitTime = HoodieActiveTimeline.createNewInstantTime();
+ List records = dataGen.generateInserts(newCommitTime, 20);
+ client.startCommitWithTime(newCommitTime);
+ List writeStatuses = client.insert(jsc.parallelize(records, 1), newCommitTime).collect();
+ assertNoWriteErrors(writeStatuses);
+ validateMetadata(client);
+
+ // Write 2 (inserts)
+ newCommitTime = HoodieActiveTimeline.createNewInstantTime();
+ client.startCommitWithTime(newCommitTime);
+ records = dataGen.generateInserts(newCommitTime, 20);
+ writeStatuses = client.insert(jsc.parallelize(records, 1), newCommitTime).collect();
+ assertNoWriteErrors(writeStatuses);
+ validateMetadata(client);
+
+ // setup clustering config.
+ HoodieClusteringConfig clusteringConfig = HoodieClusteringConfig.newBuilder().withClusteringMaxNumGroups(10)
+ .withClusteringSortColumns("_row_key").withInlineClustering(true)
+ .withClusteringTargetPartitions(0).withInlineClusteringNumCommits(1).build();
+
+ HoodieWriteConfig newWriteConfig = getConfigBuilder(TRIP_EXAMPLE_SCHEMA, HoodieIndex.IndexType.BLOOM, HoodieFailedWritesCleaningPolicy.EAGER)
+ .withAutoCommit(false)
+ .withClusteringConfig(clusteringConfig).build();
+
+ // trigger clustering
+ SparkRDDWriteClient newClient = getHoodieWriteClient(newWriteConfig);
+ String clusteringCommitTime = newClient.scheduleClustering(Option.empty()).get().toString();
+ HoodieWriteMetadata> clusterMetadata = newClient.cluster(clusteringCommitTime, true);
+
+ // manually remove clustering completed instant from .hoodie folder and to mimic succeeded clustering in metadata table, but failed in data table.
+ FileCreateUtils.deleteReplaceCommit(basePath, clusteringCommitTime);
+
+ metaClient = HoodieTableMetaClient.reload(metaClient);
+ HoodieWriteConfig updatedWriteConfig = HoodieWriteConfig.newBuilder().withProperties(initialConfig.getProps())
+ .withMetadataConfig(HoodieMetadataConfig.newBuilder().withMaxNumDeltaCommitsBeforeCompaction(4).build())
+ .withRollbackUsingMarkers(false).build();
+
+ client = getHoodieWriteClient(updatedWriteConfig);
+
+ newCommitTime = HoodieActiveTimeline.createNewInstantTime();
+ client.startCommitWithTime(newCommitTime);
+ records = dataGen.generateInserts(newCommitTime, 20);
+ writeStatuses = client.insert(jsc.parallelize(records, 1), newCommitTime).collect();
+ assertNoWriteErrors(writeStatuses);
+ validateMetadata(client, Option.of(clusteringCommitTime));
+ }
+
@Test
public void testMetadataReadWithNoCompletedCommits() throws Exception {
init(HoodieTableType.COPY_ON_WRITE);
@@ -2384,6 +2597,10 @@ public HoodieWriteConfig.Builder getConfigBuilder(String schemaStr, HoodieIndex.
}
private void validateMetadata(SparkRDDWriteClient testClient) throws IOException {
+ validateMetadata(testClient, Option.empty());
+ }
+
+ private void validateMetadata(SparkRDDWriteClient testClient, Option ignoreFilesWithCommit) throws IOException {
HoodieWriteConfig config = testClient.getConfig();
SparkRDDWriteClient client;
@@ -2432,7 +2649,12 @@ private void validateMetadata(SparkRDDWriteClient testClient) throws IOException
} else {
partitionPath = new Path(basePath, partition);
}
+
FileStatus[] fsStatuses = FSUtils.getAllDataFilesInPartition(fs, partitionPath);
+ if (ignoreFilesWithCommit.isPresent()) {
+ fsStatuses = Arrays.stream(fsStatuses).filter(fileStatus -> !fileStatus.getPath().getName().contains(ignoreFilesWithCommit.get()))
+ .collect(Collectors.toList()).toArray(new FileStatus[0]);
+ }
FileStatus[] metaStatuses = tableMetadata.getAllFilesInPartition(partitionPath);
List fsFileNames = Arrays.stream(fsStatuses)
.map(s -> s.getPath().getName()).collect(Collectors.toList());
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
index 081b717146a6e..2682a74142e73 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
@@ -28,9 +28,11 @@
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.client.clustering.plan.strategy.SparkSingleFileSortPlanStrategy;
import org.apache.hudi.client.clustering.run.strategy.SparkSingleFileSortExecutionStrategy;
+import org.apache.hudi.client.transaction.lock.InProcessLockProvider;
import org.apache.hudi.client.validator.SparkPreCommitValidator;
import org.apache.hudi.client.validator.SqlQueryEqualityPreCommitValidator;
import org.apache.hudi.client.validator.SqlQuerySingleResultPreCommitValidator;
+import org.apache.hudi.common.config.LockConfiguration;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.data.HoodieData;
import org.apache.hudi.common.engine.HoodieEngineContext;
@@ -48,6 +50,7 @@
import org.apache.hudi.common.model.HoodieReplaceCommitMetadata;
import org.apache.hudi.common.model.HoodieWriteStat;
import org.apache.hudi.common.model.IOType;
+import org.apache.hudi.common.model.WriteConcurrencyMode;
import org.apache.hudi.common.model.WriteOperationType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.marker.MarkerType;
@@ -77,6 +80,7 @@
import org.apache.hudi.config.HoodieClusteringConfig;
import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieIndexConfig;
+import org.apache.hudi.config.HoodieLockConfig;
import org.apache.hudi.config.HoodiePreCommitValidatorConfig;
import org.apache.hudi.config.HoodieStorageConfig;
import org.apache.hudi.config.HoodieWriteConfig;
@@ -2409,9 +2413,13 @@ public void testRollbackAfterConsistencyCheckFailureUsingMarkers(boolean enableO
testRollbackAfterConsistencyCheckFailureUsingFileList(true, enableOptimisticConsistencyGuard, populateMetCols);
}
- @ParameterizedTest
- @MethodSource("rollbackFailedCommitsParams")
- public void testRollbackFailedCommits(HoodieFailedWritesCleaningPolicy cleaningPolicy, boolean populateMetaFields) throws Exception {
+ //@ParameterizedTest
+ //@MethodSource("rollbackFailedCommitsParams")
+ @Test
+ public void testRollbackFailedCommits() throws Exception {
+ // HoodieFailedWritesCleaningPolicy cleaningPolicy, boolean populateMetaFields
+ HoodieFailedWritesCleaningPolicy cleaningPolicy = HoodieFailedWritesCleaningPolicy.NEVER;
+ boolean populateMetaFields = true;
HoodieTestUtils.init(hadoopConf, basePath);
SparkRDDWriteClient client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
@@ -2470,11 +2478,12 @@ public void testRollbackFailedCommits(HoodieFailedWritesCleaningPolicy cleaningP
== 0);
assertTrue(timeline.getCommitsTimeline().filterCompletedInstants().countInstants() == 3);
} else if (cleaningPolicy.isNever()) {
+ // never will get translated to Lazy if OCC is enabled.
assertTrue(
timeline
.getTimelineOfActions(CollectionUtils.createSet(ROLLBACK_ACTION))
.countInstants()
- == 0);
+ == 2);
// There should be no clean or rollback action on the timeline
assertTrue(
timeline
@@ -2540,8 +2549,9 @@ public void testRollbackFailedCommitsToggleCleaningPolicy(boolean populateMetaFi
client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
client.startCommit();
timeline = metaClient.getActiveTimeline().reload();
+ // since OCC is enabled, hudi auto flips the cleaningPolicy to Lazy.
assertTrue(timeline.getTimelineOfActions(
- CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 5);
+ CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 3);
assertTrue(timeline.getCommitsTimeline().filterCompletedInstants().countInstants() == 1);
}
@@ -2774,6 +2784,14 @@ protected HoodieInstant createRequestedReplaceInstant(HoodieTableMetaClient meta
}
private HoodieWriteConfig getParallelWritingWriteConfig(HoodieFailedWritesCleaningPolicy cleaningPolicy, boolean populateMetaFields) {
+ Properties properties = new Properties();
+ properties.setProperty(LockConfiguration.LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY, "3000");
+ properties.setProperty(LockConfiguration.LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY, "3000");
+ properties.setProperty(LockConfiguration.LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP_KEY, "20");
+ if (!populateMetaFields) {
+ getPropertiesForKeyGen(populateMetaFields).entrySet().forEach(kv ->
+ properties.put(kv.getKey(), kv.getValue()));
+ }
return getConfigBuilder()
.withEmbeddedTimelineServerEnabled(false)
.withCleanConfig(HoodieCleanConfig.newBuilder()
@@ -2784,7 +2802,11 @@ private HoodieWriteConfig getParallelWritingWriteConfig(HoodieFailedWritesCleani
.withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder()
.withRemoteServerPort(timelineServicePort).build())
.withAutoCommit(false)
- .withProperties(populateMetaFields ? new Properties() : getPropertiesForKeyGen()).build();
+ .withLockConfig(HoodieLockConfig.newBuilder()
+ .withLockProvider(InProcessLockProvider.class)
+ .build())
+ .withWriteConcurrencyMode(WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL)
+ .withProperties(properties).build();
}
public static class FailingPreCommitValidator> extends SparkPreCommitValidator {
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
new file mode 100644
index 0000000000000..63d959f8643af
--- /dev/null
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.client.functional;
+
+import org.apache.hudi.client.SparkRDDWriteClient;
+import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.common.engine.HoodieLocalEngineContext;
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.HoodieCommitMetadata;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
+import org.apache.hudi.common.table.view.FileSystemViewManager;
+import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
+import org.apache.hudi.common.table.view.FileSystemViewStorageType;
+import org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView;
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodieCompactionConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.metadata.HoodieBackedTestDelayedTableMetadata;
+import org.apache.hudi.metadata.HoodieMetadataFileSystemView;
+import org.apache.hudi.testutils.HoodieClientTestHarness;
+import org.apache.hudi.timeline.service.TimelineService;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.apache.spark.api.java.JavaRDD;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
+
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.DELTA_COMMIT_ACTION;
+import static org.apache.hudi.common.table.view.FileSystemViewStorageConfig.REMOTE_PORT_NUM;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Tests the {@link RemoteHoodieTableFileSystemView} with metadata table enabled, using
+ * {@link HoodieMetadataFileSystemView} on the timeline server.
+ */
+public class TestRemoteFileSystemViewWithMetadataTable extends HoodieClientTestHarness {
+ private static final Logger LOG = LogManager.getLogger(TestRemoteFileSystemViewWithMetadataTable.class);
+
+ @BeforeEach
+ public void setUp() throws Exception {
+ initPath();
+ initSparkContexts();
+ initFileSystem();
+ initMetaClient();
+ initTimelineService();
+ dataGen = new HoodieTestDataGenerator(0x1f86);
+ }
+
+ @AfterEach
+ public void tearDown() throws Exception {
+ cleanupTimelineService();
+ cleanupClients();
+ cleanupSparkContexts();
+ cleanupFileSystem();
+ cleanupExecutorService();
+ dataGen = null;
+ System.gc();
+ }
+
+ @Override
+ public void initTimelineService() {
+ // Start a timeline server that are running across multiple commits
+ HoodieLocalEngineContext localEngineContext = new HoodieLocalEngineContext(metaClient.getHadoopConf());
+
+ try {
+ HoodieWriteConfig config = HoodieWriteConfig.newBuilder()
+ .withPath(basePath)
+ .withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder()
+ .withRemoteServerPort(incrementTimelineServicePortToUse()).build())
+ .build();
+ timelineService = new TimelineService(localEngineContext, new Configuration(),
+ TimelineService.Config.builder().enableMarkerRequests(true)
+ .serverPort(config.getViewStorageConfig().getRemoteViewServerPort()).build(),
+ FileSystem.get(new Configuration()),
+ FileSystemViewManager.createViewManager(
+ context, config.getMetadataConfig(), config.getViewStorageConfig(),
+ config.getCommonConfig(),
+ () -> new HoodieBackedTestDelayedTableMetadata(
+ context, config.getMetadataConfig(), basePath,
+ config.getViewStorageConfig().getSpillableDir(), true)));
+ timelineService.startService();
+ timelineServicePort = timelineService.getServerPort();
+ LOG.info("Started timeline server on port: " + timelineServicePort);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testMORGetLatestFileSliceWithMetadataTable(boolean useExistingTimelineServer) throws IOException {
+ // This test utilizes the `HoodieBackedTestDelayedTableMetadata` to make sure the
+ // synced file system view is always served.
+
+ SparkRDDWriteClient writeClient = createWriteClient(
+ useExistingTimelineServer ? Option.of(timelineService) : Option.empty());
+
+ for (int i = 0; i < 3; i++) {
+ writeToTable(i, writeClient);
+ }
+
+ // At this point, there are three deltacommits and one compaction commit in the Hudi timeline,
+ // and the file system view of timeline server is not yet synced
+ HoodieTableMetaClient newMetaClient = HoodieTableMetaClient.builder()
+ .setConf(metaClient.getHadoopConf())
+ .setBasePath(basePath)
+ .build();
+ HoodieActiveTimeline timeline = newMetaClient.getActiveTimeline();
+ HoodieInstant compactionCommit = timeline.lastInstant().get();
+ assertTrue(timeline.lastInstant().get().getAction().equals(COMMIT_ACTION));
+
+ // For all the file groups compacted by the compaction commit, the file system view
+ // should return the latest file slices which is written by the latest commit
+ HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes(
+ timeline.getInstantDetails(compactionCommit).get(), HoodieCommitMetadata.class);
+ List> partitionFileIdPairList =
+ commitMetadata.getPartitionToWriteStats().entrySet().stream().flatMap(
+ entry -> {
+ String partitionPath = entry.getKey();
+ return entry.getValue().stream().map(
+ writeStat -> Pair.of(partitionPath, writeStat.getFileId()));
+ }
+ ).collect(Collectors.toList());
+ List> lookupList = new ArrayList<>();
+ // Accumulate enough threads to test read-write concurrency while syncing the file system
+ // view at the timeline server
+ while (lookupList.size() < 128) {
+ lookupList.addAll(partitionFileIdPairList);
+ }
+
+ int timelineServerPort = useExistingTimelineServer
+ ? timelineService.getServerPort()
+ : writeClient.getTimelineServer().get().getRemoteFileSystemViewConfig().getRemoteViewServerPort();
+
+ LOG.info("Connecting to Timeline Server: " + timelineServerPort);
+ RemoteHoodieTableFileSystemView view =
+ new RemoteHoodieTableFileSystemView("localhost", timelineServerPort, metaClient);
+
+ List callableList = lookupList.stream()
+ .map(pair -> new TestViewLookUpCallable(view, pair, compactionCommit.getTimestamp()))
+ .collect(Collectors.toList());
+ List> resultList = new ArrayList<>();
+
+ ExecutorService pool = Executors.newCachedThreadPool();
+ callableList.forEach(callable -> {
+ resultList.add(pool.submit(callable));
+ });
+
+ assertTrue(resultList.stream().map(future -> {
+ try {
+ return future.get();
+ } catch (Exception e) {
+ LOG.error(e);
+ return false;
+ }
+ }).reduce((a, b) -> a && b).get());
+ }
+
+ @Override
+ protected HoodieTableType getTableType() {
+ return HoodieTableType.MERGE_ON_READ;
+ }
+
+ private SparkRDDWriteClient createWriteClient(Option timelineService) {
+ HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder()
+ .withPath(basePath)
+ .withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA)
+ .withParallelism(2, 2)
+ .withBulkInsertParallelism(2)
+ .withFinalizeWriteParallelism(2)
+ .withDeleteParallelism(2)
+ .withTimelineLayoutVersion(TimelineLayoutVersion.CURR_VERSION)
+ .withMergeSmallFileGroupCandidatesLimit(0)
+ .withCompactionConfig(HoodieCompactionConfig.newBuilder()
+ .withMaxNumDeltaCommitsBeforeCompaction(3)
+ .build())
+ .withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder()
+ .withStorageType(FileSystemViewStorageType.REMOTE_ONLY)
+ .withRemoteServerPort(timelineService.isPresent()
+ ? timelineService.get().getServerPort() : REMOTE_PORT_NUM.defaultValue())
+ .build())
+ .withAutoCommit(false)
+ .forTable("test_mor_table")
+ .build();
+ return new SparkRDDWriteClient(context, writeConfig, timelineService);
+ }
+
+ private void writeToTable(int round, SparkRDDWriteClient writeClient) throws IOException {
+ String instantTime = HoodieActiveTimeline.createNewInstantTime();
+ writeClient.startCommitWithTime(instantTime);
+ List records = round == 0
+ ? dataGen.generateInserts(instantTime, 100)
+ : dataGen.generateUpdates(instantTime, 100);
+
+ JavaRDD writeStatusRDD = writeClient.upsert(jsc.parallelize(records, 1), instantTime);
+ writeClient.commit(instantTime, writeStatusRDD, Option.empty(), DELTA_COMMIT_ACTION, Collections.emptyMap());
+ // Triggers compaction
+ writeClient.scheduleCompaction(Option.empty());
+ writeClient.runAnyPendingCompactions();
+ }
+
+ /**
+ * Test callable to send lookup request to the timeline server for the latest file slice
+ * based on the partition path and file ID.
+ */
+ class TestViewLookUpCallable implements Callable {
+ private final RemoteHoodieTableFileSystemView view;
+ private final Pair partitionFileIdPair;
+ private final String expectedCommitTime;
+
+ public TestViewLookUpCallable(
+ RemoteHoodieTableFileSystemView view,
+ Pair partitionFileIdPair,
+ String expectedCommitTime) {
+ this.view = view;
+ this.partitionFileIdPair = partitionFileIdPair;
+ this.expectedCommitTime = expectedCommitTime;
+ }
+
+ @Override
+ public Boolean call() throws Exception {
+ Option latestFileSlice = view.getLatestFileSlice(
+ partitionFileIdPair.getLeft(), partitionFileIdPair.getRight());
+ boolean result = latestFileSlice.isPresent() && expectedCommitTime.equals(
+ FSUtils.getCommitTime(new Path(latestFileSlice.get().getBaseFile().get().getPath()).getName()));
+ if (!result) {
+ LOG.error("The timeline server does not return the correct result: latestFileSliceReturned="
+ + latestFileSlice + " expectedCommitTime=" + expectedCommitTime);
+ }
+ return result;
+ }
+ }
+}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bloom/TestKeyRangeLookupTree.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bloom/TestKeyRangeLookupTree.java
index 1c6973db746bc..5477cda59049b 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bloom/TestKeyRangeLookupTree.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bloom/TestKeyRangeLookupTree.java
@@ -68,7 +68,7 @@ public void testFileGroupLookUpManyEntriesWithSameStartValue() {
updateExpectedMatchesToTest(toInsert);
keyRangeLookupTree.insert(toInsert);
for (int i = 0; i < 10; i++) {
- endKey += 1 + RANDOM.nextInt(100);
+ endKey += 1 + RANDOM.nextInt(50);
toInsert = new KeyRangeNode(startKey, Long.toString(endKey), UUID.randomUUID().toString());
updateExpectedMatchesToTest(toInsert);
keyRangeLookupTree.insert(toInsert);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java
index bc7de2f175a44..7c3e35258c783 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java
@@ -807,7 +807,12 @@ public void testNoArchivalWithInflightCompactionInMiddle(boolean enableMetadata)
List originalCommits = commitsList.getKey();
List commitsAfterArchival = commitsList.getValue();
if (enableMetadata) {
- assertEquals(originalCommits, commitsAfterArchival);
+ if (i != 6) {
+ assertEquals(originalCommits, commitsAfterArchival);
+ } else {
+ // on 6th commit, archival will kick in. but will archive only one commit since 2nd compaction commit is inflight.
+ assertEquals(originalCommits.size() - commitsAfterArchival.size(), 1);
+ }
} else {
if (i != 6) {
assertEquals(originalCommits, commitsAfterArchival);
@@ -1301,8 +1306,15 @@ public void testArchivalAndCompactionInMetadataTable() throws Exception {
.setLoadActiveTimelineOnLoad(true).build();
for (int i = 1; i <= 17; i++) {
- testTable.doWriteOperation("000000" + String.format("%02d", i), WriteOperationType.UPSERT,
- i == 1 ? Arrays.asList("p1", "p2") : Collections.emptyList(), Arrays.asList("p1", "p2"), 2);
+ if (i != 2) {
+ testTable.doWriteOperation("000000" + String.format("%02d", i), WriteOperationType.UPSERT,
+ i == 1 ? Arrays.asList("p1", "p2") : Collections.emptyList(), Arrays.asList("p1", "p2"), 2);
+ } else {
+ // For i == 2, roll back the first commit "00000001", so the active timeline of the
+ // data table has one rollback instant
+ // The completed rollback should not block the archival in the metadata table
+ testTable.doRollback("00000001", "00000002");
+ }
// archival
archiveAndGetCommitsList(writeConfig);
@@ -1324,10 +1336,9 @@ public void testArchivalAndCompactionInMetadataTable() throws Exception {
} else if (i == 8) {
// i == 8
// The instant "00000000000000" was archived since it's less than
- // the earliest instant on the dataset active timeline,
- // the dataset active timeline has instants of range [00000001 ~ 00000008]
- // because when it does the archiving, no compaction instant on the
- // metadata active timeline exists yet.
+ // the earliest commit on the dataset active timeline,
+ // the dataset active timeline has instants:
+ // 00000002.rollback, 00000007.commit, 00000008.commit
assertEquals(9, metadataTableInstants.size());
assertTrue(metadataTableInstants.contains(
new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "00000007001")));
@@ -1390,6 +1401,28 @@ public void testArchivalAndCompactionInMetadataTable() throws Exception {
}
}
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testPendingClusteringAfterArchiveCommit(boolean enableMetadata) throws Exception {
+ HoodieWriteConfig writeConfig = initTestTableAndGetWriteConfig(enableMetadata, 2, 5, 2);
+ // timeline:0000000(completed)->00000001(completed)->00000002(replace&inflight)->00000003(completed)->...->00000007(completed)
+ HoodieTestDataGenerator.createPendingReplaceFile(basePath, "00000002", wrapperFs.getConf());
+ for (int i = 1; i < 8; i++) {
+ if (i != 2) {
+ testTable.doWriteOperation("0000000" + i, WriteOperationType.CLUSTER, Arrays.asList("p1", "p2"), Arrays.asList("p1", "p2"), 2);
+ }
+ // archival
+ Pair, List> commitsList = archiveAndGetCommitsList(writeConfig);
+ List originalCommits = commitsList.getKey();
+ List commitsAfterArchival = commitsList.getValue();
+ assertEquals(originalCommits, commitsAfterArchival);
+ }
+
+ HoodieTimeline timeline = metaClient.getActiveTimeline().reload().getCommitsTimeline().filterCompletedInstants();
+ assertEquals(6, timeline.countInstants(),
+ "Since we have a pending clustering instant at 00000002, we should never archive any commit after 00000000");
+ }
+
private Pair, List> archiveAndGetCommitsList(HoodieWriteConfig writeConfig) throws IOException {
metaClient.reloadActiveTimeline();
HoodieTimeline timeline = metaClient.getActiveTimeline().reload().getAllCommitsTimeline().filterCompletedInstants();
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/SparkClientFunctionalTestHarness.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/SparkClientFunctionalTestHarness.java
index cb7b2e6b3c43a..e1c2414e90eb4 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/SparkClientFunctionalTestHarness.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/SparkClientFunctionalTestHarness.java
@@ -26,6 +26,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hudi.AvroConversionUtils;
+import org.apache.hudi.HoodieSparkUtils;
import org.apache.hudi.client.SparkRDDReadClient;
import org.apache.hudi.client.SparkRDDWriteClient;
import org.apache.hudi.client.WriteStatus;
@@ -106,6 +107,17 @@ public class SparkClientFunctionalTestHarness implements SparkProvider, HoodieMe
@TempDir
protected java.nio.file.Path tempDir;
+ public static Map getSparkSqlConf() {
+ Map sqlConf = new HashMap<>();
+ sqlConf.put("spark.sql.extensions", "org.apache.spark.sql.hudi.HoodieSparkSessionExtension");
+
+ if (HoodieSparkUtils.gteqSpark3_2()) {
+ sqlConf.put("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.hudi.catalog.HoodieCatalog");
+ }
+
+ return sqlConf;
+ }
+
public String basePath() {
return tempDir.toAbsolutePath().toUri().toString();
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/HFileBootstrapIndex.java b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/HFileBootstrapIndex.java
index 718fff3cf3a73..d784cdbfe02ed 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/HFileBootstrapIndex.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/HFileBootstrapIndex.java
@@ -295,14 +295,16 @@ private HFile.Reader fileIdIndexReader() {
@Override
public List getIndexedPartitionPaths() {
- HFileScanner scanner = partitionIndexReader().getScanner(true, false);
- return getAllKeys(scanner, HFileBootstrapIndex::getPartitionFromKey);
+ try (HFileScanner scanner = partitionIndexReader().getScanner(true, false)) {
+ return getAllKeys(scanner, HFileBootstrapIndex::getPartitionFromKey);
+ }
}
@Override
public List getIndexedFileGroupIds() {
- HFileScanner scanner = fileIdIndexReader().getScanner(true, false);
- return getAllKeys(scanner, HFileBootstrapIndex::getFileGroupFromKey);
+ try (HFileScanner scanner = fileIdIndexReader().getScanner(true, false)) {
+ return getAllKeys(scanner, HFileBootstrapIndex::getFileGroupFromKey);
+ }
}
private List getAllKeys(HFileScanner scanner, Function converter) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java
index 366d19fe6ebc0..86ff141ca7200 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java
@@ -77,7 +77,13 @@ public void setDefaultValue(ConfigProperty configProperty) {
if (configProperty.getInferFunc().isPresent()) {
inferValue = configProperty.getInferFunc().get().apply(this);
}
- props.setProperty(configProperty.key(), inferValue.isPresent() ? inferValue.get().toString() : configProperty.defaultValue().toString());
+ if (inferValue.isPresent() || configProperty.hasDefaultValue()) {
+ props.setProperty(
+ configProperty.key(),
+ inferValue.isPresent()
+ ? inferValue.get().toString()
+ : configProperty.defaultValue().toString());
+ }
}
}
@@ -121,7 +127,7 @@ protected void setDefaults(String configClassName) {
.forEach(f -> {
try {
ConfigProperty cfgProp = (ConfigProperty) f.get("null");
- if (cfgProp.hasDefaultValue()) {
+ if (cfgProp.hasDefaultValue() || cfgProp.getInferFunc().isPresent()) {
setDefaultValue(cfgProp);
}
} catch (IllegalAccessException e) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
index f4895f988218c..18d4aaba09e15 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
@@ -79,7 +79,7 @@ public class FSUtils {
private static final Logger LOG = LogManager.getLogger(FSUtils.class);
// Log files are of this pattern - .b5068208-e1a4-11e6-bf01-fe55135034f3_20170101134598.log.1
public static final Pattern LOG_FILE_PATTERN =
- Pattern.compile("\\.(.*)_(.*)\\.(.*)\\.([0-9]*)(_(([0-9]*)-([0-9]*)-([0-9]*)))?");
+ Pattern.compile("^\\.(.+)_(.*)\\.(log|archive)\\.(\\d+)(_((\\d+)-(\\d+)-(\\d+)))?");
private static final int MAX_ATTEMPTS_RECOVER_LEASE = 10;
private static final long MIN_CLEAN_TO_KEEP = 10;
private static final long MIN_ROLLBACK_TO_KEEP = 10;
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieRetryWrapperFileSystem.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieRetryWrapperFileSystem.java
index 075f811a42ea7..f6450326fb1cb 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieRetryWrapperFileSystem.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieRetryWrapperFileSystem.java
@@ -254,4 +254,14 @@ public boolean exists(Path f) throws IOException {
public Configuration getConf() {
return fileSystem.getConf();
}
+
+ @Override
+ public short getDefaultReplication() {
+ return fileSystem.getDefaultReplication();
+ }
+
+ @Override
+ public short getDefaultReplication(Path path) {
+ return fileSystem.getDefaultReplication(path);
+ }
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/AbstractDebeziumAvroPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/AbstractDebeziumAvroPayload.java
index 9082d572a4bdb..e9c1e30a15263 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/AbstractDebeziumAvroPayload.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/AbstractDebeziumAvroPayload.java
@@ -56,15 +56,19 @@ public AbstractDebeziumAvroPayload(Option record) {
@Override
public Option getInsertValue(Schema schema) throws IOException {
- IndexedRecord insertRecord = getInsertRecord(schema);
- return handleDeleteOperation(insertRecord);
+ Option insertValue = getInsertRecord(schema);
+ return insertValue.isPresent() ? handleDeleteOperation(insertValue.get()) : Option.empty();
}
@Override
public Option combineAndGetUpdateValue(IndexedRecord currentValue, Schema schema) throws IOException {
// Step 1: If the time occurrence of the current record in storage is higher than the time occurrence of the
// insert record (including a delete record), pick the current record.
- if (shouldPickCurrentRecord(currentValue, getInsertRecord(schema), schema)) {
+ Option insertValue = getInsertRecord(schema);
+ if (!insertValue.isPresent()) {
+ return Option.empty();
+ }
+ if (shouldPickCurrentRecord(currentValue, insertValue.get(), schema)) {
return Option.of(currentValue);
}
// Step 2: Pick the insert record (as a delete record if its a deleted event)
@@ -84,7 +88,7 @@ private Option handleDeleteOperation(IndexedRecord insertRecord)
return delete ? Option.empty() : Option.of(insertRecord);
}
- private IndexedRecord getInsertRecord(Schema schema) throws IOException {
- return super.getInsertValue(schema).get();
+ private Option getInsertRecord(Schema schema) throws IOException {
+ return super.getInsertValue(schema);
}
}
\ No newline at end of file
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
index 16dd373486f61..64a38d1bb5469 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
@@ -58,6 +58,7 @@
import java.io.IOException;
import java.io.Serializable;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -97,6 +98,10 @@ public class HoodieTableMetaClient implements Serializable {
public static final String MARKER_EXTN = ".marker";
+ // In-memory cache for archived timeline based on the start instant time
+ // Only one entry should be present in this map
+ private final Map archivedTimelineMap = new HashMap<>();
+
// NOTE: Since those two parameters lay on the hot-path of a lot of computations, we
// use tailored extension of the {@code Path} class allowing to avoid repetitive
// computations secured by its immutability
@@ -110,7 +115,6 @@ public class HoodieTableMetaClient implements Serializable {
private TimelineLayoutVersion timelineLayoutVersion;
protected HoodieTableConfig tableConfig;
protected HoodieActiveTimeline activeTimeline;
- private HoodieArchivedTimeline archivedTimeline;
private ConsistencyGuardConfig consistencyGuardConfig = ConsistencyGuardConfig.newBuilder().build();
private FileSystemRetryConfig fileSystemRetryConfig = FileSystemRetryConfig.newBuilder().build();
protected HoodieMetastoreConfig metastoreConfig;
@@ -365,10 +369,7 @@ public FileSystemRetryConfig getFileSystemRetryConfig() {
* @return Archived commit timeline
*/
public synchronized HoodieArchivedTimeline getArchivedTimeline() {
- if (archivedTimeline == null) {
- archivedTimeline = new HoodieArchivedTimeline(this);
- }
- return archivedTimeline;
+ return getArchivedTimeline(StringUtils.EMPTY_STRING);
}
public HoodieMetastoreConfig getMetastoreConfig() {
@@ -379,21 +380,49 @@ public HoodieMetastoreConfig getMetastoreConfig() {
}
/**
- * Returns fresh new archived commits as a timeline from startTs (inclusive).
- *
- * This is costly operation if really early endTs is specified.
- * Be caution to use this only when the time range is short.
- *
- *
This method is not thread safe.
+ * Returns the cached archived timeline from startTs (inclusive).
*
- * @return Archived commit timeline
+ * @param startTs The start instant time (inclusive) of the archived timeline.
+ * @return the archived timeline.
*/
public HoodieArchivedTimeline getArchivedTimeline(String startTs) {
- return new HoodieArchivedTimeline(this, startTs);
+ return getArchivedTimeline(startTs, true);
+ }
+
+ /**
+ * Returns the cached archived timeline if using in-memory cache or a fresh new archived
+ * timeline if not using cache, from startTs (inclusive).
+ *
+ * Instantiating an archived timeline is costly operation if really early startTs is
+ * specified.
+ *
+ * This method is not thread safe.
+ *
+ * @param startTs The start instant time (inclusive) of the archived timeline.
+ * @param useCache Whether to use in-memory cache.
+ * @return the archived timeline based on the arguments.
+ */
+ public HoodieArchivedTimeline getArchivedTimeline(String startTs, boolean useCache) {
+ if (useCache) {
+ if (!archivedTimelineMap.containsKey(startTs)) {
+ // Only keep one entry in the map
+ archivedTimelineMap.clear();
+ archivedTimelineMap.put(startTs, instantiateArchivedTimeline(startTs));
+ }
+ return archivedTimelineMap.get(startTs);
+ }
+ return instantiateArchivedTimeline(startTs);
+ }
+
+ private HoodieArchivedTimeline instantiateArchivedTimeline(String startTs) {
+ return StringUtils.isNullOrEmpty(startTs)
+ ? new HoodieArchivedTimeline(this)
+ : new HoodieArchivedTimeline(this, startTs);
}
/**
* Validate table properties.
+ *
* @param properties Properties from writeConfig.
*/
public void validateTableProperties(Properties properties) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java
index a070a7e94d1c0..4642885202260 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java
@@ -24,6 +24,7 @@
import org.apache.hudi.common.model.HoodieReplaceCommitMetadata;
import org.apache.hudi.common.model.WriteOperationType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieIOException;
@@ -39,6 +40,11 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.DELTA_COMMIT_ACTION;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.REPLACE_COMMIT_ACTION;
+import static org.apache.hudi.common.table.timeline.HoodieTimeline.SAVEPOINT_ACTION;
+
/**
* TimelineUtils provides a common way to query incremental meta-data changes for a hoodie table.
*
@@ -87,15 +93,15 @@ public static List getDroppedPartitions(HoodieTimeline timeline) {
public static List getAffectedPartitions(HoodieTimeline timeline) {
return timeline.filterCompletedInstants().getInstants().flatMap(s -> {
switch (s.getAction()) {
- case HoodieTimeline.COMMIT_ACTION:
- case HoodieTimeline.DELTA_COMMIT_ACTION:
+ case COMMIT_ACTION:
+ case DELTA_COMMIT_ACTION:
try {
HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes(timeline.getInstantDetails(s).get(), HoodieCommitMetadata.class);
return commitMetadata.getPartitionToWriteStats().keySet().stream();
} catch (IOException e) {
throw new HoodieIOException("Failed to get partitions written at " + s, e);
- }
- case HoodieTimeline.REPLACE_COMMIT_ACTION:
+ }
+ case REPLACE_COMMIT_ACTION:
try {
HoodieReplaceCommitMetadata commitMetadata = HoodieReplaceCommitMetadata.fromBytes(
timeline.getInstantDetails(s).get(), HoodieReplaceCommitMetadata.class);
@@ -148,11 +154,11 @@ public static List getAffectedPartitions(HoodieTimeline timeline) {
* Get extra metadata for specified key from latest commit/deltacommit/replacecommit(eg. insert_overwrite) instant.
*/
public static Option getExtraMetadataFromLatest(HoodieTableMetaClient metaClient, String extraMetadataKey) {
- return metaClient.getCommitsTimeline().filterCompletedInstants().getReverseOrderedInstants()
+ return metaClient.getCommitsTimeline().filterCompletedInstants().getReverseOrderedInstants()
// exclude clustering commits for returning user stored extra metadata
.filter(instant -> !isClusteringCommit(metaClient, instant))
.findFirst().map(instant ->
- getMetadataValue(metaClient, extraMetadataKey, instant)).orElse(Option.empty());
+ getMetadataValue(metaClient, extraMetadataKey, instant)).orElse(Option.empty());
}
/**
@@ -170,7 +176,7 @@ public static Option getExtraMetadataFromLatestIncludeClustering(HoodieT
*/
public static Map> getAllExtraMetadataForKey(HoodieTableMetaClient metaClient, String extraMetadataKey) {
return metaClient.getCommitsTimeline().filterCompletedInstants().getReverseOrderedInstants().collect(Collectors.toMap(
- HoodieInstant::getTimestamp, instant -> getMetadataValue(metaClient, extraMetadataKey, instant)));
+ HoodieInstant::getTimestamp, instant -> getMetadataValue(metaClient, extraMetadataKey, instant)));
}
private static Option getMetadataValue(HoodieTableMetaClient metaClient, String extraMetadataKey, HoodieInstant instant) {
@@ -184,10 +190,10 @@ private static Option getMetadataValue(HoodieTableMetaClient metaClient,
throw new HoodieIOException("Unable to parse instant metadata " + instant, e);
}
}
-
+
public static boolean isClusteringCommit(HoodieTableMetaClient metaClient, HoodieInstant instant) {
try {
- if (HoodieTimeline.REPLACE_COMMIT_ACTION.equals(instant.getAction())) {
+ if (REPLACE_COMMIT_ACTION.equals(instant.getAction())) {
// replacecommit is used for multiple operations: insert_overwrite/cluster etc.
// Check operation type to see if this instant is related to clustering.
HoodieReplaceCommitMetadata replaceMetadata = HoodieReplaceCommitMetadata.fromBytes(
@@ -209,4 +215,84 @@ public static HoodieDefaultTimeline getTimeline(HoodieTableMetaClient metaClient
}
return activeTimeline;
}
+
+ /**
+ * Returns a Hudi timeline with commits after the given instant time (exclusive).
+ *
+ * @param metaClient {@link HoodieTableMetaClient} instance.
+ * @param exclusiveStartInstantTime Start instant time (exclusive).
+ * @return Hudi timeline.
+ */
+ public static HoodieTimeline getCommitsTimelineAfter(
+ HoodieTableMetaClient metaClient, String exclusiveStartInstantTime) {
+ HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline();
+ HoodieDefaultTimeline timeline =
+ activeTimeline.isBeforeTimelineStarts(exclusiveStartInstantTime)
+ ? metaClient.getArchivedTimeline(exclusiveStartInstantTime)
+ .mergeTimeline(activeTimeline)
+ : activeTimeline;
+ return timeline.getCommitsTimeline()
+ .findInstantsAfter(exclusiveStartInstantTime, Integer.MAX_VALUE);
+ }
+
+ /**
+ * Returns the commit metadata of the given instant.
+ *
+ * @param instant The hoodie instant
+ * @param timeline The timeline
+ * @return the commit metadata
+ */
+ public static HoodieCommitMetadata getCommitMetadata(
+ HoodieInstant instant,
+ HoodieTimeline timeline) throws IOException {
+ byte[] data = timeline.getInstantDetails(instant).get();
+ if (instant.getAction().equals(REPLACE_COMMIT_ACTION)) {
+ return HoodieReplaceCommitMetadata.fromBytes(data, HoodieReplaceCommitMetadata.class);
+ } else {
+ return HoodieCommitMetadata.fromBytes(data, HoodieCommitMetadata.class);
+ }
+ }
+
+ /**
+ * Gets the qualified earliest instant from the active timeline of the data table
+ * for the archival in metadata table.
+ *
+ * the qualified earliest instant is chosen as the earlier one between the earliest
+ * commit (COMMIT, DELTA_COMMIT, and REPLACE_COMMIT only, considering non-savepoint
+ * commit only if enabling archive beyond savepoint) and the earliest inflight
+ * instant (all actions).
+ *
+ * @param dataTableActiveTimeline the active timeline of the data table.
+ * @param shouldArchiveBeyondSavepoint whether to archive beyond savepoint.
+ * @return the instant meeting the requirement.
+ */
+ public static Option getEarliestInstantForMetadataArchival(
+ HoodieActiveTimeline dataTableActiveTimeline, boolean shouldArchiveBeyondSavepoint) {
+ // This is for commits only, not including CLEAN, ROLLBACK, etc.
+ // When archive beyond savepoint is enabled, there are chances that there could be holes
+ // in the timeline due to archival and savepoint interplay. So, the first non-savepoint
+ // commit in the data timeline is considered as beginning of the active timeline.
+ Option earliestCommit = shouldArchiveBeyondSavepoint
+ ? dataTableActiveTimeline.getTimelineOfActions(
+ CollectionUtils.createSet(
+ COMMIT_ACTION, DELTA_COMMIT_ACTION, REPLACE_COMMIT_ACTION, SAVEPOINT_ACTION))
+ .getFirstNonSavepointCommit()
+ : dataTableActiveTimeline.getCommitsTimeline().firstInstant();
+ // This is for all instants which are in-flight
+ Option earliestInflight =
+ dataTableActiveTimeline.filterInflightsAndRequested().firstInstant();
+
+ if (earliestCommit.isPresent() && earliestInflight.isPresent()) {
+ if (earliestCommit.get().compareTo(earliestInflight.get()) < 0) {
+ return earliestCommit;
+ }
+ return earliestInflight;
+ } else if (earliestCommit.isPresent()) {
+ return earliestCommit;
+ } else if (earliestInflight.isPresent()) {
+ return earliestInflight;
+ } else {
+ return Option.empty();
+ }
+ }
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java
index 39bb3a2a5da27..69482fd5db6d5 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java
@@ -38,6 +38,7 @@
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hadoop.fs.FileStatus;
@@ -52,6 +53,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -60,6 +62,7 @@
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import java.util.function.Function;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.stream.Collectors;
@@ -91,8 +94,8 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
// Locks to control concurrency. Sync operations use write-lock blocking all fetch operations.
// For the common-case, we allow concurrent read of single or multiple partitions
private final ReentrantReadWriteLock globalLock = new ReentrantReadWriteLock();
- private final ReadLock readLock = globalLock.readLock();
- private final WriteLock writeLock = globalLock.writeLock();
+ protected final ReadLock readLock = globalLock.readLock();
+ protected final WriteLock writeLock = globalLock.writeLock();
private BootstrapIndex bootstrapIndex;
@@ -117,7 +120,7 @@ protected void init(HoodieTableMetaClient metaClient, HoodieTimeline visibleActi
/**
* Refresh commits timeline.
- *
+ *
* @param visibleActiveTimeline Visible Active Timeline
*/
protected void refreshTimeline(HoodieTimeline visibleActiveTimeline) {
@@ -159,13 +162,13 @@ public List addFilesToView(FileStatus[] statuses) {
* Build FileGroups from passed in file-status.
*/
protected List buildFileGroups(FileStatus[] statuses, HoodieTimeline timeline,
- boolean addPendingCompactionFileSlice) {
+ boolean addPendingCompactionFileSlice) {
return buildFileGroups(convertFileStatusesToBaseFiles(statuses), convertFileStatusesToLogFiles(statuses), timeline,
addPendingCompactionFileSlice);
}
protected List buildFileGroups(Stream baseFileStream,
- Stream logFileStream, HoodieTimeline timeline, boolean addPendingCompactionFileSlice) {
+ Stream logFileStream, HoodieTimeline timeline, boolean addPendingCompactionFileSlice) {
Map, List> baseFiles =
baseFileStream.collect(Collectors.groupingBy(baseFile -> {
String partitionPathStr = getPartitionPathFor(baseFile);
@@ -223,7 +226,7 @@ private void resetFileGroupsReplaced(HoodieTimeline timeline) {
// get replace instant mapping for each partition, fileId
return replaceMetadata.getPartitionToReplaceFileIds().entrySet().stream().flatMap(entry -> entry.getValue().stream().map(e ->
- new AbstractMap.SimpleEntry<>(new HoodieFileGroupId(entry.getKey(), e), instant)));
+ new AbstractMap.SimpleEntry<>(new HoodieFileGroupId(entry.getKey(), e), instant)));
} catch (HoodieIOException ex) {
if (ex.getIOException() instanceof FileNotFoundException) {
@@ -258,6 +261,8 @@ public void close() {
/**
* Clears the partition Map and reset view states.
+ *
+ * NOTE: The logic MUST BE guarded by the write lock.
*/
@Override
public void reset() {
@@ -274,7 +279,7 @@ public void reset() {
/**
* Clear the resource.
*/
- private void clear() {
+ protected void clear() {
addedPartitions.clear();
resetViewState();
bootstrapIndex = null;
@@ -285,6 +290,114 @@ private void clear() {
*/
protected abstract void resetViewState();
+ /**
+ * Batch loading all the partitions if needed.
+ *
+ * @return A list of relative partition paths of all partitions.
+ */
+ private List ensureAllPartitionsLoadedCorrectly() {
+ ValidationUtils.checkArgument(!isClosed(), "View is already closed");
+ try {
+ List formattedPartitionList = getAllPartitionPaths().stream()
+ .map(this::formatPartitionKey).collect(Collectors.toList());
+ ensurePartitionsLoadedCorrectly(formattedPartitionList);
+ return formattedPartitionList;
+ } catch (IOException e) {
+ throw new HoodieIOException("Failed to get all partition paths", e);
+ }
+ }
+
+ /**
+ * Allows lazily loading the partitions if needed.
+ *
+ * @param partitionList list of partitions to be loaded if not present.
+ */
+ private void ensurePartitionsLoadedCorrectly(List partitionList) {
+
+ ValidationUtils.checkArgument(!isClosed(), "View is already closed");
+
+ Set partitionSet = new HashSet<>();
+ synchronized (addedPartitions) {
+ partitionList.forEach(partition -> {
+ if (!addedPartitions.containsKey(partition) && !isPartitionAvailableInStore(partition)) {
+ partitionSet.add(partition);
+ }
+ });
+
+ if (!partitionSet.isEmpty()) {
+ long beginTs = System.currentTimeMillis();
+ // Not loaded yet
+ try {
+ LOG.info("Building file system view for partitions " + partitionSet);
+
+ // Pairs of relative partition path and absolute partition path
+ List> absolutePartitionPathList = partitionSet.stream()
+ .map(partition -> Pair.of(
+ partition, FSUtils.getPartitionPath(metaClient.getBasePathV2(), partition)))
+ .collect(Collectors.toList());
+ long beginLsTs = System.currentTimeMillis();
+ Map, FileStatus[]> statusesMap =
+ listPartitions(absolutePartitionPathList);
+ long endLsTs = System.currentTimeMillis();
+ LOG.debug("Time taken to list partitions " + partitionSet + " =" + (endLsTs - beginLsTs));
+ statusesMap.forEach((partitionPair, statuses) -> {
+ String relativePartitionStr = partitionPair.getLeft();
+ List groups = addFilesToView(statuses);
+ if (groups.isEmpty()) {
+ storePartitionView(relativePartitionStr, new ArrayList<>());
+ }
+ LOG.debug("#files found in partition (" + relativePartitionStr + ") =" + statuses.length);
+ });
+ } catch (IOException e) {
+ throw new HoodieIOException("Failed to list base files in partitions " + partitionSet, e);
+ }
+ long endTs = System.currentTimeMillis();
+ LOG.debug("Time to load partition " + partitionSet + " =" + (endTs - beginTs));
+ }
+
+ partitionSet.forEach(partition ->
+ addedPartitions.computeIfAbsent(partition, partitionPathStr -> true)
+ );
+ }
+ }
+
+ /***
+ * @return A list of relative partition paths of all partitions.
+ * @throws IOException upon error.
+ */
+ protected List getAllPartitionPaths() throws IOException {
+ throw new HoodieException("Getting all partition paths with file system listing sequentially "
+ + "can be very slow. This should not be invoked.");
+ }
+
+ /**
+ * @param partitionPathList A list of pairs of the relative and absolute paths of the partitions.
+ * @return all the files from the partitions.
+ * @throws IOException upon error.
+ */
+ protected Map, FileStatus[]> listPartitions(
+ List> partitionPathList) throws IOException {
+ Map, FileStatus[]> fileStatusMap = new HashMap<>();
+
+ for (Pair partitionPair : partitionPathList) {
+ Path absolutePartitionPath = partitionPair.getRight();
+ try {
+ fileStatusMap.put(partitionPair, metaClient.getFs().listStatus(absolutePartitionPath));
+ } catch (IOException e) {
+ // Create the path if it does not exist already
+ if (!metaClient.getFs().exists(absolutePartitionPath)) {
+ metaClient.getFs().mkdirs(absolutePartitionPath);
+ fileStatusMap.put(partitionPair, new FileStatus[0]);
+ } else {
+ // in case the partition path was created by another caller
+ fileStatusMap.put(partitionPair, metaClient.getFs().listStatus(absolutePartitionPath));
+ }
+ }
+ }
+
+ return fileStatusMap;
+ }
+
/**
* Allows lazily loading the partitions if needed.
*
@@ -523,19 +636,38 @@ public final Stream getLatestBaseFilesBeforeOrOn(String partitio
readLock.lock();
String partitionPath = formatPartitionKey(partitionStr);
ensurePartitionLoadedCorrectly(partitionPath);
- return fetchAllStoredFileGroups(partitionPath)
- .filter(fileGroup -> !isFileGroupReplacedBeforeOrOn(fileGroup.getFileGroupId(), maxCommitTime))
- .map(fileGroup -> Option.fromJavaOptional(fileGroup.getAllBaseFiles()
- .filter(baseFile -> HoodieTimeline.compareTimestamps(baseFile.getCommitTime(), HoodieTimeline.LESSER_THAN_OR_EQUALS, maxCommitTime
- ))
- .filter(df -> !isBaseFileDueToPendingCompaction(df) && !isBaseFileDueToPendingClustering(df)).findFirst()))
- .filter(Option::isPresent).map(Option::get)
- .map(df -> addBootstrapBaseFileIfPresent(new HoodieFileGroupId(partitionPath, df.getFileId()), df));
+ return getLatestBaseFilesBeforeOrOnFromCache(partitionPath, maxCommitTime);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public final Map> getAllLatestBaseFilesBeforeOrOn(String maxCommitTime) {
+ try {
+ readLock.lock();
+
+ List formattedPartitionList = ensureAllPartitionsLoadedCorrectly();
+ return formattedPartitionList.stream().collect(Collectors.toMap(
+ Function.identity(),
+ partitionPath -> getLatestBaseFilesBeforeOrOnFromCache(partitionPath, maxCommitTime)
+ ));
} finally {
readLock.unlock();
}
}
+ private Stream getLatestBaseFilesBeforeOrOnFromCache(String partitionPath, String maxCommitTime) {
+ return fetchAllStoredFileGroups(partitionPath)
+ .filter(fileGroup -> !isFileGroupReplacedBeforeOrOn(fileGroup.getFileGroupId(), maxCommitTime))
+ .map(fileGroup -> Option.fromJavaOptional(fileGroup.getAllBaseFiles()
+ .filter(baseFile -> HoodieTimeline.compareTimestamps(baseFile.getCommitTime(), HoodieTimeline.LESSER_THAN_OR_EQUALS, maxCommitTime
+ ))
+ .filter(df -> !isBaseFileDueToPendingCompaction(df) && !isBaseFileDueToPendingClustering(df)).findFirst()))
+ .filter(Option::isPresent).map(Option::get)
+ .map(df -> addBootstrapBaseFileIfPresent(new HoodieFileGroupId(partitionPath, df.getFileId()), df));
+ }
+
@Override
public final Option getBaseFileOn(String partitionStr, String instantTime, String fileId) {
try {
@@ -546,8 +678,8 @@ public final Option getBaseFileOn(String partitionStr, String in
return Option.empty();
} else {
return fetchHoodieFileGroup(partitionPath, fileId).map(fileGroup -> fileGroup.getAllBaseFiles()
- .filter(baseFile -> HoodieTimeline.compareTimestamps(baseFile.getCommitTime(), HoodieTimeline.EQUALS,
- instantTime)).filter(df -> !isBaseFileDueToPendingCompaction(df) && !isBaseFileDueToPendingClustering(df)).findFirst().orElse(null))
+ .filter(baseFile -> HoodieTimeline.compareTimestamps(baseFile.getCommitTime(), HoodieTimeline.EQUALS,
+ instantTime)).filter(df -> !isBaseFileDueToPendingCompaction(df) && !isBaseFileDueToPendingClustering(df)).findFirst().orElse(null))
.map(df -> addBootstrapBaseFileIfPresent(new HoodieFileGroupId(partitionPath, fileId), df));
}
} finally {
@@ -582,8 +714,8 @@ public final Stream getLatestBaseFilesInRange(List commi
return fetchAllStoredFileGroups()
.filter(fileGroup -> !isFileGroupReplacedBeforeAny(fileGroup.getFileGroupId(), commitsToReturn))
.map(fileGroup -> Pair.of(fileGroup.getFileGroupId(), Option.fromJavaOptional(
- fileGroup.getAllBaseFiles().filter(baseFile -> commitsToReturn.contains(baseFile.getCommitTime())
- && !isBaseFileDueToPendingCompaction(baseFile) && !isBaseFileDueToPendingClustering(baseFile)).findFirst()))).filter(p -> p.getValue().isPresent())
+ fileGroup.getAllBaseFiles().filter(baseFile -> commitsToReturn.contains(baseFile.getCommitTime())
+ && !isBaseFileDueToPendingCompaction(baseFile) && !isBaseFileDueToPendingClustering(baseFile)).findFirst()))).filter(p -> p.getValue().isPresent())
.map(p -> addBootstrapBaseFileIfPresent(p.getKey(), p.getValue().get()));
} finally {
readLock.unlock();
@@ -613,9 +745,9 @@ public final Stream getLatestFileSlices(String partitionStr) {
String partitionPath = formatPartitionKey(partitionStr);
ensurePartitionLoadedCorrectly(partitionPath);
return fetchLatestFileSlices(partitionPath)
- .filter(slice -> !isFileGroupReplaced(slice.getFileGroupId()))
- .flatMap(slice -> this.filterBaseFileAfterPendingCompaction(slice, true))
- .map(this::addBootstrapBaseFileIfPresent);
+ .filter(slice -> !isFileGroupReplaced(slice.getFileGroupId()))
+ .flatMap(slice -> this.filterBaseFileAfterPendingCompaction(slice, true))
+ .map(this::addBootstrapBaseFileIfPresent);
} finally {
readLock.unlock();
}
@@ -670,26 +802,26 @@ public final Stream getLatestUnCompactedFileSlices(String partitionSt
@Override
public final Stream