metrics, long defaultBaseFileSize) {
+ int numLogFiles = 0;
+ long totalLogFileSize = 0;
+ long totalIORead = 0;
+ long totalIOWrite = 0;
+ long totalIO = 0;
+
+ for (FileSlice slice : fileSlices) {
+ numLogFiles += slice.getLogFiles().count();
+ // Total size of all the log files
+ totalLogFileSize += slice.getLogFiles().map(HoodieLogFile::getFileSize).filter(size -> size >= 0)
+ .reduce(Long::sum).orElse(0L);
+
+ long baseFileSize = slice.getBaseFile().isPresent() ? slice.getBaseFile().get().getFileSize() : 0L;
+ // Total read will be the base file + all the log files
+ totalIORead = FSUtils.getSizeInMB(baseFileSize + totalLogFileSize);
+ // Total write will be similar to the size of the base file
+ totalIOWrite = FSUtils.getSizeInMB(baseFileSize > 0 ? baseFileSize : defaultBaseFileSize);
+ // Total IO will the the IO for read + write
+ totalIO = totalIORead + totalIOWrite;
+ }
+
+ metrics.put(TOTAL_IO_READ_MB, (double) totalIORead);
+ metrics.put(TOTAL_IO_WRITE_MB, (double) totalIOWrite);
+ metrics.put(TOTAL_IO_MB, (double) totalIO);
+ metrics.put(TOTAL_LOG_FILE_SIZE, (double) totalLogFileSize);
+ metrics.put(TOTAL_LOG_FILES, (double) numLogFiles);
+ }
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java
new file mode 100644
index 0000000000000..91acd3075377a
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieClusteringConfig.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.config;
+
+import org.apache.hudi.common.config.DefaultHoodieConfig;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.Properties;
+
+/**
+ * Clustering specific configs.
+ */
+public class HoodieClusteringConfig extends DefaultHoodieConfig {
+
+ // Config to provide a strategy class to create ClusteringPlan. Class has to be subclass of ClusteringPlanStrategy
+ public static final String CLUSTERING_PLAN_STRATEGY_CLASS = "hoodie.clustering.plan.strategy.class";
+ public static final String DEFAULT_CLUSTERING_PLAN_STRATEGY_CLASS =
+ "org.apache.hudi.client.clustering.plan.strategy.SparkRecentDaysClusteringPlanStrategy";
+
+ // Config to provide a strategy class to execute a ClusteringPlan. Class has to be subclass of RunClusteringStrategy
+ public static final String CLUSTERING_EXECUTION_STRATEGY_CLASS = "hoodie.clustering.execution.strategy.class";
+ public static final String DEFAULT_CLUSTERING_EXECUTION_STRATEGY_CLASS =
+ "org.apache.hudi.client.clustering.run.strategy.SparkSortAndSizeExecutionStrategy";
+
+ // Turn on inline clustering - clustering will be run after write operation is complete.
+ public static final String INLINE_CLUSTERING_PROP = "hoodie.clustering.inline";
+ private static final String DEFAULT_INLINE_CLUSTERING = "false";
+
+ // Config to control frequency of clustering
+ public static final String INLINE_CLUSTERING_MAX_COMMIT_PROP = "hoodie.clustering.inline.max.commits";
+ private static final String DEFAULT_INLINE_CLUSTERING_NUM_COMMITS = "4";
+
+ // Any strategy specific params can be saved with this prefix
+ public static final String CLUSTERING_STRATEGY_PARAM_PREFIX = "hoodie.clustering.plan.strategy.";
+
+ // Number of partitions to list to create ClusteringPlan.
+ public static final String CLUSTERING_TARGET_PARTITIONS = CLUSTERING_STRATEGY_PARAM_PREFIX + "daybased.lookback.partitions";
+ public static final String DEFAULT_CLUSTERING_TARGET_PARTITIONS = String.valueOf(2);
+
+ // Files smaller than the size specified here are candidates for clustering.
+ public static final String CLUSTERING_PLAN_SMALL_FILE_LIMIT = CLUSTERING_STRATEGY_PARAM_PREFIX + "small.file.limit";
+ public static final String DEFAULT_CLUSTERING_PLAN_SMALL_FILE_LIMIT = String.valueOf(600 * 1024 * 1024L); // 600MB
+
+ // Each clustering operation can create multiple groups. Total amount of data processed by clustering operation
+ // is defined by below two properties (CLUSTERING_MAX_BYTES_PER_GROUP * CLUSTERING_MAX_NUM_GROUPS).
+ // Max amount of data to be included in one group
+ public static final String CLUSTERING_MAX_BYTES_PER_GROUP = CLUSTERING_STRATEGY_PARAM_PREFIX + "max.bytes.per.group";
+ public static final String DEFAULT_CLUSTERING_MAX_GROUP_SIZE = String.valueOf(2 * 1024 * 1024 * 1024L);
+
+ // Maximum number of groups to create as part of ClusteringPlan. Increasing groups will increase parallelism.
+ public static final String CLUSTERING_MAX_NUM_GROUPS = CLUSTERING_STRATEGY_PARAM_PREFIX + "max.num.groups";
+ public static final String DEFAULT_CLUSTERING_MAX_NUM_GROUPS = "30";
+
+ // Each group can produce 'N' (CLUSTERING_MAX_GROUP_SIZE/CLUSTERING_TARGET_FILE_SIZE) output file groups.
+ public static final String CLUSTERING_TARGET_FILE_MAX_BYTES = CLUSTERING_STRATEGY_PARAM_PREFIX + "target.file.max.bytes";
+ public static final String DEFAULT_CLUSTERING_TARGET_FILE_MAX_BYTES = String.valueOf(1 * 1024 * 1024 * 1024L); // 1GB
+
+ // Constants related to clustering that may be used by more than 1 strategy.
+ public static final String CLUSTERING_SORT_COLUMNS_PROPERTY = HoodieClusteringConfig.CLUSTERING_STRATEGY_PARAM_PREFIX + "sort.columns";
+
+ // When file groups is in clustering, need to handle the update to these file groups. Default strategy just reject the update
+ public static final String CLUSTERING_UPDATES_STRATEGY_PROP = "hoodie.clustering.updates.strategy";
+ public static final String DEFAULT_CLUSTERING_UPDATES_STRATEGY = "org.apache.hudi.client.clustering.update.strategy.SparkRejectUpdateStrategy";
+
+ // Async clustering
+ public static final String ASYNC_CLUSTERING_ENABLE_OPT_KEY = "hoodie.clustering.async.enabled";
+ public static final String DEFAULT_ASYNC_CLUSTERING_ENABLE_OPT_VAL = "false";
+
+ public HoodieClusteringConfig(Properties props) {
+ super(props);
+ }
+
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ public static class Builder {
+
+ private final Properties props = new Properties();
+
+ public Builder fromFile(File propertiesFile) throws IOException {
+ try (FileReader reader = new FileReader(propertiesFile)) {
+ this.props.load(reader);
+ return this;
+ }
+ }
+
+ public Builder withClusteringPlanStrategyClass(String clusteringStrategyClass) {
+ props.setProperty(CLUSTERING_PLAN_STRATEGY_CLASS, clusteringStrategyClass);
+ return this;
+ }
+
+ public Builder withClusteringExecutionStrategyClass(String runClusteringStrategyClass) {
+ props.setProperty(CLUSTERING_EXECUTION_STRATEGY_CLASS, runClusteringStrategyClass);
+ return this;
+ }
+
+ public Builder withClusteringTargetPartitions(int clusteringTargetPartitions) {
+ props.setProperty(CLUSTERING_TARGET_PARTITIONS, String.valueOf(clusteringTargetPartitions));
+ return this;
+ }
+
+ public Builder withClusteringPlanSmallFileLimit(long clusteringSmallFileLimit) {
+ props.setProperty(CLUSTERING_PLAN_SMALL_FILE_LIMIT, String.valueOf(clusteringSmallFileLimit));
+ return this;
+ }
+
+ public Builder withClusteringSortColumns(String sortColumns) {
+ props.setProperty(CLUSTERING_SORT_COLUMNS_PROPERTY, sortColumns);
+ return this;
+ }
+
+ public Builder withClusteringMaxBytesInGroup(long clusteringMaxGroupSize) {
+ props.setProperty(CLUSTERING_MAX_BYTES_PER_GROUP, String.valueOf(clusteringMaxGroupSize));
+ return this;
+ }
+
+ public Builder withClusteringMaxNumGroups(int maxNumGroups) {
+ props.setProperty(CLUSTERING_MAX_NUM_GROUPS, String.valueOf(maxNumGroups));
+ return this;
+ }
+
+ public Builder withClusteringTargetFileMaxBytes(long targetFileSize) {
+ props.setProperty(CLUSTERING_TARGET_FILE_MAX_BYTES, String.valueOf(targetFileSize));
+ return this;
+ }
+
+ public Builder withInlineClustering(Boolean inlineClustering) {
+ props.setProperty(INLINE_CLUSTERING_PROP, String.valueOf(inlineClustering));
+ return this;
+ }
+
+ public Builder withInlineClusteringNumCommits(int numCommits) {
+ props.setProperty(INLINE_CLUSTERING_MAX_COMMIT_PROP, String.valueOf(numCommits));
+ return this;
+ }
+
+ public Builder fromProperties(Properties props) {
+ this.props.putAll(props);
+ return this;
+ }
+
+ public Builder withClusteringUpdatesStrategy(String updatesStrategyClass) {
+ props.setProperty(CLUSTERING_UPDATES_STRATEGY_PROP, updatesStrategyClass);
+ return this;
+ }
+
+ public Builder withAsyncClustering(Boolean asyncClustering) {
+ props.setProperty(ASYNC_CLUSTERING_ENABLE_OPT_KEY, String.valueOf(asyncClustering));
+ return this;
+ }
+
+ public HoodieClusteringConfig build() {
+ HoodieClusteringConfig config = new HoodieClusteringConfig(props);
+
+ setDefaultOnCondition(props, !props.containsKey(CLUSTERING_PLAN_STRATEGY_CLASS),
+ CLUSTERING_PLAN_STRATEGY_CLASS, DEFAULT_CLUSTERING_PLAN_STRATEGY_CLASS);
+ setDefaultOnCondition(props, !props.containsKey(CLUSTERING_EXECUTION_STRATEGY_CLASS),
+ CLUSTERING_EXECUTION_STRATEGY_CLASS, DEFAULT_CLUSTERING_EXECUTION_STRATEGY_CLASS);
+ setDefaultOnCondition(props, !props.containsKey(CLUSTERING_MAX_BYTES_PER_GROUP), CLUSTERING_MAX_BYTES_PER_GROUP,
+ DEFAULT_CLUSTERING_MAX_GROUP_SIZE);
+ setDefaultOnCondition(props, !props.containsKey(CLUSTERING_MAX_NUM_GROUPS), CLUSTERING_MAX_NUM_GROUPS,
+ DEFAULT_CLUSTERING_MAX_NUM_GROUPS);
+ setDefaultOnCondition(props, !props.containsKey(CLUSTERING_TARGET_FILE_MAX_BYTES), CLUSTERING_TARGET_FILE_MAX_BYTES,
+ DEFAULT_CLUSTERING_TARGET_FILE_MAX_BYTES);
+ setDefaultOnCondition(props, !props.containsKey(INLINE_CLUSTERING_PROP), INLINE_CLUSTERING_PROP,
+ DEFAULT_INLINE_CLUSTERING);
+ setDefaultOnCondition(props, !props.containsKey(INLINE_CLUSTERING_MAX_COMMIT_PROP), INLINE_CLUSTERING_MAX_COMMIT_PROP,
+ DEFAULT_INLINE_CLUSTERING_NUM_COMMITS);
+ setDefaultOnCondition(props, !props.containsKey(CLUSTERING_TARGET_PARTITIONS), CLUSTERING_TARGET_PARTITIONS,
+ DEFAULT_CLUSTERING_TARGET_PARTITIONS);
+ setDefaultOnCondition(props, !props.containsKey(CLUSTERING_PLAN_SMALL_FILE_LIMIT), CLUSTERING_PLAN_SMALL_FILE_LIMIT,
+ DEFAULT_CLUSTERING_PLAN_SMALL_FILE_LIMIT);
+ setDefaultOnCondition(props, !props.containsKey(CLUSTERING_UPDATES_STRATEGY_PROP), CLUSTERING_UPDATES_STRATEGY_PROP,
+ DEFAULT_CLUSTERING_UPDATES_STRATEGY);
+ setDefaultOnCondition(props, !props.containsKey(ASYNC_CLUSTERING_ENABLE_OPT_KEY), ASYNC_CLUSTERING_ENABLE_OPT_KEY,
+ DEFAULT_ASYNC_CLUSTERING_ENABLE_OPT_VAL);
+ return config;
+ }
+ }
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java
index 83e9f678bc7ba..2fbd71dc6ab26 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java
@@ -20,6 +20,8 @@
import org.apache.hudi.common.bloom.BloomFilterTypeCode;
import org.apache.hudi.common.config.DefaultHoodieConfig;
+import org.apache.hudi.common.engine.EngineType;
+import org.apache.hudi.exception.HoodieNotSupportedException;
import org.apache.hudi.index.HoodieIndex;
import javax.annotation.concurrent.Immutable;
@@ -36,7 +38,6 @@
public class HoodieIndexConfig extends DefaultHoodieConfig {
public static final String INDEX_TYPE_PROP = "hoodie.index.type";
- public static final String DEFAULT_INDEX_TYPE = HoodieIndex.IndexType.BLOOM.name();
public static final String INDEX_CLASS_PROP = "hoodie.index.class";
public static final String DEFAULT_INDEX_CLASS = "";
@@ -103,8 +104,18 @@ public class HoodieIndexConfig extends DefaultHoodieConfig {
public static final String SIMPLE_INDEX_UPDATE_PARTITION_PATH = "hoodie.simple.index.update.partition.path";
public static final String DEFAULT_SIMPLE_INDEX_UPDATE_PARTITION_PATH = "false";
+ private EngineType engineType;
+
+ /**
+ * Use Spark engine by default.
+ */
private HoodieIndexConfig(Properties props) {
+ this(EngineType.SPARK, props);
+ }
+
+ private HoodieIndexConfig(EngineType engineType, Properties props) {
super(props);
+ this.engineType = engineType;
}
public static HoodieIndexConfig.Builder newBuilder() {
@@ -113,6 +124,7 @@ public static HoodieIndexConfig.Builder newBuilder() {
public static class Builder {
+ private EngineType engineType = EngineType.SPARK;
private final Properties props = new Properties();
public Builder fromFile(File propertiesFile) throws IOException {
@@ -237,9 +249,14 @@ public Builder withGlobalSimpleIndexUpdatePartitionPath(boolean updatePartitionP
return this;
}
+ public Builder withEngineType(EngineType engineType) {
+ this.engineType = engineType;
+ return this;
+ }
+
public HoodieIndexConfig build() {
- HoodieIndexConfig config = new HoodieIndexConfig(props);
- setDefaultOnCondition(props, !props.containsKey(INDEX_TYPE_PROP), INDEX_TYPE_PROP, DEFAULT_INDEX_TYPE);
+ HoodieIndexConfig config = new HoodieIndexConfig(engineType, props);
+ setDefaultOnCondition(props, !props.containsKey(INDEX_TYPE_PROP), INDEX_TYPE_PROP, getDefaultIndexType(engineType));
setDefaultOnCondition(props, !props.containsKey(INDEX_CLASS_PROP), INDEX_CLASS_PROP, DEFAULT_INDEX_CLASS);
setDefaultOnCondition(props, !props.containsKey(BLOOM_FILTER_NUM_ENTRIES), BLOOM_FILTER_NUM_ENTRIES,
DEFAULT_BLOOM_FILTER_NUM_ENTRIES);
@@ -260,9 +277,9 @@ public HoodieIndexConfig build() {
BLOOM_INDEX_BUCKETIZED_CHECKING_PROP, DEFAULT_BLOOM_INDEX_BUCKETIZED_CHECKING);
setDefaultOnCondition(props, !props.containsKey(BLOOM_INDEX_KEYS_PER_BUCKET_PROP),
BLOOM_INDEX_KEYS_PER_BUCKET_PROP, DEFAULT_BLOOM_INDEX_KEYS_PER_BUCKET);
- setDefaultOnCondition(props, !props.contains(BLOOM_INDEX_FILTER_TYPE),
+ setDefaultOnCondition(props, !props.containsKey(BLOOM_INDEX_FILTER_TYPE),
BLOOM_INDEX_FILTER_TYPE, DEFAULT_BLOOM_INDEX_FILTER_TYPE);
- setDefaultOnCondition(props, !props.contains(HOODIE_BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES),
+ setDefaultOnCondition(props, !props.containsKey(HOODIE_BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES),
HOODIE_BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES, DEFAULT_HOODIE_BLOOM_INDEX_FILTER_DYNAMIC_MAX_ENTRIES);
setDefaultOnCondition(props, !props.containsKey(SIMPLE_INDEX_PARALLELISM_PROP), SIMPLE_INDEX_PARALLELISM_PROP,
DEFAULT_SIMPLE_INDEX_PARALLELISM);
@@ -278,5 +295,20 @@ public HoodieIndexConfig build() {
HoodieIndex.IndexType.valueOf(props.getProperty(INDEX_TYPE_PROP));
return config;
}
+
+ private String getDefaultIndexType(EngineType engineType) {
+ switch (engineType) {
+ case SPARK:
+ return HoodieIndex.IndexType.BLOOM.name();
+ case FLINK:
+ return HoodieIndex.IndexType.INMEMORY.name();
+ default:
+ throw new HoodieNotSupportedException("Unsupported engine " + engineType);
+ }
+ }
+
+ public EngineType getEngineType() {
+ return engineType;
+ }
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java
index 800c75f824fd4..b6cb6e5add99d 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java
@@ -62,6 +62,9 @@ public class HoodieMetricsConfig extends DefaultHoodieConfig {
public static final String METRICS_REPORTER_CLASS = METRIC_PREFIX + ".reporter.class";
public static final String DEFAULT_METRICS_REPORTER_CLASS = "";
+ // Enable metrics collection from executors
+ public static final String ENABLE_EXECUTOR_METRICS = METRIC_PREFIX + ".executor.enable";
+
private HoodieMetricsConfig(Properties props) {
super(props);
}
@@ -126,6 +129,11 @@ public Builder withReporterClass(String className) {
return this;
}
+ public Builder withExecutorMetrics(boolean enable) {
+ props.setProperty(ENABLE_EXECUTOR_METRICS, String.valueOf(enable));
+ return this;
+ }
+
public HoodieMetricsConfig build() {
HoodieMetricsConfig config = new HoodieMetricsConfig(props);
setDefaultOnCondition(props, !props.containsKey(METRICS_ON), METRICS_ON, String.valueOf(DEFAULT_METRICS_ON));
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePayloadConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePayloadConfig.java
new file mode 100644
index 0000000000000..442bd02c67a25
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodiePayloadConfig.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.config;
+
+import org.apache.hudi.common.config.DefaultHoodieConfig;
+import org.apache.hudi.config.HoodieMemoryConfig.Builder;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.Properties;
+
+import static org.apache.hudi.common.model.HoodiePayloadProps.DEFAULT_PAYLOAD_ORDERING_FIELD_VAL;
+import static org.apache.hudi.common.model.HoodiePayloadProps.PAYLOAD_ORDERING_FIELD_PROP;
+
+/**
+ * Hoodie payload related configs.
+ */
+public class HoodiePayloadConfig extends DefaultHoodieConfig {
+
+ public HoodiePayloadConfig(Properties props) {
+ super(props);
+ }
+
+ public static HoodiePayloadConfig.Builder newBuilder() {
+ return new HoodiePayloadConfig.Builder();
+ }
+
+ public static class Builder {
+
+ private final Properties props = new Properties();
+
+ public Builder fromFile(File propertiesFile) throws IOException {
+ try (FileReader reader = new FileReader(propertiesFile)) {
+ this.props.load(reader);
+ return this;
+ }
+ }
+
+ public Builder fromProperties(Properties props) {
+ this.props.putAll(props);
+ return this;
+ }
+
+ public Builder withPayloadOrderingField(String payloadOrderingField) {
+ props.setProperty(PAYLOAD_ORDERING_FIELD_PROP, String.valueOf(payloadOrderingField));
+ return this;
+ }
+
+ public HoodiePayloadConfig build() {
+ HoodiePayloadConfig config = new HoodiePayloadConfig(props);
+ setDefaultOnCondition(props, !props.containsKey(PAYLOAD_ORDERING_FIELD_PROP), DEFAULT_PAYLOAD_ORDERING_FIELD_VAL,
+ String.valueOf(DEFAULT_PAYLOAD_ORDERING_FIELD_VAL));
+ return config;
+ }
+ }
+
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index 42d3e2b404568..d8135d44135b0 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -18,21 +18,25 @@
package org.apache.hudi.config;
-import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.client.bootstrap.BootstrapMode;
import org.apache.hudi.common.config.DefaultHoodieConfig;
+import org.apache.hudi.common.config.HoodieMetadataConfig;
+import org.apache.hudi.common.engine.EngineType;
import org.apache.hudi.common.fs.ConsistencyGuardConfig;
import org.apache.hudi.common.model.HoodieCleaningPolicy;
+import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.execution.bulkinsert.BulkInsertSortMode;
import org.apache.hudi.index.HoodieIndex;
+import org.apache.hudi.keygen.SimpleAvroKeyGenerator;
import org.apache.hudi.metrics.MetricsReporterType;
import org.apache.hudi.metrics.datadog.DatadogHttpClient.ApiSite;
import org.apache.hudi.table.action.compact.strategy.CompactionStrategy;
+import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import javax.annotation.concurrent.Immutable;
@@ -49,6 +53,7 @@
import java.util.function.Supplier;
import java.util.stream.Collectors;
+
/**
* Class storing configs for the HoodieWriteClient.
*/
@@ -58,6 +63,11 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
private static final long serialVersionUID = 0L;
public static final String TABLE_NAME = "hoodie.table.name";
+ public static final String PRECOMBINE_FIELD_PROP = "hoodie.datasource.write.precombine.field";
+ public static final String WRITE_PAYLOAD_CLASS = "hoodie.datasource.write.payload.class";
+ public static final String DEFAULT_WRITE_PAYLOAD_CLASS = OverwriteWithLatestAvroPayload.class.getName();
+ public static final String KEYGENERATOR_CLASS_PROP = "hoodie.datasource.write.keygenerator.class";
+ public static final String DEFAULT_KEYGENERATOR_CLASS = SimpleAvroKeyGenerator.class.getName();
public static final String DEFAULT_ROLLBACK_USING_MARKERS = "false";
public static final String ROLLBACK_USING_MARKERS = "hoodie.rollback.using.markers";
public static final String TIMELINE_LAYOUT_VERSION = "hoodie.timeline.layout.version";
@@ -69,6 +79,7 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
public static final String INSERT_PARALLELISM = "hoodie.insert.shuffle.parallelism";
public static final String BULKINSERT_PARALLELISM = "hoodie.bulkinsert.shuffle.parallelism";
public static final String BULKINSERT_USER_DEFINED_PARTITIONER_CLASS = "hoodie.bulkinsert.user.defined.partitioner.class";
+ public static final String BULKINSERT_INPUT_DATA_SCHEMA_DDL = "hoodie.bulkinsert.schema.ddl";
public static final String UPSERT_PARALLELISM = "hoodie.upsert.shuffle.parallelism";
public static final String DELETE_PARALLELISM = "hoodie.delete.shuffle.parallelism";
public static final String DEFAULT_ROLLBACK_PARALLELISM = "100";
@@ -85,8 +96,7 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
public static final String DEFAULT_WRITE_STATUS_STORAGE_LEVEL = "MEMORY_AND_DISK_SER";
public static final String HOODIE_AUTO_COMMIT_PROP = "hoodie.auto.commit";
public static final String DEFAULT_HOODIE_AUTO_COMMIT = "true";
- public static final String HOODIE_ASSUME_DATE_PARTITIONING_PROP = "hoodie.assume.date.partitioning";
- public static final String DEFAULT_ASSUME_DATE_PARTITIONING = "false";
+
public static final String HOODIE_WRITE_STATUS_CLASS_PROP = "hoodie.writestatus.class";
public static final String DEFAULT_HOODIE_WRITE_STATUS_CLASS = WriteStatus.class.getName();
public static final String FINALIZE_WRITE_PARALLELISM = "hoodie.finalize.write.parallelism";
@@ -117,13 +127,17 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
public static final String MAX_CONSISTENCY_CHECKS_PROP = "hoodie.consistency.check.max_checks";
public static int DEFAULT_MAX_CONSISTENCY_CHECKS = 7;
+ // Data validation check performed during merges before actual commits
+ private static final String MERGE_DATA_VALIDATION_CHECK_ENABLED = "hoodie.merge.data.validation.enabled";
+ private static final String DEFAULT_MERGE_DATA_VALIDATION_CHECK_ENABLED = "false";
+
/**
* HUDI-858 : There are users who had been directly using RDD APIs and have relied on a behavior in 0.4.x to allow
* multiple write operations (upsert/buk-insert/...) to be executed within a single commit.
- *
+ *
* Given Hudi commit protocol, these are generally unsafe operations and user need to handle failure scenarios. It
* only works with COW table. Hudi 0.5.x had stopped this behavior.
- *
+ *
* Given the importance of supporting such cases for the user's migration to 0.5.x, we are proposing a safety flag
* (disabled by default) which will allow this old behavior.
*/
@@ -140,14 +154,28 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
// We keep track of original config and rewritten config
private final FileSystemViewStorageConfig clientSpecifiedViewStorageConfig;
private FileSystemViewStorageConfig viewStorageConfig;
+ private HoodiePayloadConfig hoodiePayloadConfig;
+ private HoodieMetadataConfig metadataConfig;
+
+ private EngineType engineType;
+ /**
+ * Use Spark engine by default.
+ */
protected HoodieWriteConfig(Properties props) {
+ this(EngineType.SPARK, props);
+ }
+
+ protected HoodieWriteConfig(EngineType engineType, Properties props) {
super(props);
Properties newProps = new Properties();
newProps.putAll(props);
+ this.engineType = engineType;
this.consistencyGuardConfig = ConsistencyGuardConfig.newBuilder().fromProperties(newProps).build();
this.clientSpecifiedViewStorageConfig = FileSystemViewStorageConfig.newBuilder().fromProperties(newProps).build();
this.viewStorageConfig = clientSpecifiedViewStorageConfig;
+ this.hoodiePayloadConfig = HoodiePayloadConfig.newBuilder().fromProperties(newProps).build();
+ this.metadataConfig = HoodieMetadataConfig.newBuilder().fromProperties(props).build();
}
public static HoodieWriteConfig.Builder newBuilder() {
@@ -177,12 +205,24 @@ public String getTableName() {
return props.getProperty(TABLE_NAME);
}
+ public String getPreCombineField() {
+ return props.getProperty(PRECOMBINE_FIELD_PROP);
+ }
+
+ public String getWritePayloadClass() {
+ return props.getProperty(WRITE_PAYLOAD_CLASS);
+ }
+
+ public String getKeyGeneratorClass() {
+ return props.getProperty(KEYGENERATOR_CLASS_PROP);
+ }
+
public Boolean shouldAutoCommit() {
return Boolean.parseBoolean(props.getProperty(HOODIE_AUTO_COMMIT_PROP));
}
public Boolean shouldAssumeDatePartitioning() {
- return Boolean.parseBoolean(props.getProperty(HOODIE_ASSUME_DATE_PARTITIONING_PROP));
+ return metadataConfig.shouldAssumeDatePartitioning();
}
public boolean shouldUseExternalSchemaTransformation() {
@@ -217,6 +257,10 @@ public int getRollbackParallelism() {
return Integer.parseInt(props.getProperty(ROLLBACK_PARALLELISM));
}
+ public int getFileListingParallelism() {
+ return metadataConfig.getFileListingParallelism();
+ }
+
public boolean shouldRollbackUsingMarkers() {
return Boolean.parseBoolean(props.getProperty(ROLLBACK_USING_MARKERS));
}
@@ -282,6 +326,14 @@ public BulkInsertSortMode getBulkInsertSortMode() {
return BulkInsertSortMode.valueOf(sortMode.toUpperCase());
}
+ public boolean isMergeDataValidationCheckEnabled() {
+ return Boolean.parseBoolean(props.getProperty(MERGE_DATA_VALIDATION_CHECK_ENABLED));
+ }
+
+ public EngineType getEngineType() {
+ return engineType;
+ }
+
/**
* compaction properties.
*/
@@ -365,6 +417,23 @@ public Boolean getCompactionReverseLogReadEnabled() {
return Boolean.valueOf(props.getProperty(HoodieCompactionConfig.COMPACTION_REVERSE_LOG_READ_ENABLED_PROP));
}
+ public boolean isInlineClustering() {
+ return Boolean.parseBoolean(props.getProperty(HoodieClusteringConfig.INLINE_CLUSTERING_PROP));
+ }
+
+ public boolean isAsyncClusteringEnabled() {
+ return Boolean.parseBoolean(props.getProperty(HoodieClusteringConfig.ASYNC_CLUSTERING_ENABLE_OPT_KEY));
+ }
+
+ public boolean isClusteringEnabled() {
+ // TODO: future support async clustering
+ return isInlineClustering() || isAsyncClusteringEnabled();
+ }
+
+ public int getInlineClusterMaxCommits() {
+ return Integer.parseInt(props.getProperty(HoodieClusteringConfig.INLINE_CLUSTERING_MAX_COMMIT_PROP));
+ }
+
public String getPayloadClass() {
return props.getProperty(HoodieCompactionConfig.PAYLOAD_CLASS_PROP);
}
@@ -381,6 +450,45 @@ public Boolean shouldCleanBootstrapBaseFile() {
return Boolean.valueOf(props.getProperty(HoodieCompactionConfig.CLEANER_BOOTSTRAP_BASE_FILE_ENABLED));
}
+ public String getClusteringUpdatesStrategyClass() {
+ return props.getProperty(HoodieClusteringConfig.CLUSTERING_UPDATES_STRATEGY_PROP);
+ }
+
+ /**
+ * Clustering properties.
+ */
+ public String getClusteringPlanStrategyClass() {
+ return props.getProperty(HoodieClusteringConfig.CLUSTERING_PLAN_STRATEGY_CLASS);
+ }
+
+ public String getClusteringExecutionStrategyClass() {
+ return props.getProperty(HoodieClusteringConfig.CLUSTERING_EXECUTION_STRATEGY_CLASS);
+ }
+
+ public long getClusteringMaxBytesInGroup() {
+ return Long.parseLong(props.getProperty(HoodieClusteringConfig.CLUSTERING_MAX_BYTES_PER_GROUP));
+ }
+
+ public long getClusteringSmallFileLimit() {
+ return Long.parseLong(props.getProperty(HoodieClusteringConfig.CLUSTERING_PLAN_SMALL_FILE_LIMIT));
+ }
+
+ public int getClusteringMaxNumGroups() {
+ return Integer.parseInt(props.getProperty(HoodieClusteringConfig.CLUSTERING_MAX_NUM_GROUPS));
+ }
+
+ public long getClusteringTargetFileMaxBytes() {
+ return Long.parseLong(props.getProperty(HoodieClusteringConfig.CLUSTERING_TARGET_FILE_MAX_BYTES));
+ }
+
+ public int getTargetPartitionsForClustering() {
+ return Integer.parseInt(props.getProperty(HoodieClusteringConfig.CLUSTERING_TARGET_PARTITIONS));
+ }
+
+ public String getClusteringSortColumns() {
+ return props.getProperty(HoodieClusteringConfig.CLUSTERING_SORT_COLUMNS_PROPERTY);
+ }
+
/**
* index properties.
*/
@@ -587,6 +695,10 @@ public boolean isMetricsOn() {
return Boolean.parseBoolean(props.getProperty(HoodieMetricsConfig.METRICS_ON));
}
+ public boolean isExecutorMetricsEnabled() {
+ return Boolean.parseBoolean(props.getProperty(HoodieMetricsConfig.ENABLE_EXECUTOR_METRICS, "false"));
+ }
+
public MetricsReporterType getMetricsReporterType() {
return MetricsReporterType.valueOf(props.getProperty(HoodieMetricsConfig.METRICS_REPORTER_TYPE));
}
@@ -721,6 +833,14 @@ public FileSystemViewStorageConfig getClientSpecifiedViewStorageConfig() {
return clientSpecifiedViewStorageConfig;
}
+ public HoodiePayloadConfig getPayloadConfig() {
+ return hoodiePayloadConfig;
+ }
+
+ public HoodieMetadataConfig getMetadataConfig() {
+ return metadataConfig;
+ }
+
/**
* Commit call back configs.
*/
@@ -768,18 +888,62 @@ public Long getMaxMemoryPerPartitionMerge() {
return Long.valueOf(props.getProperty(HoodieMemoryConfig.MAX_MEMORY_FOR_MERGE_PROP));
}
+ /**
+ * File listing metadata configs.
+ */
+ public boolean useFileListingMetadata() {
+ return metadataConfig.useFileListingMetadata();
+ }
+
+ public boolean getFileListingMetadataVerify() {
+ return metadataConfig.validateFileListingMetadata();
+ }
+
+ public int getMetadataInsertParallelism() {
+ return Integer.parseInt(props.getProperty(HoodieMetadataConfig.METADATA_INSERT_PARALLELISM_PROP));
+ }
+
+ public int getMetadataCompactDeltaCommitMax() {
+ return Integer.parseInt(props.getProperty(HoodieMetadataConfig.METADATA_COMPACT_NUM_DELTA_COMMITS_PROP));
+ }
+
+ public boolean isMetadataAsyncClean() {
+ return Boolean.parseBoolean(props.getProperty(HoodieMetadataConfig.METADATA_ASYNC_CLEAN_PROP));
+ }
+
+ public int getMetadataMaxCommitsToKeep() {
+ return Integer.parseInt(props.getProperty(HoodieMetadataConfig.MAX_COMMITS_TO_KEEP_PROP));
+ }
+
+ public int getMetadataMinCommitsToKeep() {
+ return Integer.parseInt(props.getProperty(HoodieMetadataConfig.MIN_COMMITS_TO_KEEP_PROP));
+ }
+
+ public int getMetadataCleanerCommitsRetained() {
+ return Integer.parseInt(props.getProperty(HoodieMetadataConfig.CLEANER_COMMITS_RETAINED_PROP));
+ }
+
public static class Builder {
protected final Properties props = new Properties();
+ protected EngineType engineType = EngineType.SPARK;
private boolean isIndexConfigSet = false;
private boolean isStorageConfigSet = false;
private boolean isCompactionConfigSet = false;
+ private boolean isClusteringConfigSet = false;
private boolean isMetricsConfigSet = false;
private boolean isBootstrapConfigSet = false;
private boolean isMemoryConfigSet = false;
private boolean isViewConfigSet = false;
private boolean isConsistencyGuardSet = false;
private boolean isCallbackConfigSet = false;
+ private boolean isPayloadConfigSet = false;
+ private boolean isMetadataConfigSet = false;
+
+ public Builder withEngineType(EngineType engineType) {
+ this.engineType = engineType;
+ return this;
+ }
public Builder fromFile(File propertiesFile) throws IOException {
try (FileReader reader = new FileReader(propertiesFile)) {
@@ -822,6 +986,21 @@ public Builder forTable(String tableName) {
return this;
}
+ public Builder withPreCombineField(String preCombineField) {
+ props.setProperty(PRECOMBINE_FIELD_PROP, preCombineField);
+ return this;
+ }
+
+ public Builder withWritePayLoad(String payload) {
+ props.setProperty(WRITE_PAYLOAD_CLASS, payload);
+ return this;
+ }
+
+ public Builder withKeyGenerator(String keyGeneratorClass) {
+ props.setProperty(KEYGENERATOR_CLASS_PROP, keyGeneratorClass);
+ return this;
+ }
+
public Builder withTimelineLayoutVersion(int version) {
props.setProperty(TIMELINE_LAYOUT_VERSION, String.valueOf(version));
return this;
@@ -897,6 +1076,12 @@ public Builder withCompactionConfig(HoodieCompactionConfig compactionConfig) {
return this;
}
+ public Builder withClusteringConfig(HoodieClusteringConfig clusteringConfig) {
+ props.putAll(clusteringConfig.getProps());
+ isClusteringConfigSet = true;
+ return this;
+ }
+
public Builder withMetricsConfig(HoodieMetricsConfig metricsConfig) {
props.putAll(metricsConfig.getProps());
isMetricsConfigSet = true;
@@ -915,13 +1100,20 @@ public Builder withBootstrapConfig(HoodieBootstrapConfig bootstrapConfig) {
return this;
}
- public Builder withAutoCommit(boolean autoCommit) {
- props.setProperty(HOODIE_AUTO_COMMIT_PROP, String.valueOf(autoCommit));
+ public Builder withPayloadConfig(HoodiePayloadConfig payloadConfig) {
+ props.putAll(payloadConfig.getProps());
+ isPayloadConfigSet = true;
+ return this;
+ }
+
+ public Builder withMetadataConfig(HoodieMetadataConfig metadataConfig) {
+ props.putAll(metadataConfig.getProps());
+ isMetadataConfigSet = true;
return this;
}
- public Builder withAssumeDatePartitioning(boolean assumeDatePartitioning) {
- props.setProperty(HOODIE_ASSUME_DATE_PARTITIONING_PROP, String.valueOf(assumeDatePartitioning));
+ public Builder withAutoCommit(boolean autoCommit) {
+ props.setProperty(HOODIE_AUTO_COMMIT_PROP, String.valueOf(autoCommit));
return this;
}
@@ -983,6 +1175,11 @@ public Builder withExternalSchemaTrasformation(boolean enabled) {
return this;
}
+ public Builder withMergeDataValidationCheckEnabled(boolean enabled) {
+ props.setProperty(MERGE_DATA_VALIDATION_CHECK_ENABLED, String.valueOf(enabled));
+ return this;
+ }
+
public Builder withProperties(Properties properties) {
this.props.putAll(properties);
return this;
@@ -995,8 +1192,13 @@ protected void setDefaults() {
DEFAULT_PARALLELISM);
setDefaultOnCondition(props, !props.containsKey(UPSERT_PARALLELISM), UPSERT_PARALLELISM, DEFAULT_PARALLELISM);
setDefaultOnCondition(props, !props.containsKey(DELETE_PARALLELISM), DELETE_PARALLELISM, DEFAULT_PARALLELISM);
+
setDefaultOnCondition(props, !props.containsKey(ROLLBACK_PARALLELISM), ROLLBACK_PARALLELISM,
DEFAULT_ROLLBACK_PARALLELISM);
+ setDefaultOnCondition(props, !props.containsKey(KEYGENERATOR_CLASS_PROP),
+ KEYGENERATOR_CLASS_PROP, DEFAULT_KEYGENERATOR_CLASS);
+ setDefaultOnCondition(props, !props.containsKey(WRITE_PAYLOAD_CLASS),
+ WRITE_PAYLOAD_CLASS, DEFAULT_WRITE_PAYLOAD_CLASS);
setDefaultOnCondition(props, !props.containsKey(ROLLBACK_USING_MARKERS), ROLLBACK_USING_MARKERS,
DEFAULT_ROLLBACK_USING_MARKERS);
setDefaultOnCondition(props, !props.containsKey(COMBINE_BEFORE_INSERT_PROP), COMBINE_BEFORE_INSERT_PROP,
@@ -1011,8 +1213,6 @@ protected void setDefaults() {
DEFAULT_WRITE_STATUS_STORAGE_LEVEL);
setDefaultOnCondition(props, !props.containsKey(HOODIE_AUTO_COMMIT_PROP), HOODIE_AUTO_COMMIT_PROP,
DEFAULT_HOODIE_AUTO_COMMIT);
- setDefaultOnCondition(props, !props.containsKey(HOODIE_ASSUME_DATE_PARTITIONING_PROP),
- HOODIE_ASSUME_DATE_PARTITIONING_PROP, DEFAULT_ASSUME_DATE_PARTITIONING);
setDefaultOnCondition(props, !props.containsKey(HOODIE_WRITE_STATUS_CLASS_PROP), HOODIE_WRITE_STATUS_CLASS_PROP,
DEFAULT_HOODIE_WRITE_STATUS_CLASS);
setDefaultOnCondition(props, !props.containsKey(FINALIZE_WRITE_PARALLELISM), FINALIZE_WRITE_PARALLELISM,
@@ -1032,12 +1232,16 @@ protected void setDefaults() {
setDefaultOnCondition(props, !props.containsKey(AVRO_SCHEMA_VALIDATE), AVRO_SCHEMA_VALIDATE, DEFAULT_AVRO_SCHEMA_VALIDATE);
setDefaultOnCondition(props, !props.containsKey(BULKINSERT_SORT_MODE),
BULKINSERT_SORT_MODE, DEFAULT_BULKINSERT_SORT_MODE);
+ setDefaultOnCondition(props, !props.containsKey(MERGE_DATA_VALIDATION_CHECK_ENABLED),
+ MERGE_DATA_VALIDATION_CHECK_ENABLED, DEFAULT_MERGE_DATA_VALIDATION_CHECK_ENABLED);
// Make sure the props is propagated
- setDefaultOnCondition(props, !isIndexConfigSet, HoodieIndexConfig.newBuilder().fromProperties(props).build());
+ setDefaultOnCondition(props, !isIndexConfigSet, HoodieIndexConfig.newBuilder().withEngineType(engineType).fromProperties(props).build());
setDefaultOnCondition(props, !isStorageConfigSet, HoodieStorageConfig.newBuilder().fromProperties(props).build());
setDefaultOnCondition(props, !isCompactionConfigSet,
HoodieCompactionConfig.newBuilder().fromProperties(props).build());
+ setDefaultOnCondition(props, !isClusteringConfigSet,
+ HoodieClusteringConfig.newBuilder().fromProperties(props).build());
setDefaultOnCondition(props, !isMetricsConfigSet, HoodieMetricsConfig.newBuilder().fromProperties(props).build());
setDefaultOnCondition(props, !isBootstrapConfigSet,
HoodieBootstrapConfig.newBuilder().fromProperties(props).build());
@@ -1048,6 +1252,10 @@ protected void setDefaults() {
ConsistencyGuardConfig.newBuilder().fromProperties(props).build());
setDefaultOnCondition(props, !isCallbackConfigSet,
HoodieWriteCommitCallbackConfig.newBuilder().fromProperties(props).build());
+ setDefaultOnCondition(props, !isPayloadConfigSet,
+ HoodiePayloadConfig.newBuilder().fromProperties(props).build());
+ setDefaultOnCondition(props, !isMetadataConfigSet,
+ HoodieMetadataConfig.newBuilder().fromProperties(props).build());
setDefaultOnCondition(props, !props.containsKey(EXTERNAL_RECORD_AND_SCHEMA_TRANSFORMATION),
EXTERNAL_RECORD_AND_SCHEMA_TRANSFORMATION, DEFAULT_EXTERNAL_RECORD_AND_SCHEMA_TRANSFORMATION);
@@ -1066,7 +1274,7 @@ public HoodieWriteConfig build() {
setDefaults();
validate();
// Build WriteConfig at the end
- HoodieWriteConfig config = new HoodieWriteConfig(props);
+ HoodieWriteConfig config = new HoodieWriteConfig(engineType, props);
return config;
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/exception/HoodieClusteringException.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/exception/HoodieClusteringException.java
new file mode 100644
index 0000000000000..bb6aaa24777cf
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/exception/HoodieClusteringException.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.exception;
+
+public class HoodieClusteringException extends HoodieException {
+
+ public HoodieClusteringException(String msg) {
+ super(msg);
+ }
+
+ public HoodieClusteringException(String msg, Throwable e) {
+ super(msg, e);
+ }
+}
+
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/exception/HoodieClusteringUpdateException.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/exception/HoodieClusteringUpdateException.java
new file mode 100644
index 0000000000000..68b62a5421706
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/exception/HoodieClusteringUpdateException.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.exception;
+
+public class HoodieClusteringUpdateException extends HoodieException {
+ public HoodieClusteringUpdateException(String msg) {
+ super(msg);
+ }
+
+ public HoodieClusteringUpdateException(String msg, Throwable e) {
+ super(msg, e);
+ }
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/CopyOnWriteInsertHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/CopyOnWriteInsertHandler.java
index 8af72f351fff0..5e1f832b7f239 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/CopyOnWriteInsertHandler.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/CopyOnWriteInsertHandler.java
@@ -19,7 +19,7 @@
package org.apache.hudi.execution;
import org.apache.hudi.client.WriteStatus;
-import org.apache.hudi.client.common.TaskContextSupplier;
+import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer;
@@ -71,7 +71,7 @@ public CopyOnWriteInsertHandler(HoodieWriteConfig config, String instantTime,
public void consumeOneRecord(HoodieInsertValueGenResult payload) {
final HoodieRecord insertPayload = payload.record;
String partitionPath = insertPayload.getPartitionPath();
- HoodieWriteHandle handle = handles.get(partitionPath);
+ HoodieWriteHandle handle = handles.get(partitionPath);
if (handle == null) {
// If the records are sorted, this means that we encounter a new partition path
// and the records for the previous partition path are all written,
@@ -87,7 +87,7 @@ public void consumeOneRecord(HoodieInsertValueGenResult payload) {
if (!handle.canWrite(payload.record)) {
// Handle is full. Close the handle and add the WriteStatus
- statuses.add(handle.close());
+ statuses.addAll(handle.close());
// Open new handle
handle = writeHandleFactory.create(config, instantTime, hoodieTable,
insertPayload.getPartitionPath(), idPrefix, taskContextSupplier);
@@ -108,8 +108,8 @@ public List getResult() {
}
private void closeOpenHandles() {
- for (HoodieWriteHandle handle : handles.values()) {
- statuses.add(handle.close());
+ for (HoodieWriteHandle handle : handles.values()) {
+ statuses.addAll(handle.close());
}
handles.clear();
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/HoodieLazyInsertIterable.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/HoodieLazyInsertIterable.java
index b435c68de5e2b..cf544814cdba6 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/HoodieLazyInsertIterable.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/HoodieLazyInsertIterable.java
@@ -20,7 +20,7 @@
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.client.utils.LazyIterableIterator;
-import org.apache.hudi.client.common.TaskContextSupplier;
+import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.util.Option;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndex.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndex.java
index 6d04594cbab63..2e1915ff20431 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndex.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndex.java
@@ -21,7 +21,7 @@
import org.apache.hudi.ApiMaturityLevel;
import org.apache.hudi.PublicAPIClass;
import org.apache.hudi.PublicAPIMethod;
-import org.apache.hudi.client.common.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecordPayload;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java
index ad7807b707dcb..55be8fc6598af 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java
@@ -18,7 +18,7 @@
package org.apache.hudi.index;
-import org.apache.hudi.client.common.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordLocation;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/AppendHandleFactory.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/AppendHandleFactory.java
index 5c54dce31cd60..b4c83c141b2bc 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/AppendHandleFactory.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/AppendHandleFactory.java
@@ -18,7 +18,7 @@
package org.apache.hudi.io;
-import org.apache.hudi.client.common.TaskContextSupplier;
+import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/CreateHandleFactory.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/CreateHandleFactory.java
index 67ebadb2dd300..3a04c061a51a3 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/CreateHandleFactory.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/CreateHandleFactory.java
@@ -18,7 +18,7 @@
package org.apache.hudi.io;
-import org.apache.hudi.client.common.TaskContextSupplier;
+import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
index 0c590fe8818c4..c6ea7bab2ca7f 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
@@ -20,8 +20,9 @@
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.client.WriteStatus;
-import org.apache.hudi.client.common.TaskContextSupplier;
+import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.model.BaseFile;
import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieDeltaWriteStat;
import org.apache.hudi.common.model.HoodieKey;
@@ -30,9 +31,9 @@
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordLocation;
import org.apache.hudi.common.model.HoodieRecordPayload;
-import org.apache.hudi.common.model.HoodieWriteStat;
import org.apache.hudi.common.model.HoodieWriteStat.RuntimeStats;
import org.apache.hudi.common.model.IOType;
+import org.apache.hudi.common.table.log.AppendResult;
import org.apache.hudi.common.table.log.HoodieLogFormat;
import org.apache.hudi.common.table.log.HoodieLogFormat.Writer;
import org.apache.hudi.common.table.log.block.HoodieDataBlock;
@@ -42,6 +43,7 @@
import org.apache.hudi.common.table.view.TableFileSystemView.SliceView;
import org.apache.hudi.common.util.DefaultSizeEstimator;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.common.util.SizeEstimator;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieAppendException;
@@ -61,6 +63,7 @@
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
/**
* IO Operation to append data onto an existing file.
@@ -69,48 +72,49 @@ public class HoodieAppendHandle extends
private static final Logger LOG = LogManager.getLogger(HoodieAppendHandle.class);
// This acts as the sequenceID for records written
- private static AtomicLong recordIndex = new AtomicLong(1);
+ private static final AtomicLong RECORD_COUNTER = new AtomicLong(1);
+
private final String fileId;
// Buffer for holding records in memory before they are flushed to disk
- private List recordList = new ArrayList<>();
+ private final List recordList = new ArrayList<>();
// Buffer for holding records (to be deleted) in memory before they are flushed to disk
- private List keysToDelete = new ArrayList<>();
+ private final List keysToDelete = new ArrayList<>();
+ // Incoming records to be written to logs.
+ private final Iterator> recordItr;
+ // Writer to log into the file group's latest slice.
+ private Writer writer;
- private Iterator> recordItr;
+ private final List statuses;
// Total number of records written during an append
private long recordsWritten = 0;
// Total number of records deleted during an append
private long recordsDeleted = 0;
// Total number of records updated during an append
private long updatedRecordsWritten = 0;
+ // Total number of new records inserted into the delta file
+ private long insertRecordsWritten = 0;
+
// Average record size for a HoodieRecord. This size is updated at the end of every log block flushed to disk
private long averageRecordSize = 0;
- private HoodieLogFile currentLogFile;
- private Writer writer;
// Flag used to initialize some metadata
private boolean doInit = true;
// Total number of bytes written during this append phase (an estimation)
private long estimatedNumberOfBytesWritten;
- // Total number of bytes written to file
- private long sizeInBytes = 0;
// Number of records that must be written to meet the max block size for a log block
private int numberOfRecords = 0;
// Max block size to limit to for a log block
- private int maxBlockSize = config.getLogFileDataBlockMaxSize();
+ private final int maxBlockSize = config.getLogFileDataBlockMaxSize();
// Header metadata for a log block
- private Map header = new HashMap<>();
- // Total number of new records inserted into the delta file
- private long insertRecordsWritten = 0;
-
+ private final Map header = new HashMap<>();
private SizeEstimator sizeEstimator;
public HoodieAppendHandle(HoodieWriteConfig config, String instantTime, HoodieTable hoodieTable,
String partitionPath, String fileId, Iterator> recordItr, TaskContextSupplier taskContextSupplier) {
super(config, instantTime, partitionPath, fileId, hoodieTable, taskContextSupplier);
- writeStatus.setStat(new HoodieDeltaWriteStat());
this.fileId = fileId;
this.recordItr = recordItr;
sizeEstimator = new DefaultSizeEstimator();
+ this.statuses = new ArrayList<>();
}
public HoodieAppendHandle(HoodieWriteConfig config, String instantTime, HoodieTable hoodieTable,
@@ -125,20 +129,32 @@ private void init(HoodieRecord record) {
Option fileSlice = rtView.getLatestFileSlice(partitionPath, fileId);
// Set the base commit time as the current instantTime for new inserts into log files
String baseInstantTime;
+ String baseFile = "";
+ List logFiles = new ArrayList<>();
if (fileSlice.isPresent()) {
baseInstantTime = fileSlice.get().getBaseInstantTime();
+ baseFile = fileSlice.get().getBaseFile().map(BaseFile::getFileName).orElse("");
+ logFiles = fileSlice.get().getLogFiles().map(HoodieLogFile::getFileName).collect(Collectors.toList());
} else {
baseInstantTime = instantTime;
// This means there is no base data file, start appending to a new log file
fileSlice = Option.of(new FileSlice(partitionPath, baseInstantTime, this.fileId));
- LOG.info("New InsertHandle for partition :" + partitionPath);
+ LOG.info("New AppendHandle for partition :" + partitionPath);
}
- writeStatus.getStat().setPrevCommit(baseInstantTime);
+
+ // Prepare the first write status
+ writeStatus.setStat(new HoodieDeltaWriteStat());
writeStatus.setFileId(fileId);
writeStatus.setPartitionPath(partitionPath);
- writeStatus.getStat().setPartitionPath(partitionPath);
- writeStatus.getStat().setFileId(fileId);
averageRecordSize = sizeEstimator.sizeEstimate(record);
+
+ HoodieDeltaWriteStat deltaWriteStat = (HoodieDeltaWriteStat) writeStatus.getStat();
+ deltaWriteStat.setPrevCommit(baseInstantTime);
+ deltaWriteStat.setPartitionPath(partitionPath);
+ deltaWriteStat.setFileId(fileId);
+ deltaWriteStat.setBaseFile(baseFile);
+ deltaWriteStat.setLogFiles(logFiles);
+
try {
//save hoodie partition meta in the partition path
HoodiePartitionMetadata partitionMetadata = new HoodiePartitionMetadata(fs, baseInstantTime,
@@ -148,34 +164,29 @@ private void init(HoodieRecord record) {
// Since the actual log file written to can be different based on when rollover happens, we use the
// base file to denote some log appends happened on a slice. writeToken will still fence concurrent
// writers.
+ // https://issues.apache.org/jira/browse/HUDI-1517
createMarkerFile(partitionPath, FSUtils.makeDataFileName(baseInstantTime, writeToken, fileId, hoodieTable.getBaseFileExtension()));
this.writer = createLogWriter(fileSlice, baseInstantTime);
- this.currentLogFile = writer.getLogFile();
- ((HoodieDeltaWriteStat) writeStatus.getStat()).setLogVersion(currentLogFile.getLogVersion());
- ((HoodieDeltaWriteStat) writeStatus.getStat()).setLogOffset(writer.getCurrentSize());
} catch (Exception e) {
LOG.error("Error in update task at commit " + instantTime, e);
writeStatus.setGlobalError(e);
throw new HoodieUpsertException("Failed to initialize HoodieAppendHandle for FileId: " + fileId + " on commit "
+ instantTime + " on HDFS path " + hoodieTable.getMetaClient().getBasePath() + partitionPath, e);
}
- Path path = partitionPath.length() == 0 ? new Path(writer.getLogFile().getFileName())
- : new Path(partitionPath, writer.getLogFile().getFileName());
- writeStatus.getStat().setPath(path.toString());
doInit = false;
}
}
private Option getIndexedRecord(HoodieRecord hoodieRecord) {
- Option recordMetadata = hoodieRecord.getData().getMetadata();
+ Option> recordMetadata = hoodieRecord.getData().getMetadata();
try {
Option avroRecord = hoodieRecord.getData().getInsertValue(writerSchema);
if (avroRecord.isPresent()) {
// Convert GenericRecord to GenericRecord with hoodie commit metadata in schema
avroRecord = Option.of(rewriteRecord((GenericRecord) avroRecord.get()));
String seqId =
- HoodieRecord.generateSequenceId(instantTime, getPartitionId(), recordIndex.getAndIncrement());
+ HoodieRecord.generateSequenceId(instantTime, getPartitionId(), RECORD_COUNTER.getAndIncrement());
HoodieAvroUtils.addHoodieKeyToRecord((GenericRecord) avroRecord.get(), hoodieRecord.getRecordKey(),
hoodieRecord.getPartitionPath(), fileId);
HoodieAvroUtils.addCommitMetadataToRecord((GenericRecord) avroRecord.get(), instantTime, seqId);
@@ -203,6 +214,105 @@ private Option getIndexedRecord(HoodieRecord hoodieRecord) {
return Option.empty();
}
+ private void initNewStatus() {
+ HoodieDeltaWriteStat prevStat = (HoodieDeltaWriteStat) this.writeStatus.getStat();
+ // Make a new write status and copy basic fields over.
+ HoodieDeltaWriteStat stat = new HoodieDeltaWriteStat();
+ stat.setFileId(fileId);
+ stat.setPartitionPath(partitionPath);
+ stat.setPrevCommit(prevStat.getPrevCommit());
+ stat.setBaseFile(prevStat.getBaseFile());
+ stat.setLogFiles(new ArrayList<>(prevStat.getLogFiles()));
+
+ this.writeStatus = (WriteStatus) ReflectionUtils.loadClass(config.getWriteStatusClassName(),
+ !hoodieTable.getIndex().isImplicitWithStorage(), config.getWriteStatusFailureFraction());
+ this.writeStatus.setFileId(fileId);
+ this.writeStatus.setPartitionPath(partitionPath);
+ this.writeStatus.setStat(stat);
+ }
+
+ private String makeFilePath(HoodieLogFile logFile) {
+ return partitionPath.length() == 0
+ ? new Path(logFile.getFileName()).toString()
+ : new Path(partitionPath, logFile.getFileName()).toString();
+ }
+
+ private void resetWriteCounts() {
+ recordsWritten = 0;
+ updatedRecordsWritten = 0;
+ insertRecordsWritten = 0;
+ recordsDeleted = 0;
+ }
+
+ private void updateWriteCounts(HoodieDeltaWriteStat stat, AppendResult result) {
+ stat.setNumWrites(recordsWritten);
+ stat.setNumUpdateWrites(updatedRecordsWritten);
+ stat.setNumInserts(insertRecordsWritten);
+ stat.setNumDeletes(recordsDeleted);
+ stat.setTotalWriteBytes(result.size());
+ }
+
+ private void accumulateWriteCounts(HoodieDeltaWriteStat stat, AppendResult result) {
+ stat.setNumWrites(stat.getNumWrites() + recordsWritten);
+ stat.setNumUpdateWrites(stat.getNumUpdateWrites() + updatedRecordsWritten);
+ stat.setNumInserts(stat.getNumInserts() + insertRecordsWritten);
+ stat.setNumDeletes(stat.getNumDeletes() + recordsDeleted);
+ stat.setTotalWriteBytes(stat.getTotalWriteBytes() + result.size());
+ }
+
+ private void updateWriteStat(HoodieDeltaWriteStat stat, AppendResult result) {
+ stat.setPath(makeFilePath(result.logFile()));
+ stat.setLogOffset(result.offset());
+ stat.setLogVersion(result.logFile().getLogVersion());
+ if (!stat.getLogFiles().contains(result.logFile().getFileName())) {
+ stat.addLogFiles(result.logFile().getFileName());
+ }
+ }
+
+ private void updateRuntimeStats(HoodieDeltaWriteStat stat) {
+ RuntimeStats runtimeStats = new RuntimeStats();
+ runtimeStats.setTotalUpsertTime(timer.endTimer());
+ stat.setRuntimeStats(runtimeStats);
+ }
+
+ private void accumulateRuntimeStats(HoodieDeltaWriteStat stat) {
+ RuntimeStats runtimeStats = stat.getRuntimeStats();
+ assert runtimeStats != null;
+ runtimeStats.setTotalUpsertTime(runtimeStats.getTotalUpsertTime() + timer.endTimer());
+ }
+
+ private void updateWriteStatus(HoodieDeltaWriteStat stat, AppendResult result) {
+ updateWriteStat(stat, result);
+ updateWriteCounts(stat, result);
+ updateRuntimeStats(stat);
+ statuses.add(this.writeStatus);
+ }
+
+ private void processAppendResult(AppendResult result) {
+ HoodieDeltaWriteStat stat = (HoodieDeltaWriteStat) this.writeStatus.getStat();
+
+ if (stat.getPath() == null) {
+ // first time writing to this log block.
+ updateWriteStatus(stat, result);
+ } else if (stat.getPath().endsWith(result.logFile().getFileName())) {
+ // append/continued writing to the same log file
+ stat.setLogOffset(Math.min(stat.getLogOffset(), result.offset()));
+ accumulateWriteCounts(stat, result);
+ accumulateRuntimeStats(stat);
+ } else {
+ // written to a newer log file, due to rollover/otherwise.
+ initNewStatus();
+ stat = (HoodieDeltaWriteStat) this.writeStatus.getStat();
+ updateWriteStatus(stat, result);
+ }
+
+ resetWriteCounts();
+ assert stat.getRuntimeStats() != null;
+ LOG.info(String.format("AppendHandle for partitionPath %s filePath %s, took %d ms.", partitionPath,
+ stat.getPath(), stat.getRuntimeStats().getTotalUpsertTime()));
+ timer.startTimer();
+ }
+
public void doAppend() {
while (recordItr.hasNext()) {
HoodieRecord record = recordItr.next();
@@ -210,24 +320,30 @@ public void doAppend() {
flushToDiskIfRequired(record);
writeToBuffer(record);
}
- doAppend(header);
+ appendDataAndDeleteBlocks(header);
estimatedNumberOfBytesWritten += averageRecordSize * numberOfRecords;
}
- private void doAppend(Map header) {
+ private void appendDataAndDeleteBlocks(Map header) {
try {
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, instantTime);
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, writerSchemaWithMetafields.toString());
+ List blocks = new ArrayList<>(2);
if (recordList.size() > 0) {
- writer = writer.appendBlock(HoodieDataBlock.getBlock(hoodieTable.getLogDataBlockFormat(), recordList, header));
- recordList.clear();
+ blocks.add(HoodieDataBlock.getBlock(hoodieTable.getLogDataBlockFormat(), recordList, header));
}
if (keysToDelete.size() > 0) {
- writer = writer.appendBlock(new HoodieDeleteBlock(keysToDelete.toArray(new HoodieKey[keysToDelete.size()]), header));
+ blocks.add(new HoodieDeleteBlock(keysToDelete.toArray(new HoodieKey[keysToDelete.size()]), header));
+ }
+
+ if (blocks.size() > 0) {
+ AppendResult appendResult = writer.appendBlocks(blocks);
+ processAppendResult(appendResult);
+ recordList.clear();
keysToDelete.clear();
}
} catch (Exception e) {
- throw new HoodieAppendException("Failed while appending records to " + currentLogFile.getPath(), e);
+ throw new HoodieAppendException("Failed while appending records to " + writer.getLogFile().getPath(), e);
}
}
@@ -239,7 +355,7 @@ public boolean canWrite(HoodieRecord record) {
@Override
public void write(HoodieRecord record, Option insertValue) {
- Option recordMetadata = record.getData().getMetadata();
+ Option> recordMetadata = record.getData().getMetadata();
try {
init(record);
flushToDiskIfRequired(record);
@@ -253,48 +369,34 @@ public void write(HoodieRecord record, Option insertValue) {
}
@Override
- public WriteStatus close() {
+ public List close() {
try {
// flush any remaining records to disk
- doAppend(header);
-
+ appendDataAndDeleteBlocks(header);
if (writer != null) {
- sizeInBytes = writer.getCurrentSize();
writer.close();
- }
- HoodieWriteStat stat = writeStatus.getStat();
- stat.setFileId(this.fileId);
- stat.setNumWrites(recordsWritten);
- stat.setNumUpdateWrites(updatedRecordsWritten);
- stat.setNumInserts(insertRecordsWritten);
- stat.setNumDeletes(recordsDeleted);
- stat.setTotalWriteBytes(estimatedNumberOfBytesWritten);
- stat.setFileSizeInBytes(sizeInBytes);
- stat.setTotalWriteErrors(writeStatus.getTotalErrorRecords());
- RuntimeStats runtimeStats = new RuntimeStats();
- runtimeStats.setTotalUpsertTime(timer.endTimer());
- stat.setRuntimeStats(runtimeStats);
-
- LOG.info(String.format("AppendHandle for partitionPath %s fileID %s, took %d ms.", stat.getPartitionPath(),
- stat.getFileId(), runtimeStats.getTotalUpsertTime()));
-
- return writeStatus;
+ // update final size, once for all log files
+ for (WriteStatus status: statuses) {
+ long logFileSize = FSUtils.getFileSize(fs, new Path(config.getBasePath(), status.getStat().getPath()));
+ status.getStat().setFileSizeInBytes(logFileSize);
+ }
+ }
+ return statuses;
} catch (IOException e) {
throw new HoodieUpsertException("Failed to close UpdateHandle", e);
}
}
- @Override
- public WriteStatus getWriteStatus() {
- return writeStatus;
- }
-
@Override
public IOType getIOType() {
return IOType.APPEND;
}
+ public List writeStatuses() {
+ return statuses;
+ }
+
private Writer createLogWriter(Option fileSlice, String baseCommitTime)
throws IOException, InterruptedException {
Option latestLogFile = fileSlice.get().getLatestLogFile();
@@ -303,9 +405,10 @@ private Writer createLogWriter(Option fileSlice, String baseCommitTim
.onParentPath(FSUtils.getPartitionPath(hoodieTable.getMetaClient().getBasePath(), partitionPath))
.withFileId(fileId).overBaseCommit(baseCommitTime)
.withLogVersion(latestLogFile.map(HoodieLogFile::getLogVersion).orElse(HoodieLogFile.LOGFILE_BASE_VERSION))
+ .withFileSize(latestLogFile.map(HoodieLogFile::getFileSize).orElse(0L))
.withSizeThreshold(config.getLogFileMaxSize()).withFs(fs)
- .withLogWriteToken(latestLogFile.map(x -> FSUtils.getWriteTokenFromLogPath(x.getPath())).orElse(writeToken))
.withRolloverLogWriteToken(writeToken)
+ .withLogWriteToken(latestLogFile.map(x -> FSUtils.getWriteTokenFromLogPath(x.getPath())).orElse(writeToken))
.withFileExtension(HoodieLogFile.DELTA_EXTENSION).build();
}
@@ -340,7 +443,7 @@ private void flushToDiskIfRequired(HoodieRecord record) {
// avg of new and old
LOG.info("AvgRecordSize => " + averageRecordSize);
averageRecordSize = (averageRecordSize + sizeEstimator.sizeEstimate(record)) / 2;
- doAppend(header);
+ appendDataAndDeleteBlocks(header);
estimatedNumberOfBytesWritten += averageRecordSize * numberOfRecords;
numberOfRecords = 0;
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieBootstrapHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieBootstrapHandle.java
index b2eaedd53ce7d..d91f5c805203e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieBootstrapHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieBootstrapHandle.java
@@ -19,7 +19,7 @@
package org.apache.hudi.io;
import org.apache.hudi.avro.HoodieAvroUtils;
-import org.apache.hudi.client.common.TaskContextSupplier;
+import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.util.collection.Pair;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
index 6a8e7735093d5..8fbd1be247217 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
@@ -20,7 +20,7 @@
import org.apache.avro.Schema;
import org.apache.hudi.client.WriteStatus;
-import org.apache.hudi.client.common.TaskContextSupplier;
+import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodiePartitionMetadata;
import org.apache.hudi.common.model.HoodieRecord;
@@ -44,7 +44,9 @@
import org.apache.log4j.Logger;
import java.io.IOException;
+import java.util.Collections;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
public class HoodieCreateHandle extends HoodieWriteHandle {
@@ -162,11 +164,6 @@ public void write() {
}
}
- @Override
- public WriteStatus getWriteStatus() {
- return writeStatus;
- }
-
@Override
public IOType getIOType() {
return IOType.CREATE;
@@ -176,9 +173,8 @@ public IOType getIOType() {
* Performs actions to durably, persist the current changes and returns a WriteStatus object.
*/
@Override
- public WriteStatus close() {
- LOG
- .info("Closing the file " + writeStatus.getFileId() + " as we are done with all the records " + recordsWritten);
+ public List close() {
+ LOG.info("Closing the file " + writeStatus.getFileId() + " as we are done with all the records " + recordsWritten);
try {
fileWriter.close();
@@ -203,7 +199,7 @@ public WriteStatus close() {
LOG.info(String.format("CreateHandle for partitionPath %s fileID %s, took %d ms.", stat.getPartitionPath(),
stat.getFileId(), runtimeStats.getTotalCreateTime()));
- return writeStatus;
+ return Collections.singletonList(writeStatus);
} catch (IOException e) {
throw new HoodieInsertException("Failed to close the Insert Handle for path " + path, e);
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLocationFetchHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLocationFetchHandle.java
index 9194fc042f988..a5bc6b2fe55d2 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLocationFetchHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLocationFetchHandle.java
@@ -29,9 +29,7 @@
import org.apache.hadoop.fs.Path;
-import java.util.Iterator;
-
-import scala.Tuple2;
+import java.util.stream.Stream;
/**
* {@link HoodieRecordLocation} fetch handle for all records from {@link HoodieBaseFile} of interest.
@@ -48,10 +46,10 @@ public HoodieKeyLocationFetchHandle(HoodieWriteConfig config, HoodieTable> locations() {
+ public Stream> locations() {
HoodieBaseFile baseFile = partitionPathBaseFilePair.getRight();
return ParquetUtils.fetchRecordKeyPartitionPathFromParquet(hoodieTable.getHadoopConf(), new Path(baseFile.getPath())).stream()
- .map(entry -> new Tuple2<>(entry,
- new HoodieRecordLocation(baseFile.getCommitTime(), baseFile.getFileId()))).iterator();
+ .map(entry -> Pair.of(entry,
+ new HoodieRecordLocation(baseFile.getCommitTime(), baseFile.getFileId())));
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
index be324a3c5081a..4f5b82a67d46d 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
@@ -19,7 +19,7 @@
package org.apache.hudi.io;
import org.apache.hudi.client.WriteStatus;
-import org.apache.hudi.client.common.TaskContextSupplier;
+import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodiePartitionMetadata;
@@ -34,8 +34,11 @@
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ExternalSpillableMap;
import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieCorruptedDataException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieUpsertException;
+import org.apache.hudi.io.storage.HoodieFileReader;
+import org.apache.hudi.io.storage.HoodieFileReaderFactory;
import org.apache.hudi.io.storage.HoodieFileWriter;
import org.apache.hudi.table.HoodieTable;
@@ -47,8 +50,10 @@
import org.apache.log4j.Logger;
import java.io.IOException;
+import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -222,7 +227,8 @@ public void write(GenericRecord oldRecord) {
HoodieRecord hoodieRecord = new HoodieRecord<>(keyToNewRecords.get(key));
try {
Option combinedAvroRecord =
- hoodieRecord.getData().combineAndGetUpdateValue(oldRecord, useWriterSchema ? writerSchemaWithMetafields : writerSchema);
+ hoodieRecord.getData().combineAndGetUpdateValue(oldRecord, useWriterSchema ? writerSchemaWithMetafields : writerSchema,
+ config.getPayloadConfig().getProps());
if (writeUpdateRecord(hoodieRecord, combinedAvroRecord)) {
/*
* ONLY WHEN 1) we have an update for this key AND 2) We are able to successfully write the the combined new
@@ -241,14 +247,11 @@ public void write(GenericRecord oldRecord) {
if (copyOldRecord) {
// this should work as it is, since this is an existing record
- String errMsg = "Failed to merge old record into new file for key " + key + " from old file " + getOldFilePath()
- + " to new file " + newFilePath + " with writerSchema " + writerSchemaWithMetafields.toString(true);
try {
fileWriter.writeAvro(key, oldRecord);
- } catch (ClassCastException e) {
- LOG.debug("Old record is " + oldRecord);
- throw new HoodieUpsertException(errMsg, e);
} catch (IOException | RuntimeException e) {
+ String errMsg = String.format("Failed to merge old record into new file for key %s from old file %s to new file %s with writerSchema %s",
+ key, getOldFilePath(), newFilePath, writerSchemaWithMetafields.toString(true));
LOG.debug("Old record is " + oldRecord);
throw new HoodieUpsertException(errMsg, e);
}
@@ -257,7 +260,7 @@ public void write(GenericRecord oldRecord) {
}
@Override
- public WriteStatus close() {
+ public List close() {
try {
// write out any pending records (this can happen when inserts are turned into updates)
Iterator> newRecordsItr = (keyToNewRecords instanceof ExternalSpillableMap)
@@ -295,22 +298,41 @@ public WriteStatus close() {
runtimeStats.setTotalUpsertTime(timer.endTimer());
stat.setRuntimeStats(runtimeStats);
+ performMergeDataValidationCheck(writeStatus);
+
LOG.info(String.format("MergeHandle for partitionPath %s fileID %s, took %d ms.", stat.getPartitionPath(),
stat.getFileId(), runtimeStats.getTotalUpsertTime()));
- return writeStatus;
+ return Collections.singletonList(writeStatus);
} catch (IOException e) {
throw new HoodieUpsertException("Failed to close UpdateHandle", e);
}
}
- public Path getOldFilePath() {
- return oldFilePath;
+ public void performMergeDataValidationCheck(WriteStatus writeStatus) {
+ if (!config.isMergeDataValidationCheckEnabled()) {
+ return;
+ }
+
+ long oldNumWrites = 0;
+ try {
+ HoodieFileReader reader = HoodieFileReaderFactory.getFileReader(hoodieTable.getHadoopConf(), oldFilePath);
+ oldNumWrites = reader.getTotalRecords();
+ } catch (IOException e) {
+ throw new HoodieUpsertException("Failed to check for merge data validation", e);
+ }
+
+ if ((writeStatus.getStat().getNumWrites() + writeStatus.getStat().getNumDeletes()) < oldNumWrites) {
+ throw new HoodieCorruptedDataException(
+ String.format("Record write count decreased for file: %s, Partition Path: %s (%s:%d + %d < %s:%d)",
+ writeStatus.getFileId(), writeStatus.getPartitionPath(),
+ instantTime, writeStatus.getStat().getNumWrites(), writeStatus.getStat().getNumDeletes(),
+ FSUtils.getCommitTime(oldFilePath.toString()), oldNumWrites));
+ }
}
- @Override
- public WriteStatus getWriteStatus() {
- return writeStatus;
+ public Path getOldFilePath() {
+ return oldFilePath;
}
@Override
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandle.java
index 71610b1aa9c2f..1eee1046337a6 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandle.java
@@ -19,7 +19,7 @@
package org.apache.hudi.io;
import org.apache.hudi.client.WriteStatus;
-import org.apache.hudi.client.common.TaskContextSupplier;
+import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
@@ -31,6 +31,7 @@
import java.io.IOException;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.PriorityQueue;
import java.util.Queue;
@@ -101,7 +102,7 @@ public void write(GenericRecord oldRecord) {
}
@Override
- public WriteStatus close() {
+ public List close() {
// write out any pending records (this can happen when inserts are turned into updates)
newRecordKeysSorted.stream().forEach(key -> {
try {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
index 85898bccc7052..d2e538e2b661c 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
@@ -20,7 +20,7 @@
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.client.WriteStatus;
-import org.apache.hudi.client.common.TaskContextSupplier;
+import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
@@ -45,6 +45,8 @@
import org.apache.log4j.Logger;
import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
/**
* Base class for all write operations logically performed at the file group level.
@@ -56,7 +58,7 @@ public abstract class HoodieWriteHandle
protected final Schema writerSchema;
protected final Schema writerSchemaWithMetafields;
protected HoodieTimer timer;
- protected final WriteStatus writeStatus;
+ protected WriteStatus writeStatus;
protected final String partitionPath;
protected final String fileId;
protected final String writeToken;
@@ -167,9 +169,15 @@ protected GenericRecord rewriteRecord(GenericRecord record) {
return HoodieAvroUtils.rewriteRecord(record, writerSchemaWithMetafields);
}
- public abstract WriteStatus close();
+ public abstract List close();
- public abstract WriteStatus getWriteStatus();
+ public List writeStatuses() {
+ return Collections.singletonList(writeStatus);
+ }
+
+ public String getPartitionPath() {
+ return partitionPath;
+ }
public abstract IOType getIOType();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/IOUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/IOUtils.java
index 03719157fd998..6b5891c86b2f4 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/IOUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/IOUtils.java
@@ -18,8 +18,8 @@
package org.apache.hudi.io;
-import org.apache.hudi.client.common.EngineProperty;
-import org.apache.hudi.client.common.TaskContextSupplier;
+import org.apache.hudi.common.engine.EngineProperty;
+import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.util.Option;
import java.util.Properties;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/WriteHandleFactory.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/WriteHandleFactory.java
index c66442a48d5ad..d2923d4478cd9 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/WriteHandleFactory.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/WriteHandleFactory.java
@@ -18,7 +18,7 @@
package org.apache.hudi.io;
-import org.apache.hudi.client.common.TaskContextSupplier;
+import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieFileWriterFactory.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieFileWriterFactory.java
index 5f4eec0887003..e1e8c5e40273d 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieFileWriterFactory.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieFileWriterFactory.java
@@ -19,9 +19,9 @@
package org.apache.hudi.io.storage;
import org.apache.hudi.avro.HoodieAvroWriteSupport;
-import org.apache.hudi.client.common.TaskContextSupplier;
import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.bloom.BloomFilterFactory;
+import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.config.HoodieWriteConfig;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieHFileWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieHFileWriter.java
index 3684f9d4cd931..1358c35be2e59 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieHFileWriter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieHFileWriter.java
@@ -19,8 +19,8 @@
package org.apache.hudi.io.storage;
import org.apache.hudi.avro.HoodieAvroUtils;
-import org.apache.hudi.client.common.TaskContextSupplier;
import org.apache.hudi.common.bloom.BloomFilter;
+import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.fs.HoodieWrapperFileSystem;
import org.apache.hudi.common.model.HoodieRecord;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieParquetWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieParquetWriter.java
index 166e2bc42ab98..d1c695a45b012 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieParquetWriter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieParquetWriter.java
@@ -20,7 +20,7 @@
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.avro.HoodieAvroWriteSupport;
-import org.apache.hudi.client.common.TaskContextSupplier;
+import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.fs.HoodieWrapperFileSystem;
import org.apache.hudi.common.model.HoodieRecord;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/CustomAvroKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/CustomAvroKeyGenerator.java
index 6266fd15c6b84..724cabdc73964 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/CustomAvroKeyGenerator.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/CustomAvroKeyGenerator.java
@@ -88,7 +88,7 @@ public String getPartitionPath(GenericRecord record) {
try {
partitionPath.append(new TimestampBasedAvroKeyGenerator(config, partitionPathField).getPartitionPath(record));
} catch (IOException e) {
- throw new HoodieKeyGeneratorException("Unable to initialise TimestampBasedKeyGenerator class");
+ throw new HoodieKeyGeneratorException("Unable to initialise TimestampBasedKeyGenerator class", e);
}
break;
default:
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
new file mode 100644
index 0000000000000..1f76a5e30b803
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -0,0 +1,467 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.metadata;
+
+import org.apache.hudi.avro.model.HoodieCleanMetadata;
+import org.apache.hudi.avro.model.HoodieCleanerPlan;
+import org.apache.hudi.avro.model.HoodieMetadataRecord;
+import org.apache.hudi.avro.model.HoodieRestoreMetadata;
+import org.apache.hudi.avro.model.HoodieRollbackMetadata;
+import org.apache.hudi.common.config.HoodieMetadataConfig;
+import org.apache.hudi.common.config.SerializableConfiguration;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.fs.ConsistencyGuardConfig;
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.model.HoodieCleaningPolicy;
+import org.apache.hudi.common.model.HoodieCommitMetadata;
+import org.apache.hudi.common.model.HoodieFileFormat;
+import org.apache.hudi.common.model.HoodiePartitionMetadata;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.model.HoodieWriteStat;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
+import org.apache.hudi.common.util.HoodieTimer;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodieCompactionConfig;
+import org.apache.hudi.config.HoodieMetricsConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.exception.HoodieMetadataException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.apache.hudi.metadata.HoodieTableMetadata.METADATA_TABLE_NAME_SUFFIX;
+import static org.apache.hudi.metadata.HoodieTableMetadata.SOLO_COMMIT_TIMESTAMP;
+
+/**
+ * Writer implementation backed by an internal hudi table. Partition and file listing are saved within an internal MOR table
+ * called Metadata Table. This table is created by listing files and partitions (first time)
+ * and kept in sync using the instants on the main dataset.
+ */
+public abstract class HoodieBackedTableMetadataWriter implements HoodieTableMetadataWriter {
+
+ private static final Logger LOG = LogManager.getLogger(HoodieBackedTableMetadataWriter.class);
+
+ protected HoodieWriteConfig metadataWriteConfig;
+ protected HoodieWriteConfig datasetWriteConfig;
+ protected String tableName;
+
+ protected HoodieBackedTableMetadata metadata;
+ protected HoodieTableMetaClient metaClient;
+ protected Option metrics;
+ protected boolean enabled;
+ protected SerializableConfiguration hadoopConf;
+ protected final transient HoodieEngineContext engineContext;
+
+ protected HoodieBackedTableMetadataWriter(Configuration hadoopConf, HoodieWriteConfig writeConfig, HoodieEngineContext engineContext) {
+ this.datasetWriteConfig = writeConfig;
+ this.engineContext = engineContext;
+ this.hadoopConf = new SerializableConfiguration(hadoopConf);
+
+ if (writeConfig.useFileListingMetadata()) {
+ this.tableName = writeConfig.getTableName() + METADATA_TABLE_NAME_SUFFIX;
+ this.metadataWriteConfig = createMetadataWriteConfig(writeConfig);
+ enabled = true;
+
+ // Inline compaction and auto clean is required as we dont expose this table outside
+ ValidationUtils.checkArgument(!this.metadataWriteConfig.isAutoClean(), "Cleaning is controlled internally for Metadata table.");
+ ValidationUtils.checkArgument(!this.metadataWriteConfig.isInlineCompaction(), "Compaction is controlled internally for metadata table.");
+ // Metadata Table cannot have metadata listing turned on. (infinite loop, much?)
+ ValidationUtils.checkArgument(this.metadataWriteConfig.shouldAutoCommit(), "Auto commit is required for Metadata Table");
+ ValidationUtils.checkArgument(!this.metadataWriteConfig.useFileListingMetadata(), "File listing cannot be used for Metadata Table");
+
+ initRegistry();
+ HoodieTableMetaClient datasetMetaClient = new HoodieTableMetaClient(hadoopConf, datasetWriteConfig.getBasePath());
+ initialize(engineContext, datasetMetaClient);
+ if (enabled) {
+ // This is always called even in case the table was created for the first time. This is because
+ // initFromFilesystem() does file listing and hence may take a long time during which some new updates
+ // may have occurred on the table. Hence, calling this always ensures that the metadata is brought in sync
+ // with the active timeline.
+ HoodieTimer timer = new HoodieTimer().startTimer();
+ syncFromInstants(datasetMetaClient);
+ metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.SYNC_STR, timer.endTimer()));
+ }
+ } else {
+ enabled = false;
+ this.metrics = Option.empty();
+ }
+ }
+
+ protected abstract void initRegistry();
+
+ /**
+ * Create a {@code HoodieWriteConfig} to use for the Metadata Table.
+ *
+ * @param writeConfig {@code HoodieWriteConfig} of the main dataset writer
+ */
+ private HoodieWriteConfig createMetadataWriteConfig(HoodieWriteConfig writeConfig) {
+ int parallelism = writeConfig.getMetadataInsertParallelism();
+
+ // Create the write config for the metadata table by borrowing options from the main write config.
+ HoodieWriteConfig.Builder builder = HoodieWriteConfig.newBuilder()
+ .withTimelineLayoutVersion(TimelineLayoutVersion.CURR_VERSION)
+ .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder()
+ .withConsistencyCheckEnabled(writeConfig.getConsistencyGuardConfig().isConsistencyCheckEnabled())
+ .withInitialConsistencyCheckIntervalMs(writeConfig.getConsistencyGuardConfig().getInitialConsistencyCheckIntervalMs())
+ .withMaxConsistencyCheckIntervalMs(writeConfig.getConsistencyGuardConfig().getMaxConsistencyCheckIntervalMs())
+ .withMaxConsistencyChecks(writeConfig.getConsistencyGuardConfig().getMaxConsistencyChecks())
+ .build())
+ .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build())
+ .withAutoCommit(true)
+ .withAvroSchemaValidate(true)
+ .withEmbeddedTimelineServerEnabled(false)
+ .withPath(HoodieTableMetadata.getMetadataTableBasePath(writeConfig.getBasePath()))
+ .withSchema(HoodieMetadataRecord.getClassSchema().toString())
+ .forTable(tableName)
+ .withCompactionConfig(HoodieCompactionConfig.newBuilder()
+ .withAsyncClean(writeConfig.isMetadataAsyncClean())
+ // we will trigger cleaning manually, to control the instant times
+ .withAutoClean(false)
+ .withCleanerParallelism(parallelism)
+ .withCleanerPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS)
+ .retainCommits(writeConfig.getMetadataCleanerCommitsRetained())
+ .archiveCommitsWith(writeConfig.getMetadataMinCommitsToKeep(), writeConfig.getMetadataMaxCommitsToKeep())
+ // we will trigger compaction manually, to control the instant times
+ .withInlineCompaction(false)
+ .withMaxNumDeltaCommitsBeforeCompaction(writeConfig.getMetadataCompactDeltaCommitMax()).build())
+ .withParallelism(parallelism, parallelism)
+ .withDeleteParallelism(parallelism)
+ .withRollbackParallelism(parallelism)
+ .withFinalizeWriteParallelism(parallelism);
+
+ if (writeConfig.isMetricsOn()) {
+ HoodieMetricsConfig.Builder metricsConfig = HoodieMetricsConfig.newBuilder()
+ .withReporterType(writeConfig.getMetricsReporterType().toString())
+ .withExecutorMetrics(writeConfig.isExecutorMetricsEnabled())
+ .on(true);
+ switch (writeConfig.getMetricsReporterType()) {
+ case GRAPHITE:
+ metricsConfig.onGraphitePort(writeConfig.getGraphiteServerPort())
+ .toGraphiteHost(writeConfig.getGraphiteServerHost())
+ .usePrefix(writeConfig.getGraphiteMetricPrefix());
+ break;
+ case JMX:
+ metricsConfig.onJmxPort(writeConfig.getJmxPort())
+ .toJmxHost(writeConfig.getJmxHost());
+ break;
+ case DATADOG:
+ // TODO:
+ break;
+ case CONSOLE:
+ case INMEMORY:
+ break;
+ default:
+ throw new HoodieMetadataException("Unsupported Metrics Reporter type " + writeConfig.getMetricsReporterType());
+ }
+
+ builder.withMetricsConfig(metricsConfig.build());
+ }
+
+ return builder.build();
+ }
+
+ public HoodieWriteConfig getWriteConfig() {
+ return metadataWriteConfig;
+ }
+
+ public HoodieBackedTableMetadata metadata() {
+ return metadata;
+ }
+
+ /**
+ * Initialize the metadata table if it does not exist. Update the metadata to bring it in sync with the file system.
+ *
+ * This can happen in two ways:
+ * 1. If the metadata table did not exist, then file and partition listing is used
+ * 2. If the metadata table exists, the instants from active timeline are read in order and changes applied
+ *
+ * The above logic has been chosen because it is faster to perform #1 at scale rather than read all the Instants
+ * which are large in size (AVRO or JSON encoded and not compressed) and incur considerable IO for de-serialization
+ * and decoding.
+ */
+ protected abstract void initialize(HoodieEngineContext engineContext, HoodieTableMetaClient datasetMetaClient);
+
+ protected void initTableMetadata() {
+ try {
+ if (this.metadata != null) {
+ this.metadata.close();
+ }
+ this.metadata = new HoodieBackedTableMetadata(engineContext, datasetWriteConfig.getMetadataConfig(),
+ datasetWriteConfig.getBasePath(), datasetWriteConfig.getSpillableMapBasePath());
+ this.metaClient = metadata.getMetaClient();
+ } catch (Exception e) {
+ throw new HoodieException("Error initializing metadata table for reads", e);
+ }
+ }
+
+ protected void bootstrapIfNeeded(HoodieEngineContext engineContext, HoodieTableMetaClient datasetMetaClient) throws IOException {
+ HoodieTimer timer = new HoodieTimer().startTimer();
+ boolean exists = datasetMetaClient.getFs().exists(new Path(metadataWriteConfig.getBasePath(), HoodieTableMetaClient.METAFOLDER_NAME));
+ if (!exists) {
+ // Initialize for the first time by listing partitions and files directly from the file system
+ bootstrapFromFilesystem(engineContext, datasetMetaClient);
+ metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.INITIALIZE_STR, timer.endTimer()));
+ }
+ }
+
+ /**
+ * Initialize the Metadata Table by listing files and partitions from the file system.
+ *
+ * @param datasetMetaClient {@code HoodieTableMetaClient} for the dataset
+ */
+ private void bootstrapFromFilesystem(HoodieEngineContext engineContext, HoodieTableMetaClient datasetMetaClient) throws IOException {
+ ValidationUtils.checkState(enabled, "Metadata table cannot be initialized as it is not enabled");
+
+ // If there is no commit on the dataset yet, use the SOLO_COMMIT_TIMESTAMP as the instant time for initial commit
+ // Otherwise, we use the timestamp of the instant which does not have any non-completed instants before it.
+ Option latestInstant = Option.empty();
+ boolean foundNonComplete = false;
+ for (HoodieInstant instant : datasetMetaClient.getActiveTimeline().getInstants().collect(Collectors.toList())) {
+ if (!instant.isCompleted()) {
+ foundNonComplete = true;
+ } else if (!foundNonComplete) {
+ latestInstant = Option.of(instant);
+ }
+ }
+
+ String createInstantTime = latestInstant.map(HoodieInstant::getTimestamp).orElse(SOLO_COMMIT_TIMESTAMP);
+ LOG.info("Creating a new metadata table in " + metadataWriteConfig.getBasePath() + " at instant " + createInstantTime);
+
+ HoodieTableMetaClient.initTableType(hadoopConf.get(), metadataWriteConfig.getBasePath(),
+ HoodieTableType.MERGE_ON_READ, tableName, "archived", HoodieMetadataPayload.class.getName(),
+ HoodieFileFormat.HFILE.toString());
+ initTableMetadata();
+
+ // List all partitions in the basePath of the containing dataset
+ LOG.info("Initializing metadata table by using file listings in " + datasetWriteConfig.getBasePath());
+ Map> partitionToFileStatus = getPartitionsToFilesMapping(datasetMetaClient);
+
+ // Create a HoodieCommitMetadata with writeStats for all discovered files
+ int[] stats = {0};
+ HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
+
+ partitionToFileStatus.forEach((partition, statuses) -> {
+ // Filter the statuses to only include files which were created before or on createInstantTime
+ statuses.stream().filter(status -> {
+ String filename = status.getPath().getName();
+ return !HoodieTimeline.compareTimestamps(FSUtils.getCommitTime(filename), HoodieTimeline.GREATER_THAN,
+ createInstantTime);
+ }).forEach(status -> {
+ HoodieWriteStat writeStat = new HoodieWriteStat();
+ writeStat.setPath(partition + Path.SEPARATOR + status.getPath().getName());
+ writeStat.setPartitionPath(partition);
+ writeStat.setTotalWriteBytes(status.getLen());
+ commitMetadata.addWriteStat(partition, writeStat);
+ stats[0] += 1;
+ });
+
+ // If the partition has no files then create a writeStat with no file path
+ if (commitMetadata.getWriteStats(partition) == null) {
+ HoodieWriteStat writeStat = new HoodieWriteStat();
+ writeStat.setPartitionPath(partition);
+ commitMetadata.addWriteStat(partition, writeStat);
+ }
+ });
+
+ LOG.info("Committing " + partitionToFileStatus.size() + " partitions and " + stats[0] + " files to metadata");
+ update(commitMetadata, createInstantTime);
+ }
+
+ /**
+ * Function to find hoodie partitions and list files in them in parallel.
+ *
+ * @param datasetMetaClient
+ * @return Map of partition names to a list of FileStatus for all the files in the partition
+ */
+ private Map> getPartitionsToFilesMapping(HoodieTableMetaClient datasetMetaClient) {
+ List pathsToList = new LinkedList<>();
+ pathsToList.add(new Path(datasetWriteConfig.getBasePath()));
+
+ Map> partitionToFileStatus = new HashMap<>();
+ final int fileListingParallelism = metadataWriteConfig.getFileListingParallelism();
+ SerializableConfiguration conf = new SerializableConfiguration(datasetMetaClient.getHadoopConf());
+
+ while (!pathsToList.isEmpty()) {
+ int listingParallelism = Math.min(fileListingParallelism, pathsToList.size());
+ // List all directories in parallel
+ List> dirToFileListing = engineContext.map(pathsToList, path -> {
+ FileSystem fs = path.getFileSystem(conf.get());
+ return Pair.of(path, fs.listStatus(path));
+ }, listingParallelism);
+ pathsToList.clear();
+
+ // If the listing reveals a directory, add it to queue. If the listing reveals a hoodie partition, add it to
+ // the results.
+ dirToFileListing.forEach(p -> {
+ List filesInDir = Arrays.stream(p.getRight()).parallel()
+ .filter(fs -> !fs.getPath().getName().equals(HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE))
+ .collect(Collectors.toList());
+
+ if (p.getRight().length > filesInDir.size()) {
+ // Is a partition. Add all data files to result.
+ String partitionName = FSUtils.getRelativePartitionPath(new Path(datasetMetaClient.getBasePath()), p.getLeft());
+ partitionToFileStatus.put(partitionName, filesInDir);
+ } else {
+ // Add sub-dirs to the queue
+ pathsToList.addAll(Arrays.stream(p.getRight())
+ .filter(fs -> fs.isDirectory() && !fs.getPath().getName().equals(HoodieTableMetaClient.METAFOLDER_NAME))
+ .map(fs -> fs.getPath())
+ .collect(Collectors.toList()));
+ }
+ });
+ }
+
+ return partitionToFileStatus;
+ }
+
+ /**
+ * Sync the Metadata Table from the instants created on the dataset.
+ *
+ * @param datasetMetaClient {@code HoodieTableMetaClient} for the dataset
+ */
+ private void syncFromInstants(HoodieTableMetaClient datasetMetaClient) {
+ ValidationUtils.checkState(enabled, "Metadata table cannot be synced as it is not enabled");
+ // (re) init the metadata for reading.
+ initTableMetadata();
+ try {
+ List instantsToSync = metadata.findInstantsToSync();
+ if (instantsToSync.isEmpty()) {
+ return;
+ }
+
+ LOG.info("Syncing " + instantsToSync.size() + " instants to metadata table: " + instantsToSync);
+
+ // Read each instant in order and sync it to metadata table
+ for (HoodieInstant instant : instantsToSync) {
+ LOG.info("Syncing instant " + instant + " to metadata table");
+
+ Option> records = HoodieTableMetadataUtil.convertInstantToMetaRecords(datasetMetaClient, instant, metadata.getSyncedInstantTime());
+ if (records.isPresent()) {
+ commit(records.get(), MetadataPartitionType.FILES.partitionPath(), instant.getTimestamp());
+ }
+ }
+ initTableMetadata();
+ } catch (IOException ioe) {
+ throw new HoodieIOException("Unable to sync instants from data to metadata table.", ioe);
+ }
+ }
+
+ /**
+ * Update from {@code HoodieCommitMetadata}.
+ *
+ * @param commitMetadata {@code HoodieCommitMetadata}
+ * @param instantTime Timestamp at which the commit was performed
+ */
+ @Override
+ public void update(HoodieCommitMetadata commitMetadata, String instantTime) {
+ if (enabled) {
+ List records = HoodieTableMetadataUtil.convertMetadataToRecords(commitMetadata, instantTime);
+ commit(records, MetadataPartitionType.FILES.partitionPath(), instantTime);
+ }
+ }
+
+ /**
+ * Update from {@code HoodieCleanerPlan}.
+ *
+ * @param cleanerPlan {@code HoodieCleanerPlan}
+ * @param instantTime Timestamp at which the clean plan was generated
+ */
+ @Override
+ public void update(HoodieCleanerPlan cleanerPlan, String instantTime) {
+ if (enabled) {
+ List records = HoodieTableMetadataUtil.convertMetadataToRecords(cleanerPlan, instantTime);
+ commit(records, MetadataPartitionType.FILES.partitionPath(), instantTime);
+ }
+ }
+
+ /**
+ * Update from {@code HoodieCleanMetadata}.
+ *
+ * @param cleanMetadata {@code HoodieCleanMetadata}
+ * @param instantTime Timestamp at which the clean was completed
+ */
+ @Override
+ public void update(HoodieCleanMetadata cleanMetadata, String instantTime) {
+ if (enabled) {
+ List records = HoodieTableMetadataUtil.convertMetadataToRecords(cleanMetadata, instantTime);
+ commit(records, MetadataPartitionType.FILES.partitionPath(), instantTime);
+ }
+ }
+
+ /**
+ * Update from {@code HoodieRestoreMetadata}.
+ *
+ * @param restoreMetadata {@code HoodieRestoreMetadata}
+ * @param instantTime Timestamp at which the restore was performed
+ */
+ @Override
+ public void update(HoodieRestoreMetadata restoreMetadata, String instantTime) {
+ if (enabled) {
+ List records = HoodieTableMetadataUtil.convertMetadataToRecords(restoreMetadata, instantTime, metadata.getSyncedInstantTime());
+ commit(records, MetadataPartitionType.FILES.partitionPath(), instantTime);
+ }
+ }
+
+ /**
+ * Update from {@code HoodieRollbackMetadata}.
+ *
+ * @param rollbackMetadata {@code HoodieRollbackMetadata}
+ * @param instantTime Timestamp at which the rollback was performed
+ */
+ @Override
+ public void update(HoodieRollbackMetadata rollbackMetadata, String instantTime) {
+ if (enabled) {
+ List records = HoodieTableMetadataUtil.convertMetadataToRecords(rollbackMetadata, instantTime, metadata.getSyncedInstantTime());
+ commit(records, MetadataPartitionType.FILES.partitionPath(), instantTime);
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (metadata != null) {
+ metadata.close();
+ }
+ }
+
+ /**
+ * Commit the {@code HoodieRecord}s to Metadata Table as a new delta-commit.
+ *
+ */
+ protected abstract void commit(List records, String partitionName, String instantTime);
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataWriter.java
new file mode 100644
index 0000000000000..02c5b9e646ad0
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataWriter.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.metadata;
+
+import org.apache.hudi.avro.model.HoodieCleanMetadata;
+import org.apache.hudi.avro.model.HoodieCleanerPlan;
+import org.apache.hudi.avro.model.HoodieRestoreMetadata;
+import org.apache.hudi.avro.model.HoodieRollbackMetadata;
+import org.apache.hudi.common.model.HoodieCommitMetadata;
+
+import java.io.Serializable;
+
+/**
+ * Interface that supports updating metadata for a given table, as actions complete.
+ */
+public interface HoodieTableMetadataWriter extends Serializable, AutoCloseable {
+
+ void update(HoodieCommitMetadata commitMetadata, String instantTime);
+
+ void update(HoodieCleanerPlan cleanerPlan, String instantTime);
+
+ void update(HoodieCleanMetadata cleanMetadata, String instantTime);
+
+ void update(HoodieRestoreMetadata restoreMetadata, String instantTime);
+
+ void update(HoodieRollbackMetadata rollbackMetadata, String instantTime);
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java
index 292039b83ad41..c8c112fc203ff 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java
@@ -37,6 +37,7 @@ public class HoodieMetrics {
public String cleanTimerName = null;
public String commitTimerName = null;
public String deltaCommitTimerName = null;
+ public String replaceCommitTimerName = null;
public String finalizeTimerName = null;
public String compactionTimerName = null;
public String indexTimerName = null;
@@ -48,6 +49,7 @@ public class HoodieMetrics {
private Timer deltaCommitTimer = null;
private Timer finalizeTimer = null;
private Timer compactionTimer = null;
+ private Timer clusteringTimer = null;
private Timer indexTimer = null;
public HoodieMetrics(HoodieWriteConfig config, String tableName) {
@@ -59,6 +61,7 @@ public HoodieMetrics(HoodieWriteConfig config, String tableName) {
this.cleanTimerName = getMetricsName("timer", HoodieTimeline.CLEAN_ACTION);
this.commitTimerName = getMetricsName("timer", HoodieTimeline.COMMIT_ACTION);
this.deltaCommitTimerName = getMetricsName("timer", HoodieTimeline.DELTA_COMMIT_ACTION);
+ this.replaceCommitTimerName = getMetricsName("timer", HoodieTimeline.REPLACE_COMMIT_ACTION);
this.finalizeTimerName = getMetricsName("timer", "finalize");
this.compactionTimerName = getMetricsName("timer", HoodieTimeline.COMPACTION_ACTION);
this.indexTimerName = getMetricsName("timer", "index");
@@ -83,6 +86,13 @@ public Timer.Context getCompactionCtx() {
return compactionTimer == null ? null : compactionTimer.time();
}
+ public Timer.Context getClusteringCtx() {
+ if (config.isMetricsOn() && clusteringTimer == null) {
+ clusteringTimer = createTimer(replaceCommitTimerName);
+ }
+ return clusteringTimer == null ? null : clusteringTimer.time();
+ }
+
public Timer.Context getCleanCtx() {
if (config.isMetricsOn() && cleanTimer == null) {
cleanTimer = createTimer(cleanTimerName);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
index 0413f75ce066d..74ffdfc653173 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
@@ -24,13 +24,16 @@
import org.apache.hadoop.fs.Path;
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.avro.model.HoodieCleanMetadata;
+import org.apache.hudi.avro.model.HoodieClusteringPlan;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
import org.apache.hudi.avro.model.HoodieRestoreMetadata;
import org.apache.hudi.avro.model.HoodieRollbackMetadata;
import org.apache.hudi.avro.model.HoodieSavepointMetadata;
-import org.apache.hudi.client.common.TaskContextSupplier;
-import org.apache.hudi.client.common.HoodieEngineContext;
+import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.common.config.SerializableConfiguration;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieLocalEngineContext;
+import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.fs.ConsistencyGuard;
import org.apache.hudi.common.fs.ConsistencyGuard.FileVisibility;
import org.apache.hudi.common.fs.ConsistencyGuardConfig;
@@ -47,6 +50,7 @@
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.view.FileSystemViewManager;
+import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
import org.apache.hudi.common.table.view.SyncableFileSystemView;
import org.apache.hudi.common.table.view.TableFileSystemView;
@@ -60,6 +64,7 @@
import org.apache.hudi.exception.HoodieInsertException;
import org.apache.hudi.exception.HoodieUpsertException;
import org.apache.hudi.index.HoodieIndex;
+import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.table.action.HoodieWriteMetadata;
import org.apache.hudi.table.action.bootstrap.HoodieBootstrapWriteMetadata;
import org.apache.log4j.LogManager;
@@ -90,17 +95,25 @@ public abstract class HoodieTable implem
protected final HoodieWriteConfig config;
protected final HoodieTableMetaClient metaClient;
protected final HoodieIndex index;
-
private SerializableConfiguration hadoopConfiguration;
- private transient FileSystemViewManager viewManager;
-
protected final TaskContextSupplier taskContextSupplier;
+ private final HoodieTableMetadata metadata;
+
+ private transient FileSystemViewManager viewManager;
+ protected final transient HoodieEngineContext context;
protected HoodieTable(HoodieWriteConfig config, HoodieEngineContext context, HoodieTableMetaClient metaClient) {
this.config = config;
this.hadoopConfiguration = context.getHadoopConf();
- this.viewManager = FileSystemViewManager.createViewManager(hadoopConfiguration,
- config.getViewStorageConfig());
+ this.context = context;
+
+ // disable reuse of resources, given there is no close() called on the executors ultimately
+ HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder().fromProperties(config.getMetadataConfig().getProps())
+ .enableReuse(false).build();
+ this.metadata = HoodieTableMetadata.create(context, metadataConfig, config.getBasePath(),
+ FileSystemViewStorageConfig.DEFAULT_VIEW_SPILLABLE_DIR);
+
+ this.viewManager = FileSystemViewManager.createViewManager(context, config.getMetadataConfig(), config.getViewStorageConfig(), () -> metadata);
this.metaClient = metaClient;
this.index = getIndex(config, context);
this.taskContextSupplier = context.getTaskContextSupplier();
@@ -110,7 +123,7 @@ protected HoodieTable(HoodieWriteConfig config, HoodieEngineContext context, Hoo
private synchronized FileSystemViewManager getViewManager() {
if (null == viewManager) {
- viewManager = FileSystemViewManager.createViewManager(hadoopConfiguration, config.getViewStorageConfig());
+ viewManager = FileSystemViewManager.createViewManager(getContext(), config.getMetadataConfig(), config.getViewStorageConfig(), () -> metadata);
}
return viewManager;
}
@@ -157,6 +170,15 @@ public abstract HoodieWriteMetadata bulkInsert(HoodieEngineContext context, S
*/
public abstract HoodieWriteMetadata delete(HoodieEngineContext context, String instantTime, K keys);
+ /**
+ * Deletes all data of partitions.
+ * @param context HoodieEngineContext
+ * @param instantTime Instant Time for the action
+ * @param partitions {@link List} of partition to be deleted
+ * @return HoodieWriteMetadata
+ */
+ public abstract HoodieWriteMetadata deletePartitions(HoodieEngineContext context, String instantTime, List partitions);
+
/**
* Upserts the given prepared records into the Hoodie table, at the supplied instantTime.
*
@@ -205,6 +227,17 @@ public abstract HoodieWriteMetadata bulkInsertPrepped(HoodieEngineContext con
*/
public abstract HoodieWriteMetadata insertOverwrite(HoodieEngineContext context, String instantTime, I records);
+ /**
+ * Delete all the existing records of the Hoodie table and inserts the specified new records into Hoodie table at the supplied instantTime,
+ * for the partition paths contained in input records.
+ *
+ * @param context HoodieEngineContext
+ * @param instantTime Instant time for the replace action
+ * @param records input records
+ * @return HoodieWriteMetadata
+ */
+ public abstract HoodieWriteMetadata insertOverwriteTable(HoodieEngineContext context, String instantTime, I records);
+
public HoodieWriteConfig getConfig() {
return config;
}
@@ -326,6 +359,27 @@ public abstract Option scheduleCompaction(HoodieEngineCont
public abstract HoodieWriteMetadata compact(HoodieEngineContext context,
String compactionInstantTime);
+
+ /**
+ * Schedule clustering for the instant time.
+ *
+ * @param context HoodieEngineContext
+ * @param instantTime Instant Time for scheduling clustering
+ * @param extraMetadata additional metadata to write into plan
+ * @return HoodieClusteringPlan, if there is enough data for clustering.
+ */
+ public abstract Option scheduleClustering(HoodieEngineContext context,
+ String instantTime,
+ Option> extraMetadata);
+
+ /**
+ * Execute Clustering on the table. Clustering re-arranges the data so that it is optimized for data access.
+ *
+ * @param context HoodieEngineContext
+ * @param clusteringInstantTime Instant Time
+ */
+ public abstract HoodieWriteMetadata cluster(HoodieEngineContext context, String clusteringInstantTime);
+
/**
* Perform metadata/full bootstrap of a Hudi table.
* @param context HoodieEngineContext
@@ -392,6 +446,7 @@ public void finalizeWrite(HoodieEngineContext context, String instantTs, List>> invalidFilesByPartition) {
// Now delete partially written files
+ context.setJobStatus(this.getClass().getSimpleName(), "Delete invalid files generated during the write operation");
context.map(new ArrayList<>(invalidFilesByPartition.values()), partitionWithFileList -> {
final FileSystem fileSystem = metaClient.getFs();
LOG.info("Deleting invalid data files=" + partitionWithFileList);
@@ -449,7 +504,7 @@ protected void reconcileAgainstMarkers(HoodieEngineContext context,
if (!invalidDataPaths.isEmpty()) {
LOG.info("Removing duplicate data files created due to spark retries before committing. Paths=" + invalidDataPaths);
Map>> invalidPathsByPartition = invalidDataPaths.stream()
- .map(dp -> Pair.of(new Path(dp).getParent().toString(), new Path(basePath, dp).toString()))
+ .map(dp -> Pair.of(new Path(basePath, dp).getParent().toString(), new Path(basePath, dp).toString()))
.collect(Collectors.groupingBy(Pair::getKey));
// Ensure all files in delete list is actually present. This is mandatory for an eventually consistent FS.
@@ -597,4 +652,10 @@ public String getBaseFileExtension() {
public boolean requireSortedRecords() {
return getBaseFileFormat() == HoodieFileFormat.HFILE;
}
+
+ public HoodieEngineContext getContext() {
+ // This is to handle scenarios where this is called at the executor tasks which do not have access
+ // to engine context, and it ends up being null (as its not serializable and marked transient here).
+ return context == null ? new HoodieLocalEngineContext(hadoopConfiguration.get()) : context;
+ }
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTimelineArchiveLog.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTimelineArchiveLog.java
index a6b11af23dc36..5d4a743c39264 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTimelineArchiveLog.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTimelineArchiveLog.java
@@ -28,7 +28,7 @@
import org.apache.hudi.avro.model.HoodieRollbackMetadata;
import org.apache.hudi.avro.model.HoodieSavepointMetadata;
import org.apache.hudi.client.ReplaceArchivalHelper;
-import org.apache.hudi.client.common.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.ActionType;
import org.apache.hudi.common.model.HoodieArchivedLogFile;
import org.apache.hudi.common.model.HoodieAvroPayload;
@@ -46,6 +46,7 @@
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
+import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.table.view.TableFileSystemView;
import org.apache.hudi.common.util.CleanerUtils;
import org.apache.hudi.common.util.CollectionUtils;
@@ -56,6 +57,8 @@
import org.apache.hudi.exception.HoodieCommitException;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.metadata.HoodieTableMetadata;
+
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -106,7 +109,7 @@ private Writer openWriter() {
} else {
return this.writer;
}
- } catch (InterruptedException | IOException e) {
+ } catch (IOException e) {
throw new HoodieException("Unable to initialize HoodieLogFormat writer", e);
}
}
@@ -195,6 +198,26 @@ private Stream getInstantsToArchive() {
.collect(Collectors.groupingBy(i -> Pair.of(i.getTimestamp(),
HoodieInstant.getComparableAction(i.getAction()))));
+ // If metadata table is enabled, do not archive instants which are more recent that the latest synced
+ // instant on the metadata table. This is required for metadata table sync.
+ if (config.useFileListingMetadata()) {
+ try (HoodieTableMetadata tableMetadata = HoodieTableMetadata.create(table.getContext(), config.getMetadataConfig(),
+ config.getBasePath(), FileSystemViewStorageConfig.DEFAULT_VIEW_SPILLABLE_DIR)) {
+ Option lastSyncedInstantTime = tableMetadata.getSyncedInstantTime();
+
+ if (lastSyncedInstantTime.isPresent()) {
+ LOG.info("Limiting archiving of instants to last synced instant on metadata table at " + lastSyncedInstantTime.get());
+ instants = instants.filter(i -> HoodieTimeline.compareTimestamps(i.getTimestamp(), HoodieTimeline.LESSER_THAN,
+ lastSyncedInstantTime.get()));
+ } else {
+ LOG.info("Not archiving as there is no instants yet on the metadata table");
+ instants = Stream.empty();
+ }
+ } catch (Exception e) {
+ throw new HoodieException("Error limiting instant archival based on metadata table", e);
+ }
+ }
+
return instants.flatMap(hoodieInstant ->
groupByTsAction.get(Pair.of(hoodieInstant.getTimestamp(),
HoodieInstant.getComparableAction(hoodieInstant.getAction()))).stream());
@@ -276,10 +299,10 @@ public void archive(HoodieEngineContext context, List instants) t
LOG.info("Wrapper schema " + wrapperSchema.toString());
List records = new ArrayList<>();
for (HoodieInstant hoodieInstant : instants) {
+ // TODO HUDI-1518 Cleaner now takes care of removing replaced file groups. This call to deleteReplacedFileGroups can be removed.
boolean deleteSuccess = deleteReplacedFileGroups(context, hoodieInstant);
if (!deleteSuccess) {
- // throw error and stop archival if deleting replaced file groups failed.
- throw new HoodieCommitException("Unable to delete file(s) for " + hoodieInstant.getFileName());
+ LOG.warn("Unable to delete file(s) for " + hoodieInstant.getFileName() + ", replaced files possibly deleted by cleaner");
}
try {
deleteAnyLeftOverMarkerFiles(context, hoodieInstant);
@@ -335,7 +358,7 @@ private void writeToFile(Schema wrapperSchema, List records) thro
Map header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, wrapperSchema.toString());
HoodieAvroDataBlock block = new HoodieAvroDataBlock(records, header);
- this.writer = writer.appendBlock(block);
+ writer.appendBlock(block);
records.clear();
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/MarkerFiles.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/MarkerFiles.java
index 8826204cdafe1..9341a1d8038e3 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/MarkerFiles.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/MarkerFiles.java
@@ -18,8 +18,8 @@
package org.apache.hudi.table;
-import org.apache.hudi.client.common.HoodieEngineContext;
import org.apache.hudi.common.config.SerializableConfiguration;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.IOType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -135,6 +135,7 @@ public Set createdAndMergedDataPaths(HoodieEngineContext context, int pa
if (subDirectories.size() > 0) {
parallelism = Math.min(subDirectories.size(), parallelism);
SerializableConfiguration serializedConf = new SerializableConfiguration(fs.getConf());
+ context.setJobStatus(this.getClass().getSimpleName(), "Obtaining marker files for all created, merged paths");
dataFiles.addAll(context.flatMap(subDirectories, directory -> {
Path path = new Path(directory);
FileSystem fileSystem = path.getFileSystem(serializedConf.get());
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/BaseActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/BaseActionExecutor.java
index 96189e7cfa522..6c776cfb077ae 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/BaseActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/BaseActionExecutor.java
@@ -21,7 +21,7 @@
import java.io.Serializable;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hudi.client.common.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/bootstrap/BootstrapUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/bootstrap/BootstrapUtils.java
index ade2b1885a117..c6f0832445e5e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/bootstrap/BootstrapUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/bootstrap/BootstrapUtils.java
@@ -26,8 +26,8 @@
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hudi.avro.model.HoodieFileStatus;
-import org.apache.hudi.client.common.HoodieEngineContext;
import org.apache.hudi.common.bootstrap.FileStatusUtils;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.collection.Pair;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/BaseCleanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/BaseCleanActionExecutor.java
index 588437c5149a2..5b37820071a1e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/BaseCleanActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/BaseCleanActionExecutor.java
@@ -21,11 +21,11 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hudi.avro.model.HoodieActionInstant;
+import org.apache.hudi.avro.model.HoodieCleanFileInfo;
import org.apache.hudi.avro.model.HoodieCleanMetadata;
import org.apache.hudi.avro.model.HoodieCleanerPlan;
-import org.apache.hudi.avro.model.HoodieCleanFileInfo;
import org.apache.hudi.common.HoodieCleanStat;
-import org.apache.hudi.client.common.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieCleaningPolicy;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.table.timeline.HoodieInstant;
@@ -67,12 +67,12 @@ public BaseCleanActionExecutor(HoodieEngineContext context, HoodieWriteConfig co
*/
HoodieCleanerPlan requestClean(HoodieEngineContext context) {
try {
- CleanPlanner planner = new CleanPlanner<>(table, config);
+ CleanPlanner planner = new CleanPlanner<>(context, table, config);
Option earliestInstant = planner.getEarliestCommitToRetain();
List partitionsToClean = planner.getPartitionPathsToClean(earliestInstant);
if (partitionsToClean.isEmpty()) {
- LOG.info("Nothing to clean here. It is already clean");
+ LOG.info("Nothing to clean here.");
return HoodieCleanerPlan.newBuilder().setPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS.name()).build();
}
LOG.info("Total Partitions to clean : " + partitionsToClean.size() + ", with policy " + config.getCleanerPolicy());
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
index 405fc81b7a0c8..80727ffafb31b 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
@@ -20,6 +20,7 @@
import org.apache.hudi.avro.model.HoodieCleanMetadata;
import org.apache.hudi.avro.model.HoodieSavepointMetadata;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.CleanFileInfo;
import org.apache.hudi.common.model.CompactionOperation;
@@ -30,6 +31,7 @@
import org.apache.hudi.common.model.HoodieFileGroup;
import org.apache.hudi.common.model.HoodieFileGroupId;
import org.apache.hudi.common.model.HoodieRecordPayload;
+import org.apache.hudi.common.model.HoodieReplaceCommitMetadata;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
@@ -76,11 +78,13 @@ public class CleanPlanner implements Ser
private final Map fgIdToPendingCompactionOperations;
private HoodieTable hoodieTable;
private HoodieWriteConfig config;
+ private transient HoodieEngineContext context;
- public CleanPlanner(HoodieTable hoodieTable, HoodieWriteConfig config) {
+ public CleanPlanner(HoodieEngineContext context, HoodieTable hoodieTable, HoodieWriteConfig config) {
+ this.context = context;
this.hoodieTable = hoodieTable;
this.fileSystemView = hoodieTable.getHoodieView();
- this.commitTimeline = hoodieTable.getCompletedCommitTimeline();
+ this.commitTimeline = hoodieTable.getCompletedCommitsTimeline();
this.config = config;
this.fgIdToPendingCompactionOperations =
((SyncableFileSystemView) hoodieTable.getSliceView()).getPendingCompactionOperations()
@@ -112,14 +116,14 @@ public Stream getSavepointedDataFiles(String savepointTime) {
/**
* Returns list of partitions where clean operations needs to be performed.
*
- * @param newInstantToRetain New instant to be retained after this cleanup operation
+ * @param earliestRetainedInstant New instant to be retained after this cleanup operation
* @return list of partitions to scan for cleaning
* @throws IOException when underlying file-system throws this exception
*/
- public List getPartitionPathsToClean(Option newInstantToRetain) throws IOException {
+ public List getPartitionPathsToClean(Option earliestRetainedInstant) throws IOException {
switch (config.getCleanerPolicy()) {
case KEEP_LATEST_COMMITS:
- return getPartitionPathsForCleanByCommits(newInstantToRetain);
+ return getPartitionPathsForCleanByCommits(earliestRetainedInstant);
case KEEP_LATEST_FILE_VERSIONS:
return getPartitionPathsForFullCleaning();
default:
@@ -161,7 +165,7 @@ private List getPartitionPathsForCleanByCommits(Option in
*/
private List getPartitionPathsForIncrementalCleaning(HoodieCleanMetadata cleanMetadata,
Option newInstantToRetain) {
- LOG.warn("Incremental Cleaning mode is enabled. Looking up partition-paths that have since changed "
+ LOG.info("Incremental Cleaning mode is enabled. Looking up partition-paths that have since changed "
+ "since last cleaned at " + cleanMetadata.getEarliestCommitToRetain()
+ ". New Instant to retain : " + newInstantToRetain);
return hoodieTable.getCompletedCommitsTimeline().getInstants().filter(
@@ -169,10 +173,16 @@ private List getPartitionPathsForIncrementalCleaning(HoodieCleanMetadata
cleanMetadata.getEarliestCommitToRetain()) && HoodieTimeline.compareTimestamps(instant.getTimestamp(),
HoodieTimeline.LESSER_THAN, newInstantToRetain.get().getTimestamp())).flatMap(instant -> {
try {
- HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
- .fromBytes(hoodieTable.getActiveTimeline().getInstantDetails(instant).get(),
- HoodieCommitMetadata.class);
- return commitMetadata.getPartitionToWriteStats().keySet().stream();
+ if (HoodieTimeline.REPLACE_COMMIT_ACTION.equals(instant.getAction())) {
+ HoodieReplaceCommitMetadata replaceCommitMetadata = HoodieReplaceCommitMetadata.fromBytes(
+ hoodieTable.getActiveTimeline().getInstantDetails(instant).get(), HoodieReplaceCommitMetadata.class);
+ return Stream.concat(replaceCommitMetadata.getPartitionToReplaceFileIds().keySet().stream(), replaceCommitMetadata.getPartitionToWriteStats().keySet().stream());
+ } else {
+ HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
+ .fromBytes(hoodieTable.getActiveTimeline().getInstantDetails(instant).get(),
+ HoodieCommitMetadata.class);
+ return commitMetadata.getPartitionToWriteStats().keySet().stream();
+ }
} catch (IOException e) {
throw new HoodieIOException(e.getMessage(), e);
}
@@ -180,14 +190,13 @@ private List getPartitionPathsForIncrementalCleaning(HoodieCleanMetadata
}
/**
- * Scan and list all paritions for cleaning.
+ * Scan and list all partitions for cleaning.
* @return all partitions paths for the dataset.
* @throws IOException
*/
- private List getPartitionPathsForFullCleaning() throws IOException {
+ private List getPartitionPathsForFullCleaning() {
// Go to brute force mode of scanning all partitions
- return FSUtils.getAllPartitionPaths(hoodieTable.getMetaClient().getFs(), hoodieTable.getMetaClient().getBasePath(),
- config.shouldAssumeDatePartitioning());
+ return FSUtils.getAllPartitionPaths(context, config.getMetadataConfig(), config.getBasePath());
}
/**
@@ -198,13 +207,17 @@ private List getPartitionPathsForFullCleaning() throws IOException {
private List getFilesToCleanKeepingLatestVersions(String partitionPath) {
LOG.info("Cleaning " + partitionPath + ", retaining latest " + config.getCleanerFileVersionsRetained()
+ " file versions. ");
- List fileGroups = fileSystemView.getAllFileGroups(partitionPath).collect(Collectors.toList());
List deletePaths = new ArrayList<>();
// Collect all the datafiles savepointed by all the savepoints
List savepointedFiles = hoodieTable.getSavepoints().stream()
.flatMap(this::getSavepointedDataFiles)
.collect(Collectors.toList());
+ // In this scenario, we will assume that once replaced a file group automatically becomes eligible for cleaning completely
+ // In other words, the file versions only apply to the active file groups.
+ deletePaths.addAll(getReplacedFilesEligibleToClean(savepointedFiles, partitionPath, Option.empty()));
+
+ List fileGroups = fileSystemView.getAllFileGroups(partitionPath).collect(Collectors.toList());
for (HoodieFileGroup fileGroup : fileGroups) {
int keepVersions = config.getCleanerFileVersionsRetained();
// do not cleanup slice required for pending compaction
@@ -228,18 +241,7 @@ private List getFilesToCleanKeepingLatestVersions(String partitio
// Delete the remaining files
while (fileSliceIterator.hasNext()) {
FileSlice nextSlice = fileSliceIterator.next();
- if (nextSlice.getBaseFile().isPresent()) {
- HoodieBaseFile dataFile = nextSlice.getBaseFile().get();
- deletePaths.add(new CleanFileInfo(dataFile.getPath(), false));
- if (dataFile.getBootstrapBaseFile().isPresent() && config.shouldCleanBootstrapBaseFile()) {
- deletePaths.add(new CleanFileInfo(dataFile.getBootstrapBaseFile().get().getPath(), true));
- }
- }
- if (hoodieTable.getMetaClient().getTableType() == HoodieTableType.MERGE_ON_READ) {
- // If merge on read, then clean the log files for the commits as well
- deletePaths.addAll(nextSlice.getLogFiles().map(lf -> new CleanFileInfo(lf.getPath().toString(), false))
- .collect(Collectors.toList()));
- }
+ deletePaths.addAll(getCleanFileInfoForSlice(nextSlice));
}
}
return deletePaths;
@@ -271,7 +273,11 @@ private List getFilesToCleanKeepingLatestCommits(String partition
// determine if we have enough commits, to start cleaning.
if (commitTimeline.countInstants() > commitsRetained) {
- HoodieInstant earliestCommitToRetain = getEarliestCommitToRetain().get();
+ Option earliestCommitToRetainOption = getEarliestCommitToRetain();
+ HoodieInstant earliestCommitToRetain = earliestCommitToRetainOption.get();
+ // all replaced file groups before earliestCommitToRetain are eligible to clean
+ deletePaths.addAll(getReplacedFilesEligibleToClean(savepointedFiles, partitionPath, earliestCommitToRetainOption));
+ // add active files
List fileGroups = fileSystemView.getAllFileGroups(partitionPath).collect(Collectors.toList());
for (HoodieFileGroup fileGroup : fileGroups) {
List fileSliceList = fileGroup.getAllFileSlices().collect(Collectors.toList());
@@ -324,6 +330,20 @@ private List getFilesToCleanKeepingLatestCommits(String partition
}
return deletePaths;
}
+
+ private List getReplacedFilesEligibleToClean(List savepointedFiles, String partitionPath, Option earliestCommitToRetain) {
+ final Stream replacedGroups;
+ if (earliestCommitToRetain.isPresent()) {
+ replacedGroups = fileSystemView.getReplacedFileGroupsBefore(earliestCommitToRetain.get().getTimestamp(), partitionPath);
+ } else {
+ replacedGroups = fileSystemView.getAllReplacedFileGroups(partitionPath);
+ }
+ return replacedGroups.flatMap(HoodieFileGroup::getAllFileSlices)
+ // do not delete savepointed files (archival will make sure corresponding replacecommit file is not deleted)
+ .filter(slice -> !slice.getBaseFile().isPresent() || !savepointedFiles.contains(slice.getBaseFile().get().getFileName()))
+ .flatMap(slice -> getCleanFileInfoForSlice(slice).stream())
+ .collect(Collectors.toList());
+ }
/**
* Gets the latest version < instantTime. This version file could still be used by queries.
@@ -341,6 +361,23 @@ private String getLatestVersionBeforeCommit(List fileSliceList, Hoodi
return null;
}
+ private List getCleanFileInfoForSlice(FileSlice nextSlice) {
+ List cleanPaths = new ArrayList<>();
+ if (nextSlice.getBaseFile().isPresent()) {
+ HoodieBaseFile dataFile = nextSlice.getBaseFile().get();
+ cleanPaths.add(new CleanFileInfo(dataFile.getPath(), false));
+ if (dataFile.getBootstrapBaseFile().isPresent() && config.shouldCleanBootstrapBaseFile()) {
+ cleanPaths.add(new CleanFileInfo(dataFile.getBootstrapBaseFile().get().getPath(), true));
+ }
+ }
+ if (hoodieTable.getMetaClient().getTableType() == HoodieTableType.MERGE_ON_READ) {
+ // If merge on read, then clean the log files for the commits as well
+ cleanPaths.addAll(nextSlice.getLogFiles().map(lf -> new CleanFileInfo(lf.getPath().toString(), false))
+ .collect(Collectors.toList()));
+ }
+ return cleanPaths;
+ }
+
/**
* Returns files to be cleaned for the given partitionPath based on cleaning policy.
*/
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/BaseClusteringPlanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/BaseClusteringPlanActionExecutor.java
new file mode 100644
index 0000000000000..97407e3464d79
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/BaseClusteringPlanActionExecutor.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.action.cluster;
+
+import org.apache.hudi.avro.model.HoodieClusteringPlan;
+import org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.model.HoodieRecordPayload;
+import org.apache.hudi.common.model.WriteOperationType;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.table.HoodieTable;
+import org.apache.hudi.table.action.BaseActionExecutor;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+public abstract class BaseClusteringPlanActionExecutor extends BaseActionExecutor> {
+
+ private final Option> extraMetadata;
+
+ public BaseClusteringPlanActionExecutor(HoodieEngineContext context,
+ HoodieWriteConfig config,
+ HoodieTable table,
+ String instantTime,
+ Option> extraMetadata) {
+ super(context, config, table, instantTime);
+ this.extraMetadata = extraMetadata;
+ }
+
+ protected abstract Option createClusteringPlan();
+
+ @Override
+ public Option execute() {
+ Option planOption = createClusteringPlan();
+ if (planOption.isPresent()) {
+ HoodieInstant clusteringInstant =
+ new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.REPLACE_COMMIT_ACTION, instantTime);
+ try {
+ HoodieRequestedReplaceMetadata requestedReplaceMetadata = HoodieRequestedReplaceMetadata.newBuilder()
+ .setOperationType(WriteOperationType.CLUSTER.name())
+ .setExtraMetadata(extraMetadata.orElse(Collections.emptyMap()))
+ .setClusteringPlan(planOption.get())
+ .build();
+ table.getActiveTimeline().saveToPendingReplaceCommit(clusteringInstant,
+ TimelineMetadataUtils.serializeRequestedReplaceMetadata(requestedReplaceMetadata));
+ } catch (IOException ioe) {
+ throw new HoodieIOException("Exception scheduling clustering", ioe);
+ }
+ }
+ return planOption;
+ }
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringExecutionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringExecutionStrategy.java
new file mode 100644
index 0000000000000..dce6eeac3bd0b
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringExecutionStrategy.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.action.cluster.strategy;
+
+import org.apache.avro.Schema;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.model.HoodieRecordPayload;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.table.HoodieTable;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+import java.io.Serializable;
+import java.util.Map;
+
+/**
+ * Pluggable implementation for writing data into new file groups based on ClusteringPlan.
+ */
+public abstract class ClusteringExecutionStrategy implements Serializable {
+ private static final Logger LOG = LogManager.getLogger(ClusteringExecutionStrategy.class);
+
+ private final HoodieTable hoodieTable;
+ private final HoodieEngineContext engineContext;
+ private final HoodieWriteConfig writeConfig;
+
+ public ClusteringExecutionStrategy(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig) {
+ this.writeConfig = writeConfig;
+ this.hoodieTable = table;
+ this.engineContext = engineContext;
+ }
+
+ /**
+ * Execute clustering to write inputRecords into new files as defined by rules in strategy parameters. The number of new
+ * file groups created is bounded by numOutputGroups.
+ * Note that commit is not done as part of strategy. commit is callers responsibility.
+ */
+ public abstract O performClustering(final I inputRecords, final int numOutputGroups, final String instantTime,
+ final Map strategyParams, final Schema schema);
+
+ protected HoodieTable getHoodieTable() {
+ return this.hoodieTable;
+ }
+
+ protected HoodieEngineContext getEngineContext() {
+ return this.engineContext;
+ }
+
+ protected HoodieWriteConfig getWriteConfig() {
+ return this.writeConfig;
+ }
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringPlanStrategy.java
new file mode 100644
index 0000000000000..273ebce761e26
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/ClusteringPlanStrategy.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.action.cluster.strategy;
+
+import org.apache.hudi.avro.model.HoodieClusteringPlan;
+import org.apache.hudi.avro.model.HoodieSliceInfo;
+import org.apache.hudi.client.utils.FileSliceMetricUtils;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.model.BaseFile;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.HoodieFileGroupId;
+import org.apache.hudi.common.model.HoodieRecordPayload;
+import org.apache.hudi.common.table.view.SyncableFileSystemView;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.table.HoodieTable;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Pluggable implementation for scheduling clustering and creating ClusteringPlan.
+ */
+public abstract class ClusteringPlanStrategy implements Serializable {
+ private static final Logger LOG = LogManager.getLogger(ClusteringPlanStrategy.class);
+
+ public static final int CLUSTERING_PLAN_VERSION_1 = 1;
+
+ private final HoodieTable hoodieTable;
+ private final transient HoodieEngineContext engineContext;
+ private final HoodieWriteConfig writeConfig;
+
+ public ClusteringPlanStrategy(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig) {
+ this.writeConfig = writeConfig;
+ this.hoodieTable = table;
+ this.engineContext = engineContext;
+ }
+
+ /**
+ * Generate metadata for grouping eligible files and create a plan. Note that data is not moved around
+ * as part of this step.
+ *
+ * If there is no data available to cluster, return None.
+ */
+ public abstract Option generateClusteringPlan();
+
+ /**
+ * Return file slices eligible for clustering. FileIds in pending clustering/compaction are not eligible for clustering.
+ */
+ protected Stream getFileSlicesEligibleForClustering(String partition) {
+ SyncableFileSystemView fileSystemView = (SyncableFileSystemView) getHoodieTable().getSliceView();
+ Set fgIdsInPendingCompactionAndClustering = fileSystemView.getPendingCompactionOperations()
+ .map(instantTimeOpPair -> instantTimeOpPair.getValue().getFileGroupId())
+ .collect(Collectors.toSet());
+ fgIdsInPendingCompactionAndClustering.addAll(fileSystemView.getFileGroupsInPendingClustering().map(Pair::getKey).collect(Collectors.toSet()));
+
+ return hoodieTable.getSliceView().getLatestFileSlices(partition)
+ // file ids already in clustering are not eligible
+ .filter(slice -> !fgIdsInPendingCompactionAndClustering.contains(slice.getFileGroupId()));
+ }
+
+ /**
+ * Get parameters specific to strategy. These parameters are passed from 'schedule clustering' step to
+ * 'execute clustering' step. 'execute clustering' step is typically async. So these params help with passing any required
+ * context from schedule to run step.
+ */
+ protected abstract Map getStrategyParams();
+
+ /**
+ * Returns any specific parameters to be stored as part of clustering metadata.
+ */
+ protected Map getExtraMetadata() {
+ return Collections.emptyMap();
+ }
+
+ /**
+ * Version to support future changes for plan.
+ */
+ protected int getPlanVersion() {
+ return CLUSTERING_PLAN_VERSION_1;
+ }
+
+ /**
+ * Transform {@link FileSlice} to {@link HoodieSliceInfo}.
+ */
+ protected static List getFileSliceInfo(List slices) {
+ return slices.stream().map(slice -> new HoodieSliceInfo().newBuilder()
+ .setPartitionPath(slice.getPartitionPath())
+ .setFileId(slice.getFileId())
+ .setDataFilePath(slice.getBaseFile().map(BaseFile::getPath).orElse(StringUtils.EMPTY_STRING))
+ .setDeltaFilePaths(slice.getLogFiles().map(f -> f.getPath().toString()).collect(Collectors.toList()))
+ .setBootstrapFilePath(slice.getBaseFile().map(bf -> bf.getBootstrapBaseFile().map(bbf -> bbf.getPath()).orElse(StringUtils.EMPTY_STRING)).orElse(StringUtils.EMPTY_STRING))
+ .build()).collect(Collectors.toList());
+ }
+
+ /**
+ * Generate metrics for the data to be clustered.
+ */
+ protected Map buildMetrics(List fileSlices) {
+ Map metrics = new HashMap<>();
+ FileSliceMetricUtils.addFileSliceCommonMetrics(fileSlices, metrics, getWriteConfig().getParquetMaxFileSize());
+ return metrics;
+ }
+
+ protected HoodieTable getHoodieTable() {
+ return this.hoodieTable;
+ }
+
+ protected HoodieEngineContext getEngineContext() {
+ return this.engineContext;
+ }
+
+ protected HoodieWriteConfig getWriteConfig() {
+ return this.writeConfig;
+ }
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java
new file mode 100644
index 0000000000000..d3b39f59eda0c
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/PartitionAwareClusteringPlanStrategy.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.action.cluster.strategy;
+
+import org.apache.hudi.avro.model.HoodieClusteringGroup;
+import org.apache.hudi.avro.model.HoodieClusteringPlan;
+import org.apache.hudi.avro.model.HoodieClusteringStrategy;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.HoodieRecordPayload;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.table.HoodieTable;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Scheduling strategy with restriction that clustering groups can only contain files from same partition.
+ */
+public abstract class PartitionAwareClusteringPlanStrategy extends ClusteringPlanStrategy {
+ private static final Logger LOG = LogManager.getLogger(PartitionAwareClusteringPlanStrategy.class);
+
+ public PartitionAwareClusteringPlanStrategy(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig) {
+ super(table, engineContext, writeConfig);
+ }
+
+ /**
+ * Create Clustering group based on files eligible for clustering in the partition.
+ */
+ protected abstract Stream buildClusteringGroupsForPartition(String partitionPath,
+ List fileSlices);
+
+ /**
+ * Return list of partition paths to be considered for clustering.
+ */
+ protected List filterPartitionPaths(List partitionPaths) {
+ return partitionPaths;
+ }
+
+ @Override
+ public Option generateClusteringPlan() {
+ HoodieTableMetaClient metaClient = getHoodieTable().getMetaClient();
+ LOG.info("Scheduling clustering for " + metaClient.getBasePath());
+ HoodieWriteConfig config = getWriteConfig();
+ List partitionPaths = FSUtils.getAllPartitionPaths(getEngineContext(), config.getMetadataConfig(), metaClient.getBasePath());
+
+ // filter the partition paths if needed to reduce list status
+ partitionPaths = filterPartitionPaths(partitionPaths);
+
+ if (partitionPaths.isEmpty()) {
+ // In case no partitions could be picked, return no clustering plan
+ return Option.empty();
+ }
+
+ List clusteringGroups = getEngineContext().flatMap(partitionPaths,
+ partitionPath -> {
+ List fileSlicesEligible = getFileSlicesEligibleForClustering(partitionPath).collect(Collectors.toList());
+ return buildClusteringGroupsForPartition(partitionPath, fileSlicesEligible).limit(getWriteConfig().getClusteringMaxNumGroups());
+ },
+ partitionPaths.size())
+ .stream().limit(getWriteConfig().getClusteringMaxNumGroups()).collect(Collectors.toList());
+
+ if (clusteringGroups.isEmpty()) {
+ LOG.info("No data available to cluster");
+ return Option.empty();
+ }
+
+ HoodieClusteringStrategy strategy = HoodieClusteringStrategy.newBuilder()
+ .setStrategyClassName(getWriteConfig().getClusteringExecutionStrategyClass())
+ .setStrategyParams(getStrategyParams())
+ .build();
+
+ return Option.of(HoodieClusteringPlan.newBuilder()
+ .setStrategy(strategy)
+ .setInputGroups(clusteringGroups)
+ .setExtraMetadata(getExtraMetadata())
+ .setVersion(getPlanVersion())
+ .build());
+ }
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/UpdateStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/UpdateStrategy.java
new file mode 100644
index 0000000000000..3ce4f04e3d035
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/UpdateStrategy.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.action.cluster.strategy;
+
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.model.HoodieFileGroupId;
+import org.apache.hudi.common.model.HoodieRecordPayload;
+
+import java.util.Set;
+
+/**
+ * When file groups in clustering, write records to these file group need to check.
+ */
+public abstract class UpdateStrategy, I> {
+
+ protected final HoodieEngineContext engineContext;
+ protected Set fileGroupsInPendingClustering;
+
+ protected UpdateStrategy(HoodieEngineContext engineContext, Set fileGroupsInPendingClustering) {
+ this.engineContext = engineContext;
+ this.fileGroupsInPendingClustering = fileGroupsInPendingClustering;
+ }
+
+ /**
+ * Check the update records to the file group in clustering.
+ * @param taggedRecordsRDD the records to write, tagged with target file id,
+ * future can update tagged records location to a different fileId.
+ * @return the recordsRDD strategy updated
+ */
+ public abstract I handleUpdate(I taggedRecordsRDD);
+
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractBulkInsertHelper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractBulkInsertHelper.java
index 3ead7a07d9698..d2aa8627bd275 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractBulkInsertHelper.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractBulkInsertHelper.java
@@ -27,8 +27,21 @@
public abstract class AbstractBulkInsertHelper {
+ /**
+ * Mark instant as inflight, write input records, update index and return result.
+ */
public abstract HoodieWriteMetadata bulkInsert(I inputRecords, String instantTime,
HoodieTable table, HoodieWriteConfig config,
BaseCommitActionExecutor executor, boolean performDedupe,
Option> userDefinedBulkInsertPartitioner);
+
+ /**
+ * Only write input records. Does not change timeline/index. Return information about new files created.
+ */
+ public abstract O bulkInsert(I inputRecords, String instantTime,
+ HoodieTable table, HoodieWriteConfig config,
+ boolean performDedupe,
+ Option> userDefinedBulkInsertPartitioner,
+ boolean addMetadataFields,
+ int parallelism);
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractDeleteHelper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractDeleteHelper.java
index c2d2df7ebac99..ac0f2596f490e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractDeleteHelper.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractDeleteHelper.java
@@ -18,7 +18,7 @@
package org.apache.hudi.table.action.commit;
-import org.apache.hudi.client.common.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractWriteHelper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractWriteHelper.java
index caa6ecdb953a7..455952ae5f7d7 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractWriteHelper.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractWriteHelper.java
@@ -18,7 +18,7 @@
package org.apache.hudi.table.action.commit;
-import org.apache.hudi.client.common.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.exception.HoodieUpsertException;
import org.apache.hudi.index.HoodieIndex;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java
index 71de9b6fc6f73..4a8ddf5c57997 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java
@@ -19,8 +19,8 @@
package org.apache.hudi.table.action.commit;
import org.apache.hudi.client.WriteStatus;
-import org.apache.hudi.client.common.TaskContextSupplier;
-import org.apache.hudi.client.common.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/Partitioner.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/Partitioner.java
similarity index 100%
rename from hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/Partitioner.java
rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/Partitioner.java
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/BaseScheduleCompactionActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/BaseScheduleCompactionActionExecutor.java
index d21c7d96d113a..9e09b8c82761c 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/BaseScheduleCompactionActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/BaseScheduleCompactionActionExecutor.java
@@ -19,7 +19,7 @@
package org.apache.hudi.table.action.compact;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
-import org.apache.hudi.client.common.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/HoodieCompactor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/HoodieCompactor.java
index e5212e8f43a5b..c92c0b3a0237e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/HoodieCompactor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/HoodieCompactor.java
@@ -19,7 +19,7 @@
package org.apache.hudi.table.action.compact;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
-import org.apache.hudi.client.common.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieFileGroupId;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.config.HoodieWriteConfig;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompactionStrategy.java
index 6c631c462bda1..ff2dfbd4c1ecb 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompactionStrategy.java
@@ -20,14 +20,13 @@
import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
-import org.apache.hudi.common.fs.FSUtils;
-import org.apache.hudi.common.model.HoodieBaseFile;
-import org.apache.hudi.common.model.HoodieLogFile;
+import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.util.CompactionUtils;
-import org.apache.hudi.common.util.Option;
+import org.apache.hudi.client.utils.FileSliceMetricUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import java.io.Serializable;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -50,32 +49,14 @@ public abstract class CompactionStrategy implements Serializable {
* Callback hook when a HoodieCompactionOperation is created. Individual strategies can capture the metrics they need
* to decide on the priority.
*
- * @param dataFile - Base file to compact
- * @param partitionPath - Partition path
- * @param logFiles - List of log files to compact with the base file
+ * @param writeConfig write configuration.
+ * @param slice fileSlice to capture metrics for.
* @return Map[String, Object] - metrics captured
*/
- public Map captureMetrics(HoodieWriteConfig writeConfig, Option dataFile,
- String partitionPath, List logFiles) {
+ public Map captureMetrics(HoodieWriteConfig writeConfig, FileSlice slice) {
Map metrics = new HashMap<>();
long defaultMaxParquetFileSize = writeConfig.getParquetMaxFileSize();
- // Total size of all the log files
- Long totalLogFileSize = logFiles.stream().map(HoodieLogFile::getFileSize).filter(size -> size >= 0)
- .reduce(Long::sum).orElse(0L);
- // Total read will be the base file + all the log files
- Long totalIORead =
- FSUtils.getSizeInMB((dataFile.isPresent() ? dataFile.get().getFileSize() : 0L) + totalLogFileSize);
- // Total write will be similar to the size of the base file
- Long totalIOWrite =
- FSUtils.getSizeInMB(dataFile.isPresent() ? dataFile.get().getFileSize() : defaultMaxParquetFileSize);
- // Total IO will the the IO for read + write
- long totalIO = totalIORead + totalIOWrite;
- // Save these metrics and we will use during the filter
- metrics.put(TOTAL_IO_READ_MB, totalIORead.doubleValue());
- metrics.put(TOTAL_IO_WRITE_MB, totalIOWrite.doubleValue());
- metrics.put(TOTAL_IO_MB, (double) totalIO);
- metrics.put(TOTAL_LOG_FILE_SIZE, totalLogFileSize.doubleValue());
- metrics.put(TOTAL_LOG_FILES, (double) logFiles.size());
+ FileSliceMetricUtils.addFileSliceCommonMetrics(Collections.singletonList(slice), metrics, defaultMaxParquetFileSize);
return metrics;
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileSizeBasedCompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileSizeBasedCompactionStrategy.java
index c9a811a1cceeb..fe4b474593886 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileSizeBasedCompactionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/LogFileSizeBasedCompactionStrategy.java
@@ -20,14 +20,10 @@
import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
-import org.apache.hudi.common.model.HoodieBaseFile;
-import org.apache.hudi.common.model.HoodieLogFile;
-import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import java.util.Comparator;
import java.util.List;
-import java.util.Map;
import java.util.stream.Collectors;
/**
@@ -40,21 +36,6 @@
public class LogFileSizeBasedCompactionStrategy extends BoundedIOCompactionStrategy
implements Comparator {
- private static final String TOTAL_LOG_FILE_SIZE = "TOTAL_LOG_FILE_SIZE";
-
- @Override
- public Map captureMetrics(HoodieWriteConfig config, Option dataFile,
- String partitionPath, List logFiles) {
- Map metrics = super.captureMetrics(config, dataFile, partitionPath, logFiles);
-
- // Total size of all the log files
- Long totalLogFileSize = logFiles.stream().map(HoodieLogFile::getFileSize).filter(size -> size >= 0)
- .reduce(Long::sum).orElse(0L);
- // save the metrics needed during the order
- metrics.put(TOTAL_LOG_FILE_SIZE, totalLogFileSize.doubleValue());
- return metrics;
- }
-
@Override
public List orderAndFilter(HoodieWriteConfig writeConfig,
List operations, List pendingCompactionPlans) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java
index 0bbc0a05f9e59..0dd2a7cfb4df1 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java
@@ -20,7 +20,7 @@
import org.apache.hudi.avro.model.HoodieRestoreMetadata;
import org.apache.hudi.avro.model.HoodieRollbackMetadata;
-import org.apache.hudi.client.common.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/AbstractMarkerBasedRollbackStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/AbstractMarkerBasedRollbackStrategy.java
index 40526b86f2cdb..cc596ba3422b7 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/AbstractMarkerBasedRollbackStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/AbstractMarkerBasedRollbackStrategy.java
@@ -18,9 +18,10 @@
package org.apache.hudi.table.action.rollback;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
-import org.apache.hudi.client.common.HoodieEngineContext;
import org.apache.hudi.common.HoodieRollbackStat;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.model.HoodieRecordPayload;
@@ -37,6 +38,7 @@
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
+import java.util.Objects;
/**
* Performs rollback using marker files generated during the write..
@@ -51,9 +53,9 @@ public abstract class AbstractMarkerBasedRollbackStrategy table, HoodieEngineContext context, HoodieWriteConfig config, String instantTime) {
this.table = table;
@@ -88,6 +90,7 @@ protected HoodieRollbackStat undoAppend(String appendBaseFilePath, HoodieInstant
String fileId = FSUtils.getFileIdFromFilePath(baseFilePathForAppend);
String baseCommitTime = FSUtils.getCommitTime(baseFilePathForAppend.getName());
String partitionPath = FSUtils.getRelativePartitionPath(new Path(basePath), new Path(basePath, appendBaseFilePath).getParent());
+ final Map writtenLogFileSizeMap = getWrittenLogFileSizeMap(partitionPath, baseCommitTime, fileId);
HoodieLogFormat.Writer writer = null;
try {
@@ -108,7 +111,7 @@ protected HoodieRollbackStat undoAppend(String appendBaseFilePath, HoodieInstant
// generate metadata
Map header = RollbackUtils.generateHeader(instantToRollback.getTimestamp(), instantTime);
// if update belongs to an existing log file
- writer = writer.appendBlock(new HoodieCommandBlock(header));
+ writer.appendBlock(new HoodieCommandBlock(header));
} finally {
try {
if (writer != null) {
@@ -119,10 +122,26 @@ protected HoodieRollbackStat undoAppend(String appendBaseFilePath, HoodieInstant
}
}
+ // the information of files appended to is required for metadata sync
+ Map filesToNumBlocksRollback = Collections.singletonMap(
+ table.getMetaClient().getFs().getFileStatus(Objects.requireNonNull(writer).getLogFile().getPath()),
+ 1L);
+
return HoodieRollbackStat.newBuilder()
.withPartitionPath(partitionPath)
- // we don't use this field per se. Avoiding the extra file status call.
- .withRollbackBlockAppendResults(Collections.emptyMap())
- .build();
+ .withRollbackBlockAppendResults(filesToNumBlocksRollback)
+ .withWrittenLogFileSizeMap(writtenLogFileSizeMap).build();
+ }
+
+ /**
+ * Returns written log file size map for the respective baseCommitTime to assist in metadata table syncing.
+ * @param partitionPath partition path of interest
+ * @param baseCommitTime base commit time of interest
+ * @param fileId fileId of interest
+ * @return Map
+ * @throws IOException
+ */
+ protected Map getWrittenLogFileSizeMap(String partitionPath, String baseCommitTime, String fileId) throws IOException {
+ return Collections.EMPTY_MAP;
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseCopyOnWriteRollbackActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseCopyOnWriteRollbackActionExecutor.java
index c60cb7f5e7932..fa74f7f6e86bc 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseCopyOnWriteRollbackActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseCopyOnWriteRollbackActionExecutor.java
@@ -18,8 +18,8 @@
package org.apache.hudi.table.action.rollback;
-import org.apache.hudi.client.common.HoodieEngineContext;
import org.apache.hudi.common.HoodieRollbackStat;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseMergeOnReadRollbackActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseMergeOnReadRollbackActionExecutor.java
index 0e747a58dc673..2e751443abc00 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseMergeOnReadRollbackActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseMergeOnReadRollbackActionExecutor.java
@@ -18,8 +18,8 @@
package org.apache.hudi.table.action.rollback;
-import org.apache.hudi.client.common.HoodieEngineContext;
import org.apache.hudi.common.HoodieRollbackStat;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.HoodieTimer;
@@ -28,7 +28,6 @@
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -57,7 +56,7 @@ public BaseMergeOnReadRollbackActionExecutor(HoodieEngineContext context,
}
@Override
- protected List executeRollback() throws IOException {
+ protected List executeRollback() {
HoodieTimer rollbackTimer = new HoodieTimer();
rollbackTimer.startTimer();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
index 40206ca45e1c3..3a579535528cc 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java
@@ -19,9 +19,9 @@
package org.apache.hudi.table.action.rollback;
import org.apache.hudi.avro.model.HoodieRollbackMetadata;
-import org.apache.hudi.client.common.HoodieEngineContext;
import org.apache.hudi.common.HoodieRollbackStat;
import org.apache.hudi.common.bootstrap.index.BootstrapIndex;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackUtils.java
index 897b448fb5340..bfbe577f9429f 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/RollbackUtils.java
@@ -20,11 +20,11 @@
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hudi.client.common.HoodieEngineContext;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.hudi.common.HoodieRollbackStat;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieCommitMetadata;
@@ -74,31 +74,30 @@ static HoodieRollbackStat mergeRollbackStat(HoodieRollbackStat stat1, HoodieRoll
final List successDeleteFiles = new ArrayList<>();
final List failedDeleteFiles = new ArrayList<>();
final Map commandBlocksCount = new HashMap<>();
- final List filesToRollback = new ArrayList<>();
+ final Map writtenLogFileSizeMap = new HashMap<>();
Option.ofNullable(stat1.getSuccessDeleteFiles()).ifPresent(successDeleteFiles::addAll);
Option.ofNullable(stat2.getSuccessDeleteFiles()).ifPresent(successDeleteFiles::addAll);
Option.ofNullable(stat1.getFailedDeleteFiles()).ifPresent(failedDeleteFiles::addAll);
Option.ofNullable(stat2.getFailedDeleteFiles()).ifPresent(failedDeleteFiles::addAll);
Option.ofNullable(stat1.getCommandBlocksCount()).ifPresent(commandBlocksCount::putAll);
Option.ofNullable(stat2.getCommandBlocksCount()).ifPresent(commandBlocksCount::putAll);
- return new HoodieRollbackStat(stat1.getPartitionPath(), successDeleteFiles, failedDeleteFiles, commandBlocksCount);
+ Option.ofNullable(stat1.getWrittenLogFileSizeMap()).ifPresent(writtenLogFileSizeMap::putAll);
+ Option.ofNullable(stat2.getWrittenLogFileSizeMap()).ifPresent(writtenLogFileSizeMap::putAll);
+ return new HoodieRollbackStat(stat1.getPartitionPath(), successDeleteFiles, failedDeleteFiles, commandBlocksCount, writtenLogFileSizeMap);
}
/**
* Generate all rollback requests that needs rolling back this action without actually performing rollback for COW table type.
* @param fs instance of {@link FileSystem} to use.
* @param basePath base path of interest.
- * @param shouldAssumeDatePartitioning {@code true} if date partitioning should be assumed. {@code false} otherwise.
+ * @param config instance of {@link HoodieWriteConfig} to use.
* @return {@link List} of {@link ListingBasedRollbackRequest}s thus collected.
*/
- public static List generateRollbackRequestsByListingCOW(FileSystem fs, String basePath, boolean shouldAssumeDatePartitioning) {
- try {
- return FSUtils.getAllPartitionPaths(fs, basePath, shouldAssumeDatePartitioning).stream()
- .map(ListingBasedRollbackRequest::createRollbackRequestWithDeleteDataAndLogFilesAction)
- .collect(Collectors.toList());
- } catch (IOException e) {
- throw new HoodieIOException("Error generating rollback requests", e);
- }
+ public static List generateRollbackRequestsByListingCOW(HoodieEngineContext engineContext,
+ String basePath, HoodieWriteConfig config) {
+ return FSUtils.getAllPartitionPaths(engineContext, config.getMetadataConfig(), basePath).stream()
+ .map(ListingBasedRollbackRequest::createRollbackRequestWithDeleteDataAndLogFilesAction)
+ .collect(Collectors.toList());
}
/**
@@ -112,8 +111,7 @@ public static List generateRollbackRequestsByListin
public static List generateRollbackRequestsUsingFileListingMOR(HoodieInstant instantToRollback, HoodieTable table, HoodieEngineContext context) throws IOException {
String commit = instantToRollback.getTimestamp();
HoodieWriteConfig config = table.getConfig();
- List partitions = FSUtils.getAllPartitionPaths(table.getMetaClient().getFs(), table.getMetaClient().getBasePath(),
- config.shouldAssumeDatePartitioning());
+ List