From 58cd9d4acbad3c8850c769a7ef419b95255732a4 Mon Sep 17 00:00:00 2001 From: wangxianghu Date: Wed, 21 Oct 2020 21:54:27 +0800 Subject: [PATCH] [MINOR] Private the NoArgsConstructor of SparkMergeHelper and code clean --- .../action/commit/BaseSparkCommitActionExecutor.java | 9 +-------- .../commit/SparkInsertOverwriteCommitActionExecutor.java | 7 ++----- .../hudi/table/action/commit/SparkMergeHelper.java | 3 +++ 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java index 36cca8cd1403c..ad62db9250eda 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java @@ -199,6 +199,7 @@ protected void updateIndexAndCommitIfNeeded(JavaRDD writeStatusRDD, commitOnAutoCommit(result); } + @Override protected String getCommitActionType() { return table.getMetaClient().getCommitActionType(); } @@ -276,14 +277,6 @@ public Iterator> handleUpdate(String partitionPath, String fil return handleUpdateInternal(upsertHandle, fileId); } - public Iterator> handleUpdate(String partitionPath, String fileId, - Map> keyToNewRecords, - HoodieBaseFile oldDataFile) throws IOException { - // these are updates - HoodieMergeHandle upsertHandle = getUpdateHandle(partitionPath, fileId, keyToNewRecords, oldDataFile); - return handleUpdateInternal(upsertHandle, fileId); - } - protected Iterator> handleUpdateInternal(HoodieMergeHandle upsertHandle, String fileId) throws IOException { if (upsertHandle.getOldFilePath() == null) { diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteCommitActionExecutor.java index 627e75eda565c..2771a22c9f12c 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteCommitActionExecutor.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteCommitActionExecutor.java @@ -28,8 +28,6 @@ import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.WorkloadProfile; import org.apache.hudi.table.action.HoodieWriteMetadata; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; import org.apache.spark.Partitioner; import org.apache.spark.api.java.JavaRDD; import scala.Tuple2; @@ -41,8 +39,6 @@ public class SparkInsertOverwriteCommitActionExecutor> extends BaseSparkCommitActionExecutor { - private static final Logger LOG = LogManager.getLogger(SparkInsertOverwriteCommitActionExecutor.class); - private final JavaRDD> inputRecordsRDD; public SparkInsertOverwriteCommitActionExecutor(HoodieEngineContext context, @@ -53,7 +49,7 @@ public SparkInsertOverwriteCommitActionExecutor(HoodieEngineContext context, } @Override - public HoodieWriteMetadata execute() { + public HoodieWriteMetadata> execute() { return SparkWriteHelper.newInstance().write(instantTime, inputRecordsRDD, context, table, config.shouldCombineBeforeInsert(), config.getInsertShuffleParallelism(), this, false); } @@ -68,6 +64,7 @@ protected String getCommitActionType() { return HoodieTimeline.REPLACE_COMMIT_ACTION; } + @Override protected Map> getPartitionToReplacedFileIds(JavaRDD writeStatuses) { return writeStatuses.map(status -> status.getStat().getPartitionPath()).distinct().mapToPair(partitionPath -> new Tuple2<>(partitionPath, getAllExistingFileIds(partitionPath))).collectAsMap(); diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkMergeHelper.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkMergeHelper.java index 2d130e35d70fb..08d60b93da37a 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkMergeHelper.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkMergeHelper.java @@ -46,6 +46,9 @@ public class SparkMergeHelper extends AbstractMergeHelper>, JavaRDD, JavaRDD> { + private SparkMergeHelper() { + } + private static class MergeHelperHolder { private static final SparkMergeHelper SPARK_MERGE_HELPER = new SparkMergeHelper(); }