From b46f2d69debcc6144dd0c27a990cd4298d65675f Mon Sep 17 00:00:00 2001 From: wxp <408317717@qq.com> Date: Thu, 13 Oct 2022 10:18:35 +0000 Subject: [PATCH 1/2] create new file when rollOver --- .../org/apache/hudi/common/table/log/HoodieLogFormatWriter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java index 8dbe85efd1164..0363fa5292308 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java @@ -220,6 +220,7 @@ private void rolloverIfNeeded() throws IOException { LOG.info("CurrentSize " + getCurrentSize() + " has reached threshold " + sizeThreshold + ". Rolling over to the next version"); rollOver(); + createNewFile(); } } From 33e1c0d4e837080818ddd7faf5517c7f0c2ca3dd Mon Sep 17 00:00:00 2001 From: wxp <408317717@qq.com> Date: Tue, 8 Nov 2022 05:42:57 +0000 Subject: [PATCH 2/2] fix rollback file not found --- .../hudi/table/action/rollback/BaseRollbackHelper.java | 8 ++++++-- .../hudi/common/table/log/HoodieLogFormatWriter.java | 1 - 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackHelper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackHelper.java index 7cd09930c1f08..fa49d21391b80 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackHelper.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackHelper.java @@ -120,6 +120,7 @@ List> maybeDeleteAndCollectStats(HoodieEngineCo return partitionToRollbackStats.stream(); } else if (!rollbackRequest.getLogBlocksToBeDeleted().isEmpty()) { HoodieLogFormat.Writer writer = null; + final Path filePath; try { String fileId = rollbackRequest.getFileId(); String latestBaseInstant = rollbackRequest.getLatestBaseInstant(); @@ -135,7 +136,10 @@ List> maybeDeleteAndCollectStats(HoodieEngineCo if (doDelete) { Map header = generateHeader(instantToRollback.getTimestamp()); // if update belongs to an existing log file - writer.appendBlock(new HoodieCommandBlock(header)); + // use the log file path from AppendResult in case the file handle may roll over + filePath = writer.appendBlock(new HoodieCommandBlock(header)).logFile().getPath(); + } else { + filePath = writer.getLogFile().getPath(); } } catch (IOException | InterruptedException io) { throw new HoodieRollbackException("Failed to rollback for instant " + instantToRollback, io); @@ -153,7 +157,7 @@ List> maybeDeleteAndCollectStats(HoodieEngineCo // getFileStatus would reflect correct stats and FileNotFoundException is not thrown in // cloud-storage : HUDI-168 Map filesToNumBlocksRollback = Collections.singletonMap( - metaClient.getFs().getFileStatus(Objects.requireNonNull(writer).getLogFile().getPath()), + metaClient.getFs().getFileStatus(Objects.requireNonNull(filePath)), 1L ); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java index 0363fa5292308..8dbe85efd1164 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java @@ -220,7 +220,6 @@ private void rolloverIfNeeded() throws IOException { LOG.info("CurrentSize " + getCurrentSize() + " has reached threshold " + sizeThreshold + ". Rolling over to the next version"); rollOver(); - createNewFile(); } }