From 00e354c03ce2f8dc6f8285dcbc861027027cfbca Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 10 Jan 2025 10:18:25 -0800 Subject: [PATCH 1/3] HDDS-12065. Checkpoint directory should be cleared on startup Change-Id: Ie840ca2d5ccbce02f7a19c66f10eb7b7f810d9f6 --- .../hdds/utils/DBCheckpointServlet.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index cb1fdd3375a8..69b250a8c3be 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -24,9 +24,12 @@ import java.io.File; import java.io.IOException; import java.io.OutputStream; +import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -107,6 +110,13 @@ public void initialize(DBStore store, DBCheckpointMetrics metrics, } bootstrapTempData = Paths.get(tempData, "temp-bootstrap-data").toFile(); + if (bootstrapTempData.exists()) { + try { + deleteContents(bootstrapTempData.toPath()); + } catch (IOException e) { + throw new ServletException(e); + } + } if (!bootstrapTempData.exists() && !bootstrapTempData.mkdirs()) { throw new ServletException("Failed to make:" + bootstrapTempData); @@ -123,6 +133,24 @@ private boolean hasPermission(UserGroupInformation user) { } } + private void deleteContents(Path directory) throws IOException { + Files.walkFileTree(directory, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); // Delete file + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + if (!dir.equals(directory)) { // Skip deleting the root directory + Files.delete(dir); + } + return FileVisitResult.CONTINUE; + } + }); + } + /** * Generates Snapshot checkpoint as tar ball. * @param request the HTTP servlet request From 150c4afd03705f6e3263e2f7737e3e8a57014ff1 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Thu, 20 Feb 2025 15:26:54 -0800 Subject: [PATCH 2/3] Changed it to use Appache common's FileUtils.cleanDirectory() --- .../hdds/utils/DBCheckpointServlet.java | 23 +------------------ 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index 69b250a8c3be..42750a5ed663 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -24,12 +24,9 @@ import java.io.File; import java.io.IOException; import java.io.OutputStream; -import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -112,7 +109,7 @@ public void initialize(DBStore store, DBCheckpointMetrics metrics, "temp-bootstrap-data").toFile(); if (bootstrapTempData.exists()) { try { - deleteContents(bootstrapTempData.toPath()); + FileUtils.cleanDirectory(bootstrapTempData); } catch (IOException e) { throw new ServletException(e); } @@ -133,24 +130,6 @@ private boolean hasPermission(UserGroupInformation user) { } } - private void deleteContents(Path directory) throws IOException { - Files.walkFileTree(directory, new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Files.delete(file); // Delete file - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - if (!dir.equals(directory)) { // Skip deleting the root directory - Files.delete(dir); - } - return FileVisitResult.CONTINUE; - } - }); - } - /** * Generates Snapshot checkpoint as tar ball. * @param request the HTTP servlet request From 9f5af15542dfbf93b3b8902950e25ea4acce41cc Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Thu, 20 Feb 2025 15:32:07 -0800 Subject: [PATCH 3/3] Log the exception and changed the expcetion message. --- .../java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index 42750a5ed663..a40aba548424 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -111,7 +111,8 @@ public void initialize(DBStore store, DBCheckpointMetrics metrics, try { FileUtils.cleanDirectory(bootstrapTempData); } catch (IOException e) { - throw new ServletException(e); + LOG.error("Failed to clean-up: {} dir.", bootstrapTempData); + throw new ServletException("Failed to clean-up: " + bootstrapTempData); } } if (!bootstrapTempData.exists() &&