From bfef9298d8b529a062f2e2a9ca08f9ae87b93588 Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal Date: Wed, 12 Feb 2025 16:09:01 +0530 Subject: [PATCH 01/10] HDDS-12288. Improved the bootsrap logging to indcate the progress --- .../OmRatisSnapshotProvider.java | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java index 7031ca0f3d84..efa5423b2b71 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java @@ -18,10 +18,7 @@ package org.apache.hadoop.ozone.om.ratis_snapshot; -import java.io.DataOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.net.HttpURLConnection; import java.net.URL; import java.util.List; @@ -143,8 +140,9 @@ public void downloadSnapshot(String leaderNodeID, File targetFile) OMNodeDetails leader = peerNodesMap.get(leaderNodeID); URL omCheckpointUrl = leader.getOMDBCheckpointEndpointUrl( httpPolicy.isHttpEnabled(), true); - LOG.info("Downloading latest checkpoint from Leader OM {}. Checkpoint " + - "URL: {}", leaderNodeID, omCheckpointUrl); + LOG.info("Downloading latest checkpoint from Leader OM {}. Checkpoint: {} " + "URL: {}", + leaderNodeID, targetFile.getName() , omCheckpointUrl); + SecurityUtil.doAsCurrentUser(() -> { HttpURLConnection connection = (HttpURLConnection) connectionFactory.openConnection(omCheckpointUrl, spnegoEnabled); @@ -165,8 +163,27 @@ public void downloadSnapshot(String leaderNodeID, File targetFile) omCheckpointUrl + ". ErrorCode: " + errorCode); } - try (InputStream inputStream = connection.getInputStream()) { - FileUtils.copyInputStreamToFile(inputStream, targetFile); + try (InputStream inputStream = connection.getInputStream(); + FileOutputStream outputStream = new FileOutputStream(targetFile)) { + byte[] buffer = new byte[8 * 1024]; + long totalBytesRead = 0; + int bytesRead; + long lastLoggedTime = System.currentTimeMillis(); + + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + totalBytesRead += bytesRead; + + // Log progress every 30 seconds + if (System.currentTimeMillis() - lastLoggedTime >= 30000) { + LOG.info("Downloading '{}': {} KB downloaded so far...", + targetFile.getName(), totalBytesRead / (1024)); + lastLoggedTime = System.currentTimeMillis(); + } + } + + LOG.info("Download completed for '{}'. Total size: {} KB", + targetFile.getName(), totalBytesRead / (1024)); } catch (IOException ex) { boolean deleted = FileUtils.deleteQuietly(targetFile); if (!deleted) { From 4a810a5923b2355520f69ff798b9404ad5f469a3 Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal Date: Wed, 12 Feb 2025 16:18:18 +0530 Subject: [PATCH 02/10] HDDS-12288. Improved the bootsrap logging to indcate the progress --- .../ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java index efa5423b2b71..f0204c84b042 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java @@ -18,7 +18,10 @@ package org.apache.hadoop.ozone.om.ratis_snapshot; -import java.io.*; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; import java.util.List; @@ -142,7 +145,6 @@ public void downloadSnapshot(String leaderNodeID, File targetFile) httpPolicy.isHttpEnabled(), true); LOG.info("Downloading latest checkpoint from Leader OM {}. Checkpoint: {} " + "URL: {}", leaderNodeID, targetFile.getName() , omCheckpointUrl); - SecurityUtil.doAsCurrentUser(() -> { HttpURLConnection connection = (HttpURLConnection) connectionFactory.openConnection(omCheckpointUrl, spnegoEnabled); From 1dd90ac68753b0a3f5479041ccf03db7d795ef04 Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal Date: Wed, 12 Feb 2025 16:23:13 +0530 Subject: [PATCH 03/10] HDDS-12288. Improved the bootsrap logging to indcate the progress --- .../hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java index f0204c84b042..8fbca67541bc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java @@ -144,7 +144,7 @@ public void downloadSnapshot(String leaderNodeID, File targetFile) URL omCheckpointUrl = leader.getOMDBCheckpointEndpointUrl( httpPolicy.isHttpEnabled(), true); LOG.info("Downloading latest checkpoint from Leader OM {}. Checkpoint: {} " + "URL: {}", - leaderNodeID, targetFile.getName() , omCheckpointUrl); + leaderNodeID, targetFile.getName(), omCheckpointUrl); SecurityUtil.doAsCurrentUser(() -> { HttpURLConnection connection = (HttpURLConnection) connectionFactory.openConnection(omCheckpointUrl, spnegoEnabled); From cf05f89cfd7f8b591f808083bd64432cd4c0d6ae Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal Date: Wed, 12 Feb 2025 16:30:14 +0530 Subject: [PATCH 04/10] HDDS-12288. Improved the bootsrap logging to indcate the progress --- .../hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java index 8fbca67541bc..00ebeaab4905 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.FileOutputStream; import java.net.HttpURLConnection; import java.net.URL; import java.util.List; From 9aa8c336bfbd82c017244b56d9cb6e40a8acd97a Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal Date: Thu, 13 Feb 2025 11:09:56 +0530 Subject: [PATCH 05/10] HDDS-12288. Improved the bootsrap logging to indcate the progress --- .../OmRatisSnapshotProvider.java | 50 +++++++++++-------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java index 00ebeaab4905..be0f84767edd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java @@ -166,27 +166,8 @@ public void downloadSnapshot(String leaderNodeID, File targetFile) omCheckpointUrl + ". ErrorCode: " + errorCode); } - try (InputStream inputStream = connection.getInputStream(); - FileOutputStream outputStream = new FileOutputStream(targetFile)) { - byte[] buffer = new byte[8 * 1024]; - long totalBytesRead = 0; - int bytesRead; - long lastLoggedTime = System.currentTimeMillis(); - - while ((bytesRead = inputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - totalBytesRead += bytesRead; - - // Log progress every 30 seconds - if (System.currentTimeMillis() - lastLoggedTime >= 30000) { - LOG.info("Downloading '{}': {} KB downloaded so far...", - targetFile.getName(), totalBytesRead / (1024)); - lastLoggedTime = System.currentTimeMillis(); - } - } - - LOG.info("Download completed for '{}'. Total size: {} KB", - targetFile.getName(), totalBytesRead / (1024)); + try (InputStream inputStream = connection.getInputStream()) { + downloadFileWithProgress(inputStream, targetFile); } catch (IOException ex) { boolean deleted = FileUtils.deleteQuietly(targetFile); if (!deleted) { @@ -201,6 +182,33 @@ public void downloadSnapshot(String leaderNodeID, File targetFile) }); } + /** + * Writes data from the given InputStream to the target file while logging download progress every 30 seconds. + */ + public static void downloadFileWithProgress(InputStream inputStream, File targetFile) + throws IOException{ + FileOutputStream outputStream = new FileOutputStream(targetFile); + byte[] buffer = new byte[8 * 1024]; + long totalBytesRead = 0; + int bytesRead; + long lastLoggedTime = System.currentTimeMillis(); + + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + totalBytesRead += bytesRead; + + // Log progress every 30 seconds + if (System.currentTimeMillis() - lastLoggedTime >= 30000) { + LOG.info("Downloading '{}': {} KB downloaded so far...", + targetFile.getName(), totalBytesRead / (1024)); + lastLoggedTime = System.currentTimeMillis(); + } + } + + LOG.info("Download completed for '{}'. Total size: {} KB", + targetFile.getName(), totalBytesRead / (1024)); + } + /** * Writes form data to output stream as any HTTP client would for a * multipart/form-data request. From 2d8fe4241c8f0ca0b8403aa80872fecc5667ce63 Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal Date: Thu, 13 Feb 2025 11:12:51 +0530 Subject: [PATCH 06/10] HDDS-12288. Improved the bootsrap logging to indcate the progress --- .../hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java index be0f84767edd..07b6905bae36 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java @@ -186,7 +186,7 @@ public void downloadSnapshot(String leaderNodeID, File targetFile) * Writes data from the given InputStream to the target file while logging download progress every 30 seconds. */ public static void downloadFileWithProgress(InputStream inputStream, File targetFile) - throws IOException{ + throws IOException { FileOutputStream outputStream = new FileOutputStream(targetFile); byte[] buffer = new byte[8 * 1024]; long totalBytesRead = 0; From d3576d49cd5ca4fc9d7f6fff9d1c18f9552c50b7 Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal Date: Thu, 13 Feb 2025 11:28:04 +0530 Subject: [PATCH 07/10] HDDS-12288. Improved the bootsrap logging to indcate the progress --- .../OmRatisSnapshotProvider.java | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java index 07b6905bae36..afc7ca559ade 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java @@ -187,26 +187,27 @@ public void downloadSnapshot(String leaderNodeID, File targetFile) */ public static void downloadFileWithProgress(InputStream inputStream, File targetFile) throws IOException { - FileOutputStream outputStream = new FileOutputStream(targetFile); - byte[] buffer = new byte[8 * 1024]; - long totalBytesRead = 0; - int bytesRead; - long lastLoggedTime = System.currentTimeMillis(); + try (FileOutputStream outputStream = new FileOutputStream(targetFile)) { + byte[] buffer = new byte[8 * 1024]; + long totalBytesRead = 0; + int bytesRead; + long lastLoggedTime = System.currentTimeMillis(); - while ((bytesRead = inputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - totalBytesRead += bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + totalBytesRead += bytesRead; - // Log progress every 30 seconds - if (System.currentTimeMillis() - lastLoggedTime >= 30000) { - LOG.info("Downloading '{}': {} KB downloaded so far...", - targetFile.getName(), totalBytesRead / (1024)); - lastLoggedTime = System.currentTimeMillis(); + // Log progress every 30 seconds + if (System.currentTimeMillis() - lastLoggedTime >= 30000) { + LOG.info("Downloading '{}': {} KB downloaded so far...", + targetFile.getName(), totalBytesRead / (1024)); + lastLoggedTime = System.currentTimeMillis(); + } } - } - LOG.info("Download completed for '{}'. Total size: {} KB", - targetFile.getName(), totalBytesRead / (1024)); + LOG.info("Download completed for '{}'. Total size: {} KB", + targetFile.getName(), totalBytesRead / (1024)); + } } /** From dd03f4f83bbf7f6623036720130117796029d2f2 Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal <134698352+Gargi-jais11@users.noreply.github.com> Date: Thu, 13 Feb 2025 13:41:39 +0530 Subject: [PATCH 08/10] HDDS-12288. Improve bootstrap logging to indicate progress Co-authored-by: Doroszlai, Attila <6454655+adoroszlai@users.noreply.github.com> --- .../hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java index afc7ca559ade..9af62722b822 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java @@ -144,7 +144,7 @@ public void downloadSnapshot(String leaderNodeID, File targetFile) OMNodeDetails leader = peerNodesMap.get(leaderNodeID); URL omCheckpointUrl = leader.getOMDBCheckpointEndpointUrl( httpPolicy.isHttpEnabled(), true); - LOG.info("Downloading latest checkpoint from Leader OM {}. Checkpoint: {} " + "URL: {}", + LOG.info("Downloading latest checkpoint from Leader OM {}. Checkpoint: {} URL: {}", leaderNodeID, targetFile.getName(), omCheckpointUrl); SecurityUtil.doAsCurrentUser(() -> { HttpURLConnection connection = (HttpURLConnection) From e5f7a597f08d48ab30ed08fd765540476be5addf Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal <134698352+Gargi-jais11@users.noreply.github.com> Date: Thu, 13 Feb 2025 13:54:51 +0530 Subject: [PATCH 09/10] HDDS-12288. Improve bootstrap logging to indicate progress --- .../hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java index 9af62722b822..10c83fb6ec25 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java @@ -145,7 +145,7 @@ public void downloadSnapshot(String leaderNodeID, File targetFile) URL omCheckpointUrl = leader.getOMDBCheckpointEndpointUrl( httpPolicy.isHttpEnabled(), true); LOG.info("Downloading latest checkpoint from Leader OM {}. Checkpoint: {} URL: {}", - leaderNodeID, targetFile.getName(), omCheckpointUrl); + leaderNodeID, targetFile.getName(), omCheckpointUrl); SecurityUtil.doAsCurrentUser(() -> { HttpURLConnection connection = (HttpURLConnection) connectionFactory.openConnection(omCheckpointUrl, spnegoEnabled); From 3cb99a2b4539b3f57a1da40118f8673aed7552f3 Mon Sep 17 00:00:00 2001 From: Gargi Jaiswal <134698352+Gargi-jais11@users.noreply.github.com> Date: Thu, 13 Feb 2025 15:30:54 +0530 Subject: [PATCH 10/10] HDDS-12288. Improve bootstrap logging to indicate progress --- .../ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java index 10c83fb6ec25..6e60f14da302 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java @@ -200,13 +200,13 @@ public static void downloadFileWithProgress(InputStream inputStream, File target // Log progress every 30 seconds if (System.currentTimeMillis() - lastLoggedTime >= 30000) { LOG.info("Downloading '{}': {} KB downloaded so far...", - targetFile.getName(), totalBytesRead / (1024)); + targetFile.getName(), totalBytesRead / (1024)); lastLoggedTime = System.currentTimeMillis(); } } LOG.info("Download completed for '{}'. Total size: {} KB", - targetFile.getName(), totalBytesRead / (1024)); + targetFile.getName(), totalBytesRead / (1024)); } }