From ff94048a928166bf4ff5c8bfef49a7f9ea43f301 Mon Sep 17 00:00:00 2001 From: Sivabalan Narayanan Date: Wed, 27 Jan 2021 08:54:40 -0500 Subject: [PATCH 1/2] Optimizing mkdirs call only if dir does not exists --- .../src/main/java/org/apache/hudi/io/HoodieWriteHandle.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java index d2e538e2b661c..c3854b8671026 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java @@ -108,7 +108,9 @@ private String makeWriteToken() { public Path makeNewPath(String partitionPath) { Path path = FSUtils.getPartitionPath(config.getBasePath(), partitionPath); try { - fs.mkdirs(path); // create a new partition as needed. + if(!fs.exists(path)) { + fs.mkdirs(path); // create a new partition as needed. + } } catch (IOException e) { throw new HoodieIOException("Failed to make dir " + path, e); } From 4d4a3c28c1cd6eabde17461eb80c5cf9ac163ea3 Mon Sep 17 00:00:00 2001 From: Sivabalan Narayanan Date: Wed, 27 Jan 2021 19:14:47 -0500 Subject: [PATCH 2/2] Optimizing mkdir calls by calling only if dir not exists --- .../src/main/java/org/apache/hudi/io/HoodieWriteHandle.java | 2 +- .../src/main/java/org/apache/hudi/table/MarkerFiles.java | 4 +++- .../main/java/org/apache/hudi/io/HoodieRowCreateHandle.java | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java index c3854b8671026..df0c2b6e25c31 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java @@ -108,7 +108,7 @@ private String makeWriteToken() { public Path makeNewPath(String partitionPath) { Path path = FSUtils.getPartitionPath(config.getBasePath(), partitionPath); try { - if(!fs.exists(path)) { + if (!fs.exists(path)) { fs.mkdirs(path); // create a new partition as needed. } } catch (IOException e) { diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/MarkerFiles.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/MarkerFiles.java index 9341a1d8038e3..36112456f2660 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/MarkerFiles.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/MarkerFiles.java @@ -189,7 +189,9 @@ private String stripMarkerFolderPrefix(String fullMarkerPath) { public Path create(String partitionPath, String dataFileName, IOType type) { Path path = FSUtils.getPartitionPath(markerDirPath, partitionPath); try { - fs.mkdirs(path); // create a new partition as needed. + if (!fs.exists(path)) { + fs.mkdirs(path); // create a new partition as needed. + } } catch (IOException e) { throw new HoodieIOException("Failed to make dir " + path, e); } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/HoodieRowCreateHandle.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/HoodieRowCreateHandle.java index fa160c6919458..611e3c8b93c5b 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/HoodieRowCreateHandle.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/HoodieRowCreateHandle.java @@ -172,7 +172,9 @@ public String getFileName() { private Path makeNewPath(String partitionPath) { Path path = FSUtils.getPartitionPath(writeConfig.getBasePath(), partitionPath); try { - fs.mkdirs(path); // create a new partition as needed. + if (!fs.exists(path)) { + fs.mkdirs(path); // create a new partition as needed. + } } catch (IOException e) { throw new HoodieIOException("Failed to make dir " + path, e); }