diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java b/hudi-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java index 01585f9259cf1..afce8f6ec4546 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java @@ -18,18 +18,21 @@ package org.apache.hudi.common.config; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.ValidationUtils; +import org.apache.hudi.exception.HoodieIOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import javax.annotation.Nonnull; import javax.annotation.Nullable; + import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -53,10 +56,9 @@ public class DFSPropertiesConfiguration { private static final Logger LOG = LogManager.getLogger(DFSPropertiesConfiguration.class); public static final String DEFAULT_PROPERTIES_FILE = "hudi-defaults.conf"; - public static final String CONF_FILE_DIR_ENV_NAME = "HUDI_CONF_DIR"; - public static final String DEFAULT_CONF_FILE_DIR = "file:/etc/hudi/conf"; + public static final Path DEFAULT_PATH = new Path(DEFAULT_CONF_FILE_DIR + "/" + DEFAULT_PROPERTIES_FILE); // props read from hudi-defaults.conf private static TypedProperties GLOBAL_PROPS = loadGlobalProps(); @@ -97,11 +99,7 @@ public static TypedProperties loadGlobalProps() { if (defaultConfPath.isPresent()) { conf.addPropsFromFile(defaultConfPath.get()); } else { - try { - conf.addPropsFromFile(new Path(DEFAULT_CONF_FILE_DIR)); - } catch (Exception ignored) { - LOG.warn("Didn't find config file under default conf file dir: " + DEFAULT_CONF_FILE_DIR); - } + conf.addPropsFromFile(DEFAULT_PATH); } return conf.getProps(); } @@ -128,14 +126,19 @@ public void addPropsFromFile(Path filePath) { filePath.toString(), Option.ofNullable(hadoopConfig).orElseGet(Configuration::new) ); + try { + if (filePath.equals(DEFAULT_PATH) && !fs.exists(filePath)) { + LOG.warn("Properties file " + filePath + " not found. Ignoring to load props file"); + return; + } - try (BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(filePath)))) { + BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(filePath))); visitedFilePaths.add(filePath.toString()); currentFilePath = filePath; addPropsFromStream(reader); } catch (IOException ioe) { - LOG.error("Error reading in properties from dfs"); - throw new IllegalArgumentException("Cannot read properties from dfs", ioe); + LOG.error("Error reading in properties from dfs from file " + filePath); + throw new HoodieIOException("Cannot read properties from dfs from file " + filePath, ioe); } } @@ -192,7 +195,7 @@ private static Option getConfPathFromEnv() { } private String[] splitProperty(String line) { - line = line.replaceAll("\\s+"," "); + line = line.replaceAll("\\s+", " "); String delimiter = line.contains("=") ? "=" : " "; int ind = line.indexOf(delimiter); String k = line.substring(0, ind).trim(); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java index 6a7075f551c68..465739340dc86 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java @@ -145,7 +145,7 @@ public void testIncludes() { } @Test - public void testLocalFileSystemLoading() { + public void testLocalFileSystemLoading() throws IOException { DFSPropertiesConfiguration cfg = new DFSPropertiesConfiguration(dfs.getConf(), new Path(dfsBasePath + "/t1.props")); cfg.addPropsFromFile( @@ -156,8 +156,7 @@ public void testLocalFileSystemLoading() { .getResource("props/test.properties") .getPath() ) - ) - ); + )); TypedProperties props = cfg.getProps();