diff --git a/hudi-common/pom.xml b/hudi-common/pom.xml index 2981155a277f8..41588ca8996c6 100644 --- a/hudi-common/pom.xml +++ b/hudi-common/pom.xml @@ -144,6 +144,7 @@ * + provided org.apache.hadoop @@ -154,6 +155,7 @@ org.apache.hadoop hadoop-hdfs + provided org.apache.hadoop diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java index a2b3889829780..341e3f5bc6e44 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java @@ -80,9 +80,6 @@ public class FSUtils { private static final PathFilter ALLOW_ALL_FILTER = file -> true; public static Configuration prepareHadoopConf(Configuration conf) { - conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()); - conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName()); - // look for all properties, prefixed to be picked up for (Entry prop : System.getenv().entrySet()) { if (prop.getKey().startsWith(HOODIE_ENV_PROPS_PREFIX)) { @@ -607,8 +604,8 @@ public static HoodieWrapperFileSystem getFs(String path, SerializableConfigurati * Helper to filter out paths under metadata folder when running fs.globStatus. * @param fs File System * @param globPath Glob Path - * @return - * @throws IOException + * @return the file status list of globPath exclude the meta folder + * @throws IOException when having trouble listing the path */ public static List getGlobStatusExcludingMetaFolder(FileSystem fs, Path globPath) throws IOException { FileStatus[] statuses = fs.globStatus(globPath); diff --git a/hudi-flink/pom.xml b/hudi-flink/pom.xml index 2a0f39556fb3c..81a7849a5ba4b 100644 --- a/hudi-flink/pom.xml +++ b/hudi-flink/pom.xml @@ -153,39 +153,11 @@ provided - - - org.apache.hadoop - hadoop-common - compile - - - org.slf4j - slf4j-log4j12 - - - - - org.apache.hadoop - hadoop-hdfs - compile - - - org.slf4j - slf4j-log4j12 - - - + - org.apache.hadoop - hadoop-auth - compile - - - org.slf4j - slf4j-log4j12 - - + org.apache.parquet + parquet-avro + test @@ -197,13 +169,6 @@ compile - - - org.apache.parquet - parquet-avro - compile - - org.apache.hadoop