diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java index 06a2cfa2aa5b4..7f8b5f19d714a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java @@ -532,6 +532,13 @@ public static Path getPartitionPath(Path basePath, String partitionPath) { return ((partitionPath == null) || (partitionPath.isEmpty())) ? basePath : new Path(basePath, partitionPath); } + /** + * Get DFS full partition path (e.g. hdfs://ip-address:8020:/) + */ + public static String getDFSFullPartitionPath(FileSystem fs, Path partitionPath) { + return fs.getUri() + partitionPath.toUri().getRawPath(); + } + /** * This is due to HUDI-140 GCS has a different behavior for detecting EOF during seek(). * diff --git a/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java b/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java index 7b5aa8a541c01..05c273c1f225b 100644 --- a/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java +++ b/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java @@ -49,6 +49,7 @@ import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.storage.StorageSchemes; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; @@ -192,7 +193,9 @@ private List constructChangePartitions(List partitions) { String alterTable = "ALTER TABLE " + syncConfig.tableName; for (String partition : partitions) { String partitionClause = getPartitionClause(partition); - String fullPartitionPath = FSUtils.getPartitionPath(syncConfig.basePath, partition).toString(); + Path partitionPath = FSUtils.getPartitionPath(syncConfig.basePath, partition); + String fullPartitionPath = partitionPath.toUri().getScheme().equals(StorageSchemes.HDFS.getScheme()) + ? FSUtils.getDFSFullPartitionPath(fs, partitionPath) : partitionPath.toString(); String changePartition = alterTable + " PARTITION (" + partitionClause + ") SET LOCATION '" + fullPartitionPath + "'"; changePartitions.add(changePartition); @@ -216,7 +219,8 @@ List getPartitionEvents(List tablePartitions, List events = Lists.newArrayList(); for (String storagePartition : partitionStoragePartitions) { - String fullStoragePartitionPath = FSUtils.getPartitionPath(syncConfig.basePath, storagePartition).toString(); + Path storagePartitionPath = FSUtils.getPartitionPath(syncConfig.basePath, storagePartition); + String fullStoragePartitionPath = Path.getPathWithoutSchemeAndAuthority(storagePartitionPath).toUri().getPath(); // Check if the partition values or if hdfs path is the same List storagePartitionValues = partitionValueExtractor.extractPartitionValuesInPath(storagePartition); Collections.sort(storagePartitionValues);