From f2baeebee296d7f79c325a4d4ca63b50d189733d Mon Sep 17 00:00:00 2001 From: Y Ethan Guo Date: Thu, 13 Jan 2022 19:49:11 -0800 Subject: [PATCH 1/2] [MINOR] Fix local flaky test in TestFSUtils --- .../src/test/java/org/apache/hudi/common/fs/TestFSUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java index 3a7fa690721cc..5fe0a18641c49 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java @@ -470,12 +470,12 @@ public void testGetFileStatusAtLevel() throws IOException { List fileStatusList = FSUtils.getFileStatusAtLevel( new HoodieLocalEngineContext(fileSystem.getConf()), fileSystem, new Path(basePath), 3, 2); - assertEquals(CollectionUtils.createImmutableList( + assertEquals(CollectionUtils.createImmutableSet( "file:" + basePath + "/.hoodie/.temp/subdir1/file1.txt", "file:" + basePath + "/.hoodie/.temp/subdir2/file2.txt"), fileStatusList.stream() .map(fileStatus -> fileStatus.getPath().toString()) .filter(filePath -> filePath.endsWith(".txt")) - .collect(Collectors.toList())); + .collect(Collectors.toSet())); } } From 626cecc34f53634052875ee006cd969b6f0e927b Mon Sep 17 00:00:00 2001 From: Raymond Xu <2701446+xushiyan@users.noreply.github.com> Date: Thu, 13 Jan 2022 20:23:34 -0800 Subject: [PATCH 2/2] try file scheme --- .../java/org/apache/hudi/common/fs/TestFSUtils.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java index 5fe0a18641c49..3e8ad220ba90c 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java @@ -37,10 +37,10 @@ import org.junit.Rule; import org.junit.contrib.java.lang.system.EnvironmentVariables; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; @@ -77,6 +77,7 @@ public class TestFSUtils extends HoodieCommonTestHarness { @BeforeEach public void setUp() throws IOException { initMetaClient(); + basePath = "file:" + basePath; } @Test @@ -312,7 +313,7 @@ public void testFileNameRelatedFunctions() throws Exception { assertEquals(LOG_STR, FSUtils.getFileExtensionFromLog(new Path(logFileName))); // create three versions of log file - java.nio.file.Path partitionPath = Paths.get(basePath, partitionStr); + java.nio.file.Path partitionPath = Paths.get(URI.create(basePath + "/" + partitionStr)); Files.createDirectories(partitionPath); String log1 = FSUtils.makeLogFileName(fileId, LOG_EXTENTION, instantTime, 1, writeToken); Files.createFile(partitionPath.resolve(log1)); @@ -380,7 +381,6 @@ public void testDeleteNonExistingDir() throws IOException { new HoodieLocalEngineContext(metaClient.getHadoopConf()), fileSystem, new Path(rootDir), 2)); } - @Disabled @Test public void testDeleteSubDirectoryRecursively() throws IOException { String rootDir = basePath + "/.hoodie/.temp"; @@ -405,7 +405,6 @@ public void testDeleteSubDirectoryNonRecursively() throws IOException { subPathStr, new SerializableConfiguration(fileSystem.getConf()), false)); } - @Disabled @Test public void testDeleteSubPathAsFile() throws IOException { String rootDir = basePath + "/.hoodie/.temp"; @@ -417,7 +416,6 @@ public void testDeleteSubPathAsFile() throws IOException { subPathStr, new SerializableConfiguration(fileSystem.getConf()), false)); } - @Disabled @Test public void testDeleteNonExistingSubDirectory() throws IOException { String rootDir = basePath + "/.hoodie/.temp"; @@ -461,7 +459,6 @@ public void testParallelizeSubPathProcessWithExistingDir() throws IOException { } } - @Disabled @Test public void testGetFileStatusAtLevel() throws IOException { String rootDir = basePath + "/.hoodie/.temp"; @@ -471,8 +468,8 @@ public void testGetFileStatusAtLevel() throws IOException { new HoodieLocalEngineContext(fileSystem.getConf()), fileSystem, new Path(basePath), 3, 2); assertEquals(CollectionUtils.createImmutableSet( - "file:" + basePath + "/.hoodie/.temp/subdir1/file1.txt", - "file:" + basePath + "/.hoodie/.temp/subdir2/file2.txt"), + basePath + "/.hoodie/.temp/subdir1/file1.txt", + basePath + "/.hoodie/.temp/subdir2/file2.txt"), fileStatusList.stream() .map(fileStatus -> fileStatus.getPath().toString()) .filter(filePath -> filePath.endsWith(".txt"))