diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java index c2325b5ae1e0..17100cdd384e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java @@ -227,11 +227,9 @@ public static void teardown() { @After public void cleanup() { try { - FileStatus[] fileStatuses = fs.listStatus(ROOT); - for (FileStatus fileStatus : fileStatuses) { - fs.delete(fileStatus.getPath(), true); - } - } catch (IOException ex) { + deleteRootDir(); + } catch (IOException | InterruptedException ex) { + LOG.error("Failed to cleanup files.", ex); fail("Failed to cleanup files."); } } @@ -797,21 +795,25 @@ public void testListStatusOnKeyNameContainDelimiter() throws Exception { * * @throws IOException DB failure */ - protected void deleteRootDir() throws IOException { + protected void deleteRootDir() throws IOException, InterruptedException { FileStatus[] fileStatuses = fs.listStatus(ROOT); if (fileStatuses == null) { return; } + deleteRootRecursively(fileStatuses); + fileStatuses = fs.listStatus(ROOT); + if (fileStatuses != null) { + Assert.assertEquals( + "Delete root failed!", 0, fileStatuses.length); + } + } + private static void deleteRootRecursively(FileStatus[] fileStatuses) + throws IOException { for (FileStatus fStatus : fileStatuses) { fs.delete(fStatus.getPath(), true); } - - fileStatuses = fs.listStatus(ROOT); - if (fileStatuses != null) { - Assert.assertEquals("Delete root failed!", 0, fileStatuses.length); - } } /** diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithFSO.java index ed148bc45fcf..ce637f8ea912 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithFSO.java @@ -85,12 +85,6 @@ public TestOzoneFileSystemWithFSO(boolean setDefaultFs, @Override public void cleanup() { super.cleanup(); - try { - deleteRootDir(); - } catch (IOException e) { - LOG.info("Failed to cleanup DB tables.", e); - fail("Failed to cleanup DB tables." + e.getMessage()); - } } private static final Logger LOG = diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java index 55ffca602a16..2b9fa987363d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java @@ -70,6 +70,7 @@ import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE; import static org.apache.hadoop.hdds.client.ReplicationType.RATIS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; +import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_EXISTS; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; @@ -80,7 +81,8 @@ * Tests to verify Object store with prefix enabled cases. */ public class TestObjectStoreWithFSO { - + private static final Path ROOT = + new Path(OZONE_URI_DELIMITER); private static MiniOzoneCluster cluster = null; private static OzoneConfiguration conf; private static String clusterId; @@ -138,22 +140,25 @@ public void tearDown() throws Exception { * * @throws IOException DB failure */ - private void deleteRootDir() throws IOException { - Path root = new Path("/"); - FileStatus[] fileStatuses = fs.listStatus(root); + protected void deleteRootDir() throws IOException { + FileStatus[] fileStatuses = fs.listStatus(ROOT); if (fileStatuses == null) { return; } + deleteRootRecursively(fileStatuses); + fileStatuses = fs.listStatus(ROOT); + if (fileStatuses != null) { + Assert.assertEquals( + "Delete root failed!", 0, fileStatuses.length); + } + } + private static void deleteRootRecursively(FileStatus[] fileStatuses) + throws IOException { for (FileStatus fStatus : fileStatuses) { fs.delete(fStatus.getPath(), true); } - - fileStatuses = fs.listStatus(root); - if (fileStatuses != null) { - Assert.assertEquals("Delete root failed!", 0, fileStatuses.length); - } } @Test diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java index 2c13e5cb5ddf..758206f200b7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java @@ -51,6 +51,7 @@ import java.util.NoSuchElementException; import java.util.Collection; import java.util.Collections; +import java.util.stream.Collectors; import static org.apache.hadoop.ozone.om.exceptions.OMException. @@ -214,11 +215,12 @@ public Collection listStatusFSO(OmKeyArgs args, HeapEntry entry = heapIterator.next(); OzoneFileStatus status = entry.getStatus(prefixKey, scmBlockSize, volumeName, bucketName, replication); - map.put(entry.key, status); + map.putIfAbsent(entry.key, status); } } - return map.values(); + return map.values().stream().filter(e -> e != null).collect( + Collectors.toList()); } private String getDbKey(String key, OmKeyArgs args, @@ -285,7 +287,7 @@ private static class HeapEntry implements Comparable { private final Object value; HeapEntry(EntryType entryType, String key, Object value) { - Preconditions.checkArgument( + Preconditions.checkArgument(value == null || value instanceof OmDirectoryInfo || value instanceof OmKeyInfo); this.entryType = entryType; @@ -322,6 +324,9 @@ public OzoneFileStatus getStatus( String bucketName, ReplicationConfig bucketReplication ) { + if (value == null) { + return null; + } OmKeyInfo keyInfo; if (entryType.isDir()) { Preconditions.checkArgument(value instanceof OmDirectoryInfo); @@ -455,10 +460,6 @@ private void getCacheValues() { cacheIter.next(); String cacheKey = entry.getKey().getCacheKey(); Value cacheOmInfo = entry.getValue().getCacheValue(); - // cacheOmKeyInfo is null if an entry is deleted in cache - if (cacheOmInfo == null) { - continue; - } // Copy cache value to local copy and work on it if (cacheOmInfo instanceof CopyObject) {