HDDS-9041. Intermittent Delete root failed.

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java

@@ -227,11 +227,9 @@ public static void teardown() {
   @After
   public void cleanup() {
     try {
-      FileStatus[] fileStatuses = fs.listStatus(ROOT);
-      for (FileStatus fileStatus : fileStatuses) {
-        fs.delete(fileStatus.getPath(), true);
-      }
-    } catch (IOException ex) {
+      deleteRootDir();
+    } catch (IOException | InterruptedException ex) {
+      LOG.error("Failed to cleanup files.", ex);
       fail("Failed to cleanup files.");
     }
   }
@@ -797,21 +795,25 @@ public void testListStatusOnKeyNameContainDelimiter() throws Exception {
    *
    * @throws IOException DB failure
    */
-  protected void deleteRootDir() throws IOException {
+  protected void deleteRootDir() throws IOException, InterruptedException {
     FileStatus[] fileStatuses = fs.listStatus(ROOT);
 
     if (fileStatuses == null) {
       return;
     }
+    deleteRootRecursively(fileStatuses);
+    fileStatuses = fs.listStatus(ROOT);
+    if (fileStatuses != null) {
+      Assert.assertEquals(
+          "Delete root failed!", 0, fileStatuses.length);
+    }
+  }
 
+  private static void deleteRootRecursively(FileStatus[] fileStatuses)
+      throws IOException {
     for (FileStatus fStatus : fileStatuses) {
       fs.delete(fStatus.getPath(), true);
     }
-
-    fileStatuses = fs.listStatus(ROOT);
-    if (fileStatuses != null) {
-      Assert.assertEquals("Delete root failed!", 0, fileStatuses.length);
-    }
   }
 
   /**

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithFSO.java

@@ -85,12 +85,6 @@ public TestOzoneFileSystemWithFSO(boolean setDefaultFs,
   @Override
   public void cleanup() {
     super.cleanup();
-    try {
-      deleteRootDir();
-    } catch (IOException e) {
-      LOG.info("Failed to cleanup DB tables.", e);
-      fail("Failed to cleanup DB tables." + e.getMessage());
-    }
   }
 
   private static final Logger LOG =

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java

@@ -70,6 +70,7 @@
 import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE;
 import static org.apache.hadoop.hdds.client.ReplicationType.RATIS;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
 import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_EXISTS;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
@@ -80,7 +81,8 @@
  * Tests to verify Object store with prefix enabled cases.
  */
 public class TestObjectStoreWithFSO {
-
+  private static final Path ROOT =
+      new Path(OZONE_URI_DELIMITER);
   private static MiniOzoneCluster cluster = null;
   private static OzoneConfiguration conf;
   private static String clusterId;
@@ -138,22 +140,25 @@ public void tearDown() throws Exception {
    *
    * @throws IOException DB failure
    */
-  private void deleteRootDir() throws IOException {
-    Path root = new Path("/");
-    FileStatus[] fileStatuses = fs.listStatus(root);
+  protected void deleteRootDir() throws IOException {
+    FileStatus[] fileStatuses = fs.listStatus(ROOT);
 
     if (fileStatuses == null) {
       return;
     }
+    deleteRootRecursively(fileStatuses);
+    fileStatuses = fs.listStatus(ROOT);
+    if (fileStatuses != null) {
+      Assert.assertEquals(
+          "Delete root failed!", 0, fileStatuses.length);
+    }
+  }
 
+  private static void deleteRootRecursively(FileStatus[] fileStatuses)
+      throws IOException {
     for (FileStatus fStatus : fileStatuses) {
       fs.delete(fStatus.getPath(), true);
     }
-
-    fileStatuses = fs.listStatus(root);
-    if (fileStatuses != null) {
-      Assert.assertEquals("Delete root failed!", 0, fileStatuses.length);
-    }
   }
 
   @Test

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java

@@ -51,6 +51,7 @@
 import java.util.NoSuchElementException;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.stream.Collectors;
 
 
 import static org.apache.hadoop.ozone.om.exceptions.OMException.
@@ -214,11 +215,12 @@ public Collection<OzoneFileStatus> listStatusFSO(OmKeyArgs args,
         HeapEntry entry = heapIterator.next();
         OzoneFileStatus status = entry.getStatus(prefixKey,
             scmBlockSize, volumeName, bucketName, replication);
-        map.put(entry.key, status);
+        map.putIfAbsent(entry.key, status);
       }
     }
 
-    return map.values();
+    return map.values().stream().filter(e -> e != null).collect(
+        Collectors.toList());
   }
 
   private String getDbKey(String key, OmKeyArgs args,
@@ -285,7 +287,7 @@ private static class HeapEntry implements Comparable<HeapEntry> {
     private final Object value;
 
     HeapEntry(EntryType entryType, String key, Object value) {
-      Preconditions.checkArgument(
+      Preconditions.checkArgument(value == null ||
           value instanceof OmDirectoryInfo ||
               value instanceof OmKeyInfo);
       this.entryType = entryType;
@@ -322,6 +324,9 @@ public OzoneFileStatus getStatus(
         String bucketName,
         ReplicationConfig bucketReplication
     ) {
+      if (value == null) {
+        return null;
+      }
       OmKeyInfo keyInfo;
       if (entryType.isDir()) {
         Preconditions.checkArgument(value instanceof OmDirectoryInfo);
@@ -455,10 +460,6 @@ private void getCacheValues() {
             cacheIter.next();
         String cacheKey = entry.getKey().getCacheKey();
         Value cacheOmInfo = entry.getValue().getCacheValue();
-        // cacheOmKeyInfo is null if an entry is deleted in cache
-        if (cacheOmInfo == null) {
-          continue;
-        }
 
         // Copy cache value to local copy and work on it
         if (cacheOmInfo instanceof CopyObject) {