HDDS-9967. Intermittent failure in TestOzoneRpcClientAbstract.testListSnapshot.

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java

@@ -1462,10 +1462,6 @@ public static boolean isKeyDeleted(String key, Table keyTable) {
         && omKeyInfoCacheValue.getCacheValue() == null;
   }
 
-  public static boolean isKeyInCache(String key, Table keyTable) {
-    return keyTable.getCacheValue(new CacheKey(key)) != null;
-  }
-
   /**
    * Helper function for listStatus to find key in TableCache.
    */

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ListIterator.java

@@ -27,6 +27,7 @@
 import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 
+import javax.annotation.Nullable;
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -37,6 +38,7 @@
 import java.util.NoSuchElementException;
 import java.util.PriorityQueue;
 import java.util.TreeMap;
+import java.util.function.Predicate;
 
 import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
 
@@ -118,13 +120,16 @@ public static class DbTableIter<Value> implements
 
     private final Table<String, Value> table;
     private HeapEntry currentKey;
+    private Predicate<String> isKeyExistsInCache;
 
     DbTableIter(int entryIteratorId, Table<String, Value> table,
-                String prefixKey, String startKey) throws IOException {
+                String prefixKey, String startKey,
+                Predicate<String> isKeyExistsInCache) throws IOException {
       this.entryIteratorId = entryIteratorId;
       this.table = table;
       this.tableIterator = table.iterator(prefixKey);
       this.currentKey = null;
+      this.isKeyExistsInCache = isKeyExistsInCache;
 
       // only seek for the start key if the start key is lexicographically
       // after the prefix key. For example
@@ -144,7 +149,7 @@ private void getNextKey() throws IOException {
       while (tableIterator.hasNext() && currentKey == null) {
         Table.KeyValue<String, Value> entry = tableIterator.next();
         String entryKey = entry.getKey();
-        if (!KeyManagerImpl.isKeyInCache(entryKey, table)) {
+        if (!isKeyExistsInCache.test(entryKey)) {
           currentKey = new HeapEntry(entryIteratorId,
               table.getName(), entryKey, entry.getValue());
         }
@@ -236,6 +241,10 @@ private void populateCacheMap(Iterator<Map.Entry<CacheKey<String>,
       }
     }
 
+    public boolean isKeyExistInCache(String key) {
+      return cacheKeyMap.containsKey(key);
+    }
+
     public boolean hasNext() {
       return cacheCreatedKeyIter.hasNext();
     }
@@ -292,11 +301,13 @@ public static class MinHeapIterator implements ClosableIterator {
       try {
         int iteratorId = 0;
         for (Table table : tables) {
-          iterators.add(new CacheIter<>(iteratorId, table.getName(),
-                  table.cacheIterator(), startKey, prefixKey));
+          CacheIter cacheIter = new CacheIter<>(iteratorId, table.getName(),
+              table.cacheIterator(), startKey, prefixKey);
+          Predicate<String> isKeyExistInCache = cacheIter::isKeyExistInCache;
+          iterators.add(cacheIter);
           iteratorId++;
           iterators.add(new DbTableIter<>(iteratorId, table, prefixKey,
-              startKey));
+              startKey, isKeyExistInCache));
           iteratorId++;
         }
       } finally {
@@ -321,7 +332,9 @@ public boolean hasNext() {
       return !minHeap.isEmpty();
     }
 
-    public HeapEntry next() {
+    // HeapEntry can be null when a key is marked as deleted because
+    // value of key would be returned as null from cache iterator.
+    public @Nullable HeapEntry next() {
       HeapEntry heapEntry = minHeap.remove();
       // remove the least element and
       // reinsert the next element from the same iterator

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java

@@ -1372,8 +1372,13 @@ public List<SnapshotInfo> listSnapshot(
             bucketName, snapshotInfoTable)) {
       try {
         while (snapshotIterator.hasNext() && maxListResult > 0) {
-          SnapshotInfo snapshotInfo = (SnapshotInfo) snapshotIterator.next()
-              .getValue();
+          Object value = snapshotIterator.next().getValue();
+          // value can be null for deleted snapshot key as CacheIterator from
+          // MinHeapIterator allows null (marked as deleted) values.
+          if (null == value) {
+            continue;
+          }
+          SnapshotInfo snapshotInfo = (SnapshotInfo) value;
           if (!snapshotInfo.getName().equals(prevSnapshot)) {
             snapshotInfos.add(snapshotInfo);
             maxListResult--;