HDDS-9967. Intermittent failure in TestOzoneRpcClientAbstract.testListSnapshot.

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java

@@ -1462,10 +1462,6 @@ public static boolean isKeyDeleted(String key, Table keyTable) {
         && omKeyInfoCacheValue.getCacheValue() == null;
   }
 
-  public static boolean isKeyInCache(String key, Table keyTable) {
-    return keyTable.getCacheValue(new CacheKey(key)) != null;
-  }
-
   /**
    * Helper function for listStatus to find key in TableCache.
    */

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ListIterator.java

@@ -37,6 +37,7 @@
 import java.util.NoSuchElementException;
 import java.util.PriorityQueue;
 import java.util.TreeMap;
+import java.util.function.Predicate;
 
 import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
 
@@ -117,14 +118,17 @@ public static class DbTableIter<Value> implements
         ? extends Table.KeyValue<String, Value>> tableIterator;
 
     private final Table<String, Value> table;
-    private HeapEntry currentKey;
+    private HeapEntry currentEntry;
+    private Predicate<String> doesKeyExistInCache;
 
     DbTableIter(int entryIteratorId, Table<String, Value> table,
-                String prefixKey, String startKey) throws IOException {
+                String prefixKey, String startKey,
+                Predicate<String> doesKeyExistInCache) throws IOException {
       this.entryIteratorId = entryIteratorId;
       this.table = table;
       this.tableIterator = table.iterator(prefixKey);
-      this.currentKey = null;
+      this.currentEntry = null;
+      this.doesKeyExistInCache = doesKeyExistInCache;
 
       // only seek for the start key if the start key is lexicographically
       // after the prefix key. For example
@@ -141,11 +145,11 @@ public static class DbTableIter<Value> implements
     }
 
     private void getNextKey() throws IOException {
-      while (tableIterator.hasNext() && currentKey == null) {
+      while (tableIterator.hasNext() && currentEntry == null) {
         Table.KeyValue<String, Value> entry = tableIterator.next();
         String entryKey = entry.getKey();
-        if (!KeyManagerImpl.isKeyInCache(entryKey, table)) {
-          currentKey = new HeapEntry(entryIteratorId,
+        if (!doesKeyExistInCache.test(entryKey)) {
+          currentEntry = new HeapEntry(entryIteratorId,
               table.getName(), entryKey, entry.getValue());
         }
       }
@@ -157,13 +161,13 @@ public boolean hasNext() {
       } catch (IOException t) {
         throw new UncheckedIOException(t);
       }
-      return currentKey != null;
+      return currentEntry != null;
     }
 
     public HeapEntry next() {
       if (hasNext()) {
-        HeapEntry ret = currentKey;
-        currentKey = null;
+        HeapEntry ret = currentEntry;
+        currentEntry = null;
         return ret;
       }
       throw new NoSuchElementException();
@@ -186,15 +190,15 @@ public static class CacheIter<Value>
     private final String prefixKey;
     private final String startKey;
     private final String tableName;
-
+    private HeapEntry currentEntry;
     private final int entryIteratorId;
 
     CacheIter(int entryIteratorId, String tableName,
               Iterator<Map.Entry<CacheKey<String>,
                   CacheValue<Value>>> cacheIter, String startKey,
               String prefixKey) {
       this.cacheKeyMap = new TreeMap<>();
-
+      this.currentEntry = null;
       this.startKey = startKey;
       this.prefixKey = prefixKey;
       this.tableName = tableName;
@@ -236,14 +240,37 @@ private void populateCacheMap(Iterator<Map.Entry<CacheKey<String>,
       }
     }
 
+    public boolean doesKeyExistInCache(String key) {
+      return cacheKeyMap.containsKey(key);
+    }
+
+    private void getNextKey() throws IOException {
+      while (cacheCreatedKeyIter.hasNext() && currentEntry == null) {
+        Map.Entry<String, Value> entry = cacheCreatedKeyIter.next();
+        if (null == entry.getValue()) {
+          continue;
+        }
+        currentEntry = new HeapEntry(this.entryIteratorId, this.tableName,
+            entry.getKey(), entry.getValue());
+      }
+    }
+
     public boolean hasNext() {
-      return cacheCreatedKeyIter.hasNext();
+      try {
+        getNextKey();
+      } catch (IOException t) {
+        throw new UncheckedIOException(t);
+      }
+      return currentEntry != null;
     }
 
     public HeapEntry next() {
-      Map.Entry<String, Value> entry = cacheCreatedKeyIter.next();
-      return new HeapEntry(this.entryIteratorId, this.tableName,
-          entry.getKey(), entry.getValue());
+      if (hasNext()) {
+        HeapEntry ret = currentEntry;
+        currentEntry = null;
+        return ret;
+      }
+      throw new NoSuchElementException();
     }
 
     public void close() {
@@ -292,11 +319,13 @@ public static class MinHeapIterator implements ClosableIterator {
       try {
         int iteratorId = 0;
         for (Table table : tables) {
-          iterators.add(new CacheIter<>(iteratorId, table.getName(),
-                  table.cacheIterator(), startKey, prefixKey));
+          CacheIter cacheIter = new CacheIter<>(iteratorId, table.getName(),
+              table.cacheIterator(), startKey, prefixKey);
+          Predicate<String> doesKeyExistInCache = cacheIter::doesKeyExistInCache;
+          iterators.add(cacheIter);
           iteratorId++;
           iterators.add(new DbTableIter<>(iteratorId, table, prefixKey,
-              startKey));
+              startKey, doesKeyExistInCache));
           iteratorId++;
         }
       } finally {

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java

@@ -1372,8 +1372,8 @@ public List<SnapshotInfo> listSnapshot(
             bucketName, snapshotInfoTable)) {
       try {
         while (snapshotIterator.hasNext() && maxListResult > 0) {
-          SnapshotInfo snapshotInfo = (SnapshotInfo) snapshotIterator.next()
-              .getValue();
+          SnapshotInfo snapshotInfo =
+              (SnapshotInfo) snapshotIterator.next().getValue();
           if (!snapshotInfo.getName().equals(prevSnapshot)) {
             snapshotInfos.add(snapshotInfo);
             maxListResult--;