HDDS-13276. Use KEY_ONLY/VALUE_ONLY iterator in SCM/Datanode.

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java

@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.ozone.container.keyvalue;
 
+import static org.apache.hadoop.hdds.utils.db.Table.KeyValueIterator.Type.VALUE_ONLY;
 import static org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil.isSameSchemaVersion;
 
 import com.fasterxml.jackson.databind.JsonNode;
@@ -35,7 +36,6 @@
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
 import org.apache.hadoop.hdds.server.JsonUtils;
 import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.hdds.utils.db.TableIterator;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.container.common.helpers.BlockData;
 import org.apache.hadoop.ozone.container.common.impl.ContainerData;
@@ -500,8 +500,7 @@ static PendingDelete countPendingDeletesSchemaV2(
     Table<Long, DeletedBlocksTransaction> delTxTable =
         schemaTwoStore.getDeleteTransactionTable();
 
-    try (TableIterator<Long, ? extends Table.KeyValue<Long,
-        DeletedBlocksTransaction>> iterator = delTxTable.iterator()) {
+    try (Table.KeyValueIterator<Long, DeletedBlocksTransaction> iterator = delTxTable.iterator(VALUE_ONLY)) {
       while (iterator.hasNext()) {
         DeletedBlocksTransaction txn = iterator.next().getValue();
         final List<Long> localIDs = txn.getLocalIDList();
@@ -544,11 +543,8 @@ static PendingDelete countPendingDeletesSchemaV3(
       KeyValueContainerData containerData) throws IOException {
     long pendingDeleteBlockCountTotal = 0;
     long pendingDeleteBytes = 0;
-    try (
-        TableIterator<String, ? extends Table.KeyValue<String,
-            DeletedBlocksTransaction>>
-            iter = store.getDeleteTransactionTable()
-            .iterator(containerData.containerPrefix())) {
+    try (Table.KeyValueIterator<String, DeletedBlocksTransaction> iter
+             = store.getDeleteTransactionTable().iterator(containerData.containerPrefix(), VALUE_ONLY)) {
       while (iter.hasNext()) {
         DeletedBlocksTransaction delTx = iter.next().getValue();
         final List<Long> localIDs = delTx.getLocalIDList();

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingTask.java

@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.ozone.container.keyvalue.statemachine.background;
 
+import static org.apache.hadoop.hdds.utils.db.Table.KeyValueIterator.Type.VALUE_ONLY;
 import static org.apache.hadoop.ozone.OzoneConsts.SCHEMA_V1;
 import static org.apache.hadoop.ozone.OzoneConsts.SCHEMA_V2;
 import static org.apache.hadoop.ozone.OzoneConsts.SCHEMA_V3;
@@ -39,7 +40,6 @@
 import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter;
 import org.apache.hadoop.hdds.utils.db.BatchOperation;
 import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.hdds.utils.db.TableIterator;
 import org.apache.hadoop.ozone.container.common.helpers.BlockData;
 import org.apache.hadoop.ozone.container.common.helpers.BlockDeletingServiceMetrics;
 import org.apache.hadoop.ozone.container.common.impl.BlockDeletingService;
@@ -277,9 +277,7 @@ public ContainerBackgroundTaskResult deleteViaSchema2(
     Table<Long, DeletedBlocksTransaction> deleteTxns =
         ((DeleteTransactionStore<Long>) meta.getStore())
             .getDeleteTransactionTable();
-    try (TableIterator<Long,
-        ? extends Table.KeyValue<Long, DeletedBlocksTransaction>>
-             iterator = deleteTxns.iterator()) {
+    try (Table.KeyValueIterator<Long, DeletedBlocksTransaction> iterator = deleteTxns.iterator(VALUE_ONLY)) {
       return deleteViaTransactionStore(
           iterator, meta,
           container, dataDir, startTime, schema2Deleter);
@@ -298,18 +296,16 @@ public ContainerBackgroundTaskResult deleteViaSchema3(
     Table<String, DeletedBlocksTransaction> deleteTxns =
         ((DeleteTransactionStore<String>) meta.getStore())
             .getDeleteTransactionTable();
-    try (TableIterator<String,
-        ? extends Table.KeyValue<String, DeletedBlocksTransaction>>
-             iterator = deleteTxns.iterator(containerData.containerPrefix())) {
+    try (Table.KeyValueIterator<String, DeletedBlocksTransaction> iterator
+        = deleteTxns.iterator(containerData.containerPrefix(), VALUE_ONLY)) {
       return deleteViaTransactionStore(
           iterator, meta,
           container, dataDir, startTime, schema3Deleter);
     }
   }
 
   private ContainerBackgroundTaskResult deleteViaTransactionStore(
-      TableIterator<?, ? extends Table.KeyValue<?, DeletedBlocksTransaction>>
-          iter, DBHandle meta, Container container, File dataDir,
+      Table.KeyValueIterator<?, DeletedBlocksTransaction> iter, DBHandle meta, Container container, File dataDir,
       long startTime, Deleter deleter) throws IOException {
     ContainerBackgroundTaskResult crr = new ContainerBackgroundTaskResult();
     if (!checkDataDir(dataDir)) {

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractDatanodeStore.java

@@ -30,7 +30,6 @@
 import org.apache.hadoop.hdds.utils.db.DBStore;
 import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
 import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.hdds.utils.db.TableIterator;
 import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions;
 import org.apache.hadoop.ozone.container.common.helpers.BlockData;
 import org.apache.hadoop.ozone.container.common.helpers.ChunkInfoList;
@@ -208,9 +207,7 @@ public static class KeyValueBlockIterator implements
     private static final Logger LOG = LoggerFactory.getLogger(
             KeyValueBlockIterator.class);
 
-    private final TableIterator<String, ? extends Table.KeyValue<String,
-            BlockData>>
-            blockIterator;
+    private final Table.KeyValueIterator<String, BlockData> blockIterator;
     private static final KeyPrefixFilter DEFAULT_BLOCK_FILTER =
             MetadataKeyFilters.getUnprefixedKeyFilter();
     private final KeyPrefixFilter blockFilter;
@@ -221,9 +218,7 @@ public static class KeyValueBlockIterator implements
      * KeyValueBlockIterator to iterate unprefixed blocks in a container.
      * @param iterator - The underlying iterator to apply the block filter to.
      */
-    KeyValueBlockIterator(long containerID,
-            TableIterator<String, ? extends Table.KeyValue<String, BlockData>>
-                    iterator) {
+    KeyValueBlockIterator(long containerID, Table.KeyValueIterator<String, BlockData> iterator) {
       this.containerID = containerID;
       this.blockIterator = iterator;
       this.blockFilter = DEFAULT_BLOCK_FILTER;
@@ -235,8 +230,7 @@ public static class KeyValueBlockIterator implements
      * @param filter - Block filter, filter to be applied for blocks
      */
     KeyValueBlockIterator(long containerID,
-            TableIterator<String, ? extends Table.KeyValue<String, BlockData>>
-                    iterator, KeyPrefixFilter filter) {
+        Table.KeyValueIterator<String, BlockData> iterator, KeyPrefixFilter filter) {
       this.containerID = containerID;
       this.blockIterator = iterator;
       this.blockFilter = filter;
@@ -313,8 +307,7 @@ public static class KeyValueBlockLocalIdIterator implements
     private static final Logger LOG = LoggerFactory.getLogger(
         KeyValueBlockLocalIdIterator.class);
 
-    private final TableIterator<String, ? extends Table.KeyValue<String,
-        Long>> blockLocalIdIterator;
+    private final Table.KeyValueIterator<String, Long> blockLocalIdIterator;
     private final KeyPrefixFilter localIdFilter;
     private Long nextLocalId;
     private final long containerID;
@@ -325,8 +318,7 @@ public static class KeyValueBlockLocalIdIterator implements
      * @param filter - BlockLocalId filter to be applied for block localIds.
      */
     KeyValueBlockLocalIdIterator(long containerID,
-        TableIterator<String, ? extends Table.KeyValue<String, Long>>
-        iterator, KeyPrefixFilter filter) {
+        Table.KeyValueIterator<String, Long> iterator, KeyPrefixFilter filter) {
       this.containerID = containerID;
       this.blockLocalIdIterator = iterator;
       this.localIdFilter = filter;

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java

@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.ozone.container.ozoneimpl;
 
+import static org.apache.hadoop.hdds.utils.db.Table.KeyValueIterator.Type.KEY_ONLY;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_WORKERS;
@@ -65,7 +66,6 @@
 import org.apache.hadoop.hdds.utils.HddsServerUtil;
 import org.apache.hadoop.hdds.utils.IOUtils;
 import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.hdds.utils.db.TableIterator;
 import org.apache.hadoop.ozone.HddsDatanodeService;
 import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
@@ -346,8 +346,7 @@ public void buildContainerSet() throws IOException {
       for (Thread volumeThread : volumeThreads) {
         volumeThread.join();
       }
-      try (TableIterator<ContainerID, ? extends Table.KeyValue<ContainerID, String>> itr =
-               containerSet.getContainerIdsTable().iterator()) {
+      try (Table.KeyValueIterator<ContainerID, String> itr = containerSet.getContainerIdsTable().iterator(KEY_ONLY)) {
         final Map<ContainerID, Long> containerIds = new HashMap<>();
         while (itr.hasNext()) {
           containerIds.put(itr.next().getKey(), 0L);

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java

@@ -59,13 +59,13 @@
 import org.apache.hadoop.hdds.conf.ReconfigurationHandler;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.utils.BackgroundService;
 import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter;
 import org.apache.hadoop.hdds.utils.db.BatchOperation;
 import org.apache.hadoop.hdds.utils.db.CodecBuffer;
 import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.hdds.utils.db.TableIterator;
 import org.apache.hadoop.ozone.common.Checksum;
 import org.apache.hadoop.ozone.common.ChunkBuffer;
 import org.apache.hadoop.ozone.container.ContainerTestHelper;
@@ -371,9 +371,7 @@ private int getUnderDeletionBlocksCount(DBHandle meta,
       DatanodeStore ds = meta.getStore();
       DatanodeStoreSchemaTwoImpl dnStoreTwoImpl =
           (DatanodeStoreSchemaTwoImpl) ds;
-      try (
-          TableIterator<Long, ? extends Table.KeyValue<Long,
-              StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction>>
+      try (Table.KeyValueIterator<Long, DeletedBlocksTransaction>
               iter = dnStoreTwoImpl.getDeleteTransactionTable().iterator()) {
         while (iter.hasNext()) {
           StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction
@@ -387,9 +385,7 @@ private int getUnderDeletionBlocksCount(DBHandle meta,
       DatanodeStore ds = meta.getStore();
       DatanodeStoreSchemaThreeImpl dnStoreThreeImpl =
           (DatanodeStoreSchemaThreeImpl) ds;
-      try (
-          TableIterator<String, ? extends Table.KeyValue<String,
-              StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction>>
+      try (Table.KeyValueIterator<String, DeletedBlocksTransaction>
               iter = dnStoreThreeImpl.getDeleteTransactionTable()
               .iterator(data.containerPrefix())) {
         while (iter.hasNext()) {

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBTable.java

@@ -91,7 +91,7 @@ public void putWithBatch(BatchOperation batch, byte[] key, byte[] value) {
 
   @Override
   public boolean isEmpty() throws RocksDatabaseException {
-    try (KeyValueIterator<byte[], byte[]> keyIter = iterator((byte[]) null, KeyValueIterator.Type.NEITHER)) {
+    try (KeyValueIterator<byte[], byte[]> keyIter = iterator((byte[])null, KeyValueIterator.Type.NEITHER)) {
       keyIter.seekToFirst();
       return !keyIter.hasNext();
     }

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java

@@ -142,16 +142,22 @@ default VALUE getReadCopy(KEY key) throws RocksDatabaseException, CodecException
    */
   void deleteRange(KEY beginKey, KEY endKey) throws RocksDatabaseException, CodecException;
 
-  /** The same as iterator(null). */
+  /** The same as iterator(null, KEY_AND_VALUE). */
   default KeyValueIterator<KEY, VALUE> iterator() throws RocksDatabaseException, CodecException {
-    return iterator(null);
+    return iterator(null, KeyValueIterator.Type.KEY_AND_VALUE);
   }
 
   /** The same as iterator(prefix, KEY_AND_VALUE). */
   default KeyValueIterator<KEY, VALUE> iterator(KEY prefix) throws RocksDatabaseException, CodecException {
     return iterator(prefix, KeyValueIterator.Type.KEY_AND_VALUE);
   }
 
+  /** The same as iterator(null, type). */
+  default KeyValueIterator<KEY, VALUE> iterator(KeyValueIterator.Type type)
+      throws RocksDatabaseException, CodecException {
+    return iterator(null, type);
+  }
+
   /**
    * Iterate the elements in this table.
    *

hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestTypedTable.java

@@ -156,10 +156,10 @@ public void testContainerIDvsLong() throws Exception {
     // test iterator type
     try (TypedTable<Long, String> longTable = newTypedTable(
         1, LongCodec.get(), StringCodec.get());
-         Table.KeyValueIterator<Long, String> neither = longTable.iterator(null, NEITHER);
-         Table.KeyValueIterator<Long, String> keyOnly = longTable.iterator(null, KEY_ONLY);
-         Table.KeyValueIterator<Long, String> valueOnly = longTable.iterator(null, VALUE_ONLY);
-         Table.KeyValueIterator<Long, String> keyAndValue = longTable.iterator(null, KEY_AND_VALUE)) {
+         Table.KeyValueIterator<Long, String> neither = longTable.iterator(NEITHER);
+         Table.KeyValueIterator<Long, String> keyOnly = longTable.iterator(KEY_ONLY);
+         Table.KeyValueIterator<Long, String> valueOnly = longTable.iterator(VALUE_ONLY);
+         Table.KeyValueIterator<Long, String> keyAndValue = longTable.iterator(KEY_AND_VALUE)) {
       while (keyAndValue.hasNext()) {
         final Table.KeyValue<Long, String> keyValue = keyAndValue.next();
         final Long expectedKey = Objects.requireNonNull(keyValue.getKey());

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java

@@ -57,7 +57,6 @@
 import org.apache.hadoop.hdds.server.events.EventHandler;
 import org.apache.hadoop.hdds.server.events.EventPublisher;
 import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.hdds.utils.db.TableIterator;
 import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -137,8 +136,7 @@ public List<DeletedBlocksTransaction> getFailedTransactions(int count,
     lock.lock();
     try {
       final List<DeletedBlocksTransaction> failedTXs = Lists.newArrayList();
-      try (TableIterator<Long,
-          ? extends Table.KeyValue<Long, DeletedBlocksTransaction>> iter =
+      try (Table.KeyValueIterator<Long, DeletedBlocksTransaction> iter =
                deletedBlockLogStateManager.getReadOnlyIterator()) {
         if (count == LIST_ALL_FAILED_TRANSACTIONS) {
           while (iter.hasNext()) {
@@ -245,8 +243,7 @@ public int getNumOfValidTransactions() throws IOException {
     lock.lock();
     try {
       final AtomicInteger num = new AtomicInteger(0);
-      try (TableIterator<Long,
-          ? extends Table.KeyValue<Long, DeletedBlocksTransaction>> iter =
+      try (Table.KeyValueIterator<Long, DeletedBlocksTransaction> iter =
                deletedBlockLogStateManager.getReadOnlyIterator()) {
         while (iter.hasNext()) {
           DeletedBlocksTransaction delTX = iter.next().getValue();
@@ -394,8 +391,7 @@ public DatanodeDeletedBlockTransactions getTransactions(
           scmCommandTimeoutMs);
       DatanodeDeletedBlockTransactions transactions =
           new DatanodeDeletedBlockTransactions();
-      try (TableIterator<Long,
-          ? extends Table.KeyValue<Long, DeletedBlocksTransaction>> iter =
+      try (Table.KeyValueIterator<Long, DeletedBlocksTransaction> iter =
                deletedBlockLogStateManager.getReadOnlyIterator()) {
         if (lastProcessedTransactionId != -1) {
           iter.seek(lastProcessedTransactionId);

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManager.java

@@ -22,8 +22,6 @@
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
 import org.apache.hadoop.hdds.scm.metadata.Replicate;
 import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.hdds.utils.db.Table.KeyValue;
-import org.apache.hadoop.hdds.utils.db.TableIterator;
 
 /**
  * DeletedBlockLogStateManager interface to
@@ -46,8 +44,7 @@ void increaseRetryCountOfTransactionInDB(ArrayList<Long> txIDs)
   int resetRetryCountOfTransactionInDB(ArrayList<Long> txIDs)
       throws IOException;
 
-  TableIterator<Long,
-      KeyValue<Long, DeletedBlocksTransaction>> getReadOnlyIterator()
+  Table.KeyValueIterator<Long, DeletedBlocksTransaction> getReadOnlyIterator()
       throws IOException;
 
   void onFlush();

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManagerImpl.java

@@ -36,7 +36,6 @@
 import org.apache.hadoop.hdds.utils.db.CodecException;
 import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
 import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.hdds.utils.db.TableIterator;
 import org.apache.hadoop.hdds.utils.db.TypedTable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -68,14 +67,11 @@ public DeletedBlockLogStateManagerImpl(ConfigurationSource conf,
   }
 
   @Override
-  public TableIterator<Long, TypedTable.KeyValue<Long,
-      DeletedBlocksTransaction>> getReadOnlyIterator() throws IOException {
-    return new TableIterator<Long, TypedTable.KeyValue<Long,
-        DeletedBlocksTransaction>>() {
-
-      private TableIterator<Long,
-          ? extends Table.KeyValue<Long, DeletedBlocksTransaction>> iter =
-          deletedTable.iterator();
+  public Table.KeyValueIterator<Long, DeletedBlocksTransaction> getReadOnlyIterator()
+      throws IOException {
+    return new Table.KeyValueIterator<Long, DeletedBlocksTransaction>() {
+
+      private final Table.KeyValueIterator<Long, DeletedBlocksTransaction> iter = deletedTable.iterator();
       private TypedTable.KeyValue<Long, DeletedBlocksTransaction> nextTx;
 
       {

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerImpl.java

@@ -31,6 +31,7 @@
 import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.QUASI_CLOSED;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_LOCK_STRIPE_SIZE;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_LOCK_STRIPE_SIZE_DEFAULT;
+import static org.apache.hadoop.hdds.utils.db.Table.KeyValueIterator.Type.VALUE_ONLY;
 
 import com.google.common.util.concurrent.Striped;
 import java.io.IOException;
@@ -64,8 +65,6 @@
 import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
 import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException;
 import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.hdds.utils.db.Table.KeyValue;
-import org.apache.hadoop.hdds.utils.db.TableIterator;
 import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException;
 import org.apache.hadoop.ozone.common.statemachine.StateMachine;
 import org.apache.ratis.util.AutoCloseableLock;
@@ -234,9 +233,7 @@ private long getConfiguredContainerSize(final Configuration conf) {
    * @throws IOException in case of error while loading the containers
    */
   private void initialize() throws IOException {
-    try (TableIterator<ContainerID,
-        ? extends KeyValue<ContainerID, ContainerInfo>> iterator =
-             containerStore.iterator()) {
+    try (Table.KeyValueIterator<ContainerID, ContainerInfo> iterator = containerStore.iterator(VALUE_ONLY)) {
 
       while (iterator.hasNext()) {
         final ContainerInfo container = iterator.next().getValue();