HDDS-9079. Implement a lightweight listKeys API

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java

@@ -37,6 +37,9 @@ public enum OzoneManagerVersion implements ComponentVersion {
   OPTIMIZED_GET_KEY_INFO(3, "OzoneManager version that supports optimized"
       + " key lookups using cached container locations."),
 
+  LIGHTWEIGHT_LIST_KEYS(4, "OzoneManager version that supports lightweight"
+      + " listKeys API."),
+
   FUTURE_VERSION(-1, "Used internally in the client when the server side is "
       + " newer and an unknown server version has arrived to the client.");
 

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -107,6 +107,7 @@
 import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
 import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.DeleteTenantState;
@@ -1558,18 +1559,35 @@ public List<OzoneKey> listKeys(String volumeName, String bucketName,
                                  String keyPrefix, String prevKey,
                                  int maxListResult)
       throws IOException {
-    List<OmKeyInfo> keys = ozoneManagerClient.listKeys(
-        volumeName, bucketName, prevKey, keyPrefix, maxListResult);
-    return keys.stream().map(key -> new OzoneKey(
-        key.getVolumeName(),
-        key.getBucketName(),
-        key.getKeyName(),
-        key.getDataSize(),
-        key.getCreationTime(),
-        key.getModificationTime(),
-        key.getReplicationConfig(),
-        key.isFile()))
-        .collect(Collectors.toList());
+
+    if (omVersion.compareTo(OzoneManagerVersion.LIGHTWEIGHT_LIST_KEYS) >= 0) {
+      List<BasicOmKeyInfo> keys = ozoneManagerClient.listKeysLight(
+          volumeName, bucketName, prevKey, keyPrefix, maxListResult).getKeys();
+
+      return keys.stream().map(key -> new OzoneKey(
+              key.getVolumeName(),
+              key.getBucketName(),
+              key.getKeyName(),
+              key.getDataSize(),
+              key.getCreationTime(),
+              key.getModificationTime(),
+              key.getReplicationConfig(),
+              key.isFile()))
+          .collect(Collectors.toList());
+    } else {
+      List<OmKeyInfo> keys = ozoneManagerClient.listKeys(
+          volumeName, bucketName, prevKey, keyPrefix, maxListResult).getKeys();
+      return keys.stream().map(key -> new OzoneKey(
+              key.getVolumeName(),
+              key.getBucketName(),
+              key.getKeyName(),
+              key.getDataSize(),
+              key.getCreationTime(),
+              key.getModificationTime(),
+              key.getReplicationConfig(),
+              key.isFile()))
+          .collect(Collectors.toList());
+    }
   }
 
   @Override

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java

@@ -241,6 +241,7 @@ public static boolean isReadOnly(
     case ListBuckets:
     case LookupKey:
     case ListKeys:
+    case ListKeysLight:
     case ListTrash:
     case ServiceList:
     case ListMultiPartUploadParts:

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java

@@ -22,6 +22,8 @@
 import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult;
+import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
 import org.apache.hadoop.ozone.security.acl.OzoneObj;
 
@@ -109,8 +111,32 @@ default List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
    *   the size of the result will not exceed this limit.
    * @return a list of keys.
    */
-  List<OmKeyInfo> listKeys(String volumeName, String bucketName,
-                           String startKey, String keyPrefix, int maxKeys)
+  ListKeysResult listKeys(String volumeName, String bucketName,
+                          String startKey, String keyPrefix, int maxKeys)
+      throws IOException;
+
+  /**
+   * Lightweight listKeys implementation.
+   *
+   * @param volumeName
+   *   the name of the volume.
+   * @param bucketName
+   *   the name of the bucket.
+   * @param startKey
+   *   the start key name, only the keys whose name is
+   *   after this value will be included in the result.
+   * @param keyPrefix
+   *   key name prefix, only the keys whose name has
+   *   this prefix will be included in the result.
+   * @param maxKeys
+   *   the maximum number of keys to return. It ensures
+   *   the size of the result will not exceed this limit.
+   * @return a list of keys.
+   * @throws IOException
+   */
+  ListKeysLightResult listKeysLight(String volumeName, String bucketName,
+                                     String startKey, String keyPrefix,
+                                     int maxKeys)
       throws IOException;
 
   /**

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java

@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.helpers;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BasicKeyInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysRequest;
+
+/**
+ * Lightweight OmKeyInfo class.
+ */
+public class BasicOmKeyInfo {
+
+  private String volumeName;
+  private String bucketName;
+  private String keyName;
+  private long dataSize;
+  private long creationTime;
+  private long modificationTime;
+  private ReplicationConfig replicationConfig;
+  private boolean isFile;
+
+  @SuppressWarnings("parameternumber")
+  public BasicOmKeyInfo(String volumeName, String bucketName, String keyName,
+                        long dataSize, long creationTime, long modificationTime,
+                        ReplicationConfig replicationConfig, boolean isFile) {
+    this.volumeName = volumeName;
+    this.bucketName = bucketName;
+    this.keyName = keyName;
+    this.dataSize = dataSize;
+    this.creationTime = creationTime;
+    this.modificationTime = modificationTime;
+    this.replicationConfig = replicationConfig;
+    this.isFile = isFile;
+  }
+
+  public String getVolumeName() {
+    return volumeName;
+  }
+
+  public String getBucketName() {
+    return bucketName;
+  }
+
+  public String getKeyName() {
+    return keyName;
+  }
+
+  public long getDataSize() {
+    return dataSize;
+  }
+
+  public long getCreationTime() {
+    return creationTime;
+  }
+
+  public long getModificationTime() {
+    return modificationTime;
+  }
+
+  public ReplicationConfig getReplicationConfig() {
+    return replicationConfig;
+  }
+
+  public boolean isFile() {
+    return isFile;
+  }
+
+  /**
+   * Builder of BasicOmKeyInfo.
+   */
+  public static class Builder {
+    private String volumeName;
+    private String bucketName;
+    private String keyName;
+    private long dataSize;
+    private long creationTime;
+    private long modificationTime;
+    private ReplicationConfig replicationConfig;
+    private boolean isFile;
+
+    public Builder setVolumeName(String volumeName) {
+      this.volumeName = volumeName;
+      return this;
+    }
+
+    public Builder setBucketName(String bucketName) {
+      this.bucketName = bucketName;
+      return this;
+    }
+
+    public Builder setKeyName(String keyName) {
+      this.keyName = keyName;
+      return this;
+    }
+
+    public Builder setDataSize(long dataSize) {
+      this.dataSize = dataSize;
+      return this;
+    }
+
+    public Builder setCreationTime(long creationTime) {
+      this.creationTime = creationTime;
+      return this;
+    }
+
+    public Builder setModificationTime(long modificationTime) {
+      this.modificationTime = modificationTime;
+      return this;
+    }
+
+    public Builder setReplicationConfig(ReplicationConfig replicationConfig) {
+      this.replicationConfig = replicationConfig;
+      return this;
+    }
+
+    public Builder setIsFile(boolean isFile) {
+      this.isFile = isFile;
+      return this;
+    }
+
+    public BasicOmKeyInfo build() {
+      return new BasicOmKeyInfo(volumeName, bucketName, keyName, dataSize,
+          creationTime, modificationTime, replicationConfig, isFile);
+    }
+  }
+
+  public BasicKeyInfo getProtobuf() {
+    BasicKeyInfo.Builder builder = BasicKeyInfo.newBuilder()
+        .setKeyName(keyName)
+        .setDataSize(dataSize)
+        .setCreationTime(creationTime)
+        .setModificationTime(modificationTime)
+        .setType(replicationConfig.getReplicationType());
+    if (replicationConfig instanceof ECReplicationConfig) {
+      builder.setEcReplicationConfig(
+          ((ECReplicationConfig) replicationConfig).toProto());
+    } else {
+      builder.setFactor(ReplicationConfig.getLegacyFactor(replicationConfig));
+    }
+
+    return builder.build();
+  }
+
+  public static BasicOmKeyInfo getFromProtobuf(BasicKeyInfo basicKeyInfo,
+                                               ListKeysRequest request)
+      throws IOException {
+    if (basicKeyInfo == null || request == null) {
+      return null;
+    }
+
+    String keyName = basicKeyInfo.getKeyName();
+
+    Builder builder = new Builder()
+        .setVolumeName(request.getVolumeName())
+        .setBucketName(request.getBucketName())
+        .setKeyName(keyName)
+        .setDataSize(basicKeyInfo.getDataSize())
+        .setCreationTime(basicKeyInfo.getCreationTime())
+        .setModificationTime(basicKeyInfo.getModificationTime())
+        .setReplicationConfig(ReplicationConfig.fromProto(
+            basicKeyInfo.getType(),
+            basicKeyInfo.getFactor(),
+            basicKeyInfo.getEcReplicationConfig()))
+        .setIsFile(!keyName.endsWith("/"));
+
+    return builder.build();
+  }
+
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    BasicOmKeyInfo basicOmKeyInfo = (BasicOmKeyInfo) o;
+    return volumeName.equals(basicOmKeyInfo.volumeName) &&
+        bucketName.equals(basicOmKeyInfo.bucketName) &&
+        keyName.equals(basicOmKeyInfo.keyName) &&
+        dataSize == basicOmKeyInfo.dataSize &&
+        creationTime == basicOmKeyInfo.creationTime &&
+        modificationTime == basicOmKeyInfo.modificationTime &&
+        replicationConfig.equals(basicOmKeyInfo.replicationConfig) &&
+        isFile == basicOmKeyInfo.isFile;
+  }
+
+  public int hashCode() {
+    return Objects.hash(volumeName, bucketName, keyName);
+  }
+
+  public static BasicOmKeyInfo fromOmKeyInfo(OmKeyInfo omKeyInfo) {
+    return new BasicOmKeyInfo(
+        omKeyInfo.getVolumeName(),
+        omKeyInfo.getBucketName(),
+        omKeyInfo.getKeyName(),
+        omKeyInfo.getDataSize(),
+        omKeyInfo.getCreationTime(),
+        omKeyInfo.getModificationTime(),
+        omKeyInfo.getReplicationConfig(),
+        omKeyInfo.isFile());
+  }
+}

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ListKeysLightResult.java

@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.helpers;
+
+import java.util.List;
+
+/**
+ * Encapsulates the result of listKeys. It contains a list of
+ * {@link BasicOmKeyInfo} and a boolean flag indicating if the listing is
+ * truncated.
+ */
+public class ListKeysLightResult {
+  private List<BasicOmKeyInfo> keys;
+  private boolean isTruncated;
+
+  public ListKeysLightResult(List<BasicOmKeyInfo> keys, boolean isTruncated) {
+    this.keys = keys;
+    this.isTruncated = isTruncated;
+  }
+
+  public List<BasicOmKeyInfo> getKeys() {
+    return keys;
+  }
+
+  public boolean isTruncated() {
+    return isTruncated;
+  }
+}