HDDS-7715. Implement a lightweight listStatus API

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java

@@ -38,10 +38,12 @@
 import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
 import org.apache.hadoop.ozone.OzoneAcl;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
 import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
 import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
 import org.apache.hadoop.ozone.om.helpers.WithMetadata;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
@@ -1226,8 +1228,9 @@ protected List<OzoneKey> getNextShallowListOfKeys(String prevKey)
 
       // Elements in statuses must be sorted after startKey,
       // which means they come after the keyPrefix.
-      List<OzoneFileStatus> statuses = proxy.listStatus(volumeName, name,
-          delimiterKeyPrefix, false, startKey, listCacheSize);
+      List<OzoneFileStatusLight> statuses =
+          proxy.listStatusLight(volumeName, name, delimiterKeyPrefix, false,
+              startKey, listCacheSize);
 
       if (addedKeyPrefix) {
         // previous round already include the startKey, so remove it
@@ -1246,10 +1249,10 @@ protected List<OzoneKey> getNextShallowListOfKeys(String prevKey)
     }
 
     private List<OzoneKey> buildOzoneKeysFromFileStatus(
-        List<OzoneFileStatus> statuses) {
+        List<OzoneFileStatusLight> statuses) {
       return statuses.stream()
           .map(status -> {
-            OmKeyInfo keyInfo = status.getKeyInfo();
+            BasicOmKeyInfo keyInfo = status.getKeyInfo();
             String keyName = keyInfo.getKeyName();
             if (status.isDirectory()) {
               // add trailing slash to represent directory
@@ -1497,8 +1500,8 @@ private boolean getChildrenKeys(String keyPrefix, String startKey,
       startKey = startKey == null ? "" : startKey;
 
       // 1. Get immediate children of keyPrefix, starting with startKey
-      List<OzoneFileStatus> statuses = proxy.listStatus(volumeName, name,
-          keyPrefix, false, startKey, listCacheSize, true);
+      List<OzoneFileStatusLight> statuses = proxy.listStatusLight(volumeName,
+          name, keyPrefix, false, startKey, listCacheSize, true);
       boolean reachedLimitCacheSize = statuses.size() == listCacheSize;
 
       // 2. Special case: ListKey expects keyPrefix element should present in
@@ -1516,8 +1519,8 @@ private boolean getChildrenKeys(String keyPrefix, String startKey,
       // 4. Iterating over the resultStatuses list and add each key to the
       // resultList.
       for (int indx = 0; indx < statuses.size(); indx++) {
-        OzoneFileStatus status = statuses.get(indx);
-        OmKeyInfo keyInfo = status.getKeyInfo();
+        OzoneFileStatusLight status = statuses.get(indx);
+        BasicOmKeyInfo keyInfo = status.getKeyInfo();
         String keyName = keyInfo.getKeyName();
 
         OzoneKey ozoneKey;
@@ -1559,7 +1562,7 @@ private boolean getChildrenKeys(String keyPrefix, String startKey,
     }
 
     private void removeStartKeyIfExistsInStatusList(String startKey,
-        List<OzoneFileStatus> statuses) {
+        List<OzoneFileStatusLight> statuses) {
 
       if (!statuses.isEmpty()) {
         String firstElement = statuses.get(0).getKeyInfo().getKeyName();

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java

@@ -54,6 +54,7 @@
 import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
 import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
 import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
 import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
@@ -881,6 +882,23 @@ List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
       String keyName, boolean recursive, String startKey, long numEntries)
       throws IOException;
 
+  /**
+   * Lightweight listStatus API.
+   *
+   * @param volumeName Volume name
+   * @param bucketName Bucket name
+   * @param keyName    Absolute path of the entry to be listed
+   * @param recursive  For a directory if true all the descendants of a
+   *                   particular directory are listed
+   * @param startKey   Key from which listing needs to start. If startKey exists
+   *                   its status is included in the final list.
+   * @param numEntries Number of entries to list from the start key
+   * @return list of file status
+   */
+  List<OzoneFileStatusLight> listStatusLight(String volumeName,
+      String bucketName, String keyName, boolean recursive, String startKey,
+      long numEntries) throws IOException;
+
 
   /**
    * List the status for a file or a directory and its contents.
@@ -901,6 +919,25 @@ List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
       String keyName, boolean recursive, String startKey,
       long numEntries, boolean allowPartialPrefixes) throws IOException;
 
+  /**
+   * Lightweight listStatus API.
+   *
+   * @param volumeName Volume name
+   * @param bucketName Bucket name
+   * @param keyName    Absolute path of the entry to be listed
+   * @param recursive  For a directory if true all the descendants of a
+   *                   particular directory are listed
+   * @param startKey   Key from which listing needs to start. If startKey exists
+   *                   its status is included in the final list.
+   * @param numEntries Number of entries to list from the start key
+   * @param allowPartialPrefixes if partial prefixes should be allowed,
+   *                             this is needed in context of ListKeys
+   * @return list of file status
+   */
+  List<OzoneFileStatusLight> listStatusLight(String volumeName,
+      String bucketName, String keyName, boolean recursive, String startKey,
+      long numEntries, boolean allowPartialPrefixes) throws IOException;
+
   /**
    * Add acl for Ozone object. Return true if acl is added successfully else
    * false.

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -131,6 +131,7 @@
 import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
 import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
 import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
 import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
@@ -2064,6 +2065,15 @@ public List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
         .listStatus(keyArgs, recursive, startKey, numEntries);
   }
 
+  @Override
+  public List<OzoneFileStatusLight> listStatusLight(String volumeName,
+      String bucketName, String keyName, boolean recursive, String startKey,
+      long numEntries) throws IOException {
+    OmKeyArgs keyArgs = prepareOmKeyArgs(volumeName, bucketName, keyName);
+    return ozoneManagerClient
+        .listStatusLight(keyArgs, recursive, startKey, numEntries);
+  }
+
   @Override
   public List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
       String keyName, boolean recursive, String startKey,
@@ -2074,6 +2084,16 @@ public List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
             allowPartialPrefixes);
   }
 
+  @Override
+  public List<OzoneFileStatusLight> listStatusLight(String volumeName,
+      String bucketName, String keyName, boolean recursive, String startKey,
+      long numEntries, boolean allowPartialPrefixes) throws IOException {
+    OmKeyArgs keyArgs = prepareOmKeyArgs(volumeName, bucketName, keyName);
+    return ozoneManagerClient
+        .listStatusLight(keyArgs, recursive, startKey, numEntries,
+            allowPartialPrefixes);
+  }
+
   /**
    * Add acl for Ozone object. Return true if acl is added successfully else
    * false.

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java

@@ -248,6 +248,7 @@ public static boolean isReadOnly(
     case GetFileStatus:
     case LookupFile:
     case ListStatus:
+    case ListStatusLight:
     case GetAcl:
     case DBUpdates:
     case ListMultipartUploads:

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java

@@ -25,6 +25,7 @@
 import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult;
 import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
 import org.apache.hadoop.ozone.security.acl.OzoneObj;
 
 import java.io.IOException;
@@ -64,12 +65,38 @@ List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
                                    boolean allowPartialPrefixes)
       throws IOException;
 
+  /**
+   * Lightweight listStatus API.
+   *
+   * @param args    Key args
+   * @param recursive  For a directory if true all the descendants of a
+   *                   particular directory are listed
+   * @param startKey   Key from which listing needs to start. If startKey exists
+   *                   its status is included in the final list.
+   * @param numEntries Number of entries to list from the start key
+   * @param allowPartialPrefixes if partial prefixes should be allowed,
+   *                             this is needed in context of ListKeys
+   * @return list of file status
+   */
+  List<OzoneFileStatusLight> listStatusLight(OmKeyArgs args, boolean recursive,
+                                   String startKey, long numEntries,
+                                   boolean allowPartialPrefixes)
+      throws IOException;
+
   default List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
       String startKey, long numEntries)
       throws IOException {
     return listStatus(args, recursive, startKey, numEntries, false);
   }
 
+  default List<OzoneFileStatusLight> listStatusLight(OmKeyArgs args,
+                                                     boolean recursive,
+                                                     String startKey,
+                                                     long numEntries)
+      throws IOException {
+    return listStatusLight(args, recursive, startKey, numEntries, false);
+  }
+
   /**
    * OzoneFS api to get file status for an entry.
    *

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java

@@ -186,6 +186,30 @@ public static BasicOmKeyInfo getFromProtobuf(BasicKeyInfo basicKeyInfo,
     return builder.build();
   }
 
+  public static BasicOmKeyInfo getFromProtobuf(String volumeName,
+      String bucketName, BasicKeyInfo basicKeyInfo) throws IOException {
+    if (basicKeyInfo == null) {
+      return null;
+    }
+
+    String keyName = basicKeyInfo.getKeyName();
+
+    Builder builder = new Builder()
+        .setVolumeName(volumeName)
+        .setBucketName(bucketName)
+        .setKeyName(keyName)
+        .setDataSize(basicKeyInfo.getDataSize())
+        .setCreationTime(basicKeyInfo.getCreationTime())
+        .setModificationTime(basicKeyInfo.getModificationTime())
+        .setReplicationConfig(ReplicationConfig.fromProto(
+            basicKeyInfo.getType(),
+            basicKeyInfo.getFactor(),
+            basicKeyInfo.getEcReplicationConfig()))
+        .setIsFile(!keyName.endsWith("/"));
+
+    return builder.build();
+  }
+
   public boolean equals(Object o) {
     if (this == o) {
       return true;