HDDS-7715. Implement a lightweight listStatus API

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java

@@ -38,10 +38,12 @@
 import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
 import org.apache.hadoop.ozone.OzoneAcl;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
 import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
 import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
 import org.apache.hadoop.ozone.om.helpers.WithMetadata;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
@@ -1226,8 +1228,9 @@ protected List<OzoneKey> getNextShallowListOfKeys(String prevKey)
 
       // Elements in statuses must be sorted after startKey,
       // which means they come after the keyPrefix.
-      List<OzoneFileStatus> statuses = proxy.listStatus(volumeName, name,
-          delimiterKeyPrefix, false, startKey, listCacheSize);
+      List<OzoneFileStatusLight> statuses =
+          proxy.listStatusLight(volumeName, name, delimiterKeyPrefix, false,
+              startKey, listCacheSize, false);
 
       if (addedKeyPrefix) {
         // previous round already include the startKey, so remove it
@@ -1246,10 +1249,10 @@ protected List<OzoneKey> getNextShallowListOfKeys(String prevKey)
     }
 
     private List<OzoneKey> buildOzoneKeysFromFileStatus(
-        List<OzoneFileStatus> statuses) {
+        List<OzoneFileStatusLight> statuses) {
       return statuses.stream()
           .map(status -> {
-            OmKeyInfo keyInfo = status.getKeyInfo();
+            BasicOmKeyInfo keyInfo = status.getKeyInfo();
             String keyName = keyInfo.getKeyName();
             if (status.isDirectory()) {
               // add trailing slash to represent directory
@@ -1497,8 +1500,8 @@ private boolean getChildrenKeys(String keyPrefix, String startKey,
       startKey = startKey == null ? "" : startKey;
 
       // 1. Get immediate children of keyPrefix, starting with startKey
-      List<OzoneFileStatus> statuses = proxy.listStatus(volumeName, name,
-          keyPrefix, false, startKey, listCacheSize, true);
+      List<OzoneFileStatusLight> statuses = proxy.listStatusLight(volumeName,
+          name, keyPrefix, false, startKey, listCacheSize, true);
       boolean reachedLimitCacheSize = statuses.size() == listCacheSize;
 
       // 2. Special case: ListKey expects keyPrefix element should present in
@@ -1516,8 +1519,8 @@ private boolean getChildrenKeys(String keyPrefix, String startKey,
       // 4. Iterating over the resultStatuses list and add each key to the
       // resultList.
       for (int indx = 0; indx < statuses.size(); indx++) {
-        OzoneFileStatus status = statuses.get(indx);
-        OmKeyInfo keyInfo = status.getKeyInfo();
+        OzoneFileStatusLight status = statuses.get(indx);
+        BasicOmKeyInfo keyInfo = status.getKeyInfo();
         String keyName = keyInfo.getKeyName();
 
         OzoneKey ozoneKey;
@@ -1559,7 +1562,7 @@ private boolean getChildrenKeys(String keyPrefix, String startKey,
     }
 
     private void removeStartKeyIfExistsInStatusList(String startKey,
-        List<OzoneFileStatus> statuses) {
+        List<OzoneFileStatusLight> statuses) {
 
       if (!statuses.isEmpty()) {
         String firstElement = statuses.get(0).getKeyInfo().getKeyName();

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java

@@ -54,6 +54,7 @@
 import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
 import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
 import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
 import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
@@ -881,7 +882,6 @@ List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
       String keyName, boolean recursive, String startKey, long numEntries)
       throws IOException;
 
-
   /**
    * List the status for a file or a directory and its contents.
    *
@@ -901,6 +901,25 @@ List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
       String keyName, boolean recursive, String startKey,
       long numEntries, boolean allowPartialPrefixes) throws IOException;
 
+  /**
+   * Lightweight listStatus API.
+   *
+   * @param volumeName Volume name
+   * @param bucketName Bucket name
+   * @param keyName    Absolute path of the entry to be listed
+   * @param recursive  For a directory if true all the descendants of a
+   *                   particular directory are listed
+   * @param startKey   Key from which listing needs to start. If startKey exists
+   *                   its status is included in the final list.
+   * @param numEntries Number of entries to list from the start key
+   * @param allowPartialPrefixes if partial prefixes should be allowed,
+   *                             this is needed in context of ListKeys
+   * @return list of file status
+   */
+  List<OzoneFileStatusLight> listStatusLight(String volumeName,
+      String bucketName, String keyName, boolean recursive, String startKey,
+      long numEntries, boolean allowPartialPrefixes) throws IOException;
+
   /**
    * Add acl for Ozone object. Return true if acl is added successfully else
    * false.

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -131,6 +131,7 @@
 import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
 import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
 import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
 import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
@@ -2074,6 +2075,16 @@ public List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
             allowPartialPrefixes);
   }
 
+  @Override
+  public List<OzoneFileStatusLight> listStatusLight(String volumeName,
+      String bucketName, String keyName, boolean recursive, String startKey,
+      long numEntries, boolean allowPartialPrefixes) throws IOException {
+    OmKeyArgs keyArgs = prepareOmKeyArgs(volumeName, bucketName, keyName);
+    return ozoneManagerClient
+        .listStatusLight(keyArgs, recursive, startKey, numEntries,
+            allowPartialPrefixes);
+  }
+
   /**
    * Add acl for Ozone object. Return true if acl is added successfully else
    * false.

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java

@@ -248,6 +248,7 @@ public static boolean isReadOnly(
     case GetFileStatus:
     case LookupFile:
     case ListStatus:
+    case ListStatusLight:
     case GetAcl:
     case DBUpdates:
     case ListMultipartUploads:

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java

@@ -25,6 +25,7 @@
 import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult;
 import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
 import org.apache.hadoop.ozone.security.acl.OzoneObj;
 
 import java.io.IOException;
@@ -64,6 +65,24 @@ List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
                                    boolean allowPartialPrefixes)
       throws IOException;
 
+  /**
+   * Lightweight listStatus API.
+   *
+   * @param args    Key args
+   * @param recursive  For a directory if true all the descendants of a
+   *                   particular directory are listed
+   * @param startKey   Key from which listing needs to start. If startKey exists
+   *                   its status is included in the final list.
+   * @param numEntries Number of entries to list from the start key
+   * @param allowPartialPrefixes if partial prefixes should be allowed,
+   *                             this is needed in context of ListKeys
+   * @return list of file status
+   */
+  List<OzoneFileStatusLight> listStatusLight(OmKeyArgs args, boolean recursive,
+                                   String startKey, long numEntries,
+                                   boolean allowPartialPrefixes)
+      throws IOException;
+
   default List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
       String startKey, long numEntries)
       throws IOException {

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java

@@ -186,6 +186,30 @@ public static BasicOmKeyInfo getFromProtobuf(BasicKeyInfo basicKeyInfo,
     return builder.build();
   }
 
+  public static BasicOmKeyInfo getFromProtobuf(String volumeName,
+      String bucketName, BasicKeyInfo basicKeyInfo) throws IOException {
+    if (basicKeyInfo == null) {
+      return null;
+    }
+
+    String keyName = basicKeyInfo.getKeyName();
+
+    Builder builder = new Builder()
+        .setVolumeName(volumeName)
+        .setBucketName(bucketName)
+        .setKeyName(keyName)
+        .setDataSize(basicKeyInfo.getDataSize())
+        .setCreationTime(basicKeyInfo.getCreationTime())
+        .setModificationTime(basicKeyInfo.getModificationTime())
+        .setReplicationConfig(ReplicationConfig.fromProto(
+            basicKeyInfo.getType(),
+            basicKeyInfo.getFactor(),
+            basicKeyInfo.getEcReplicationConfig()))
+        .setIsFile(!keyName.endsWith("/"));
+
+    return builder.build();
+  }
+
   public boolean equals(Object o) {
     if (this == o) {
       return true;

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatusLight.java

@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneFileStatusProtoLight;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+
+/**
+ * Lightweight OzoneFileStatus class.
+ */
+public class OzoneFileStatusLight {
+
+  private BasicOmKeyInfo keyInfo;
+
+  private boolean isDirectory;
+
+  private long blockSize;
+
+  public OzoneFileStatusLight() {
+    isDirectory = true;
+  }
+
+  public OzoneFileStatusLight(BasicOmKeyInfo keyInfo,
+                              long blockSize, boolean isDirectory) {
+    this.keyInfo = keyInfo;
+    this.isDirectory = isDirectory;
+    this.blockSize = blockSize;
+  }
+
+  public BasicOmKeyInfo getKeyInfo() {
+    return keyInfo;
+  }
+
+  public long getBlockSize() {
+    return blockSize;
+  }
+
+  public String getTrimmedName() {
+    String keyName = keyInfo.getKeyName();
+    if (keyName.endsWith(OZONE_URI_DELIMITER)) {
+      return keyName.substring(0, keyName.length() - 1);
+    } else {
+      return keyName;
+    }
+  }
+
+  public String getPath() {
+    if (keyInfo == null) {
+      return OZONE_URI_DELIMITER;
+    } else {
+      String path = OZONE_URI_DELIMITER + keyInfo.getKeyName();
+      if (path.endsWith(OZONE_URI_DELIMITER)) {
+        return path.substring(0, path.length() - 1);
+      } else {
+        return path;
+      }
+    }
+  }
+
+  public boolean isDirectory() {
+    if (keyInfo == null) {
+      return true;
+    }
+    return isDirectory;
+  }
+
+  public boolean isFile() {
+    return !isDirectory();
+  }
+
+  public OzoneFileStatusProtoLight getProtobuf() {
+    OzoneFileStatusProtoLight.Builder builder =
+        OzoneFileStatusProtoLight.newBuilder()
+            .setBlockSize(blockSize)
+            .setIsDirectory(isDirectory);
+    //key info can be null for the fake root entry.
+    if (keyInfo != null) {
+      builder.setBasicKeyInfo(keyInfo.getProtobuf())
+          .setVolumeName(keyInfo.getVolumeName())
+          .setBucketName(keyInfo.getBucketName());
+    }
+    return builder.build();
+  }
+
+  public static OzoneFileStatusLight getFromProtobuf(
+      OzoneFileStatusProtoLight status) throws IOException {
+    return new OzoneFileStatusLight(
+        BasicOmKeyInfo.getFromProtobuf(status.getVolumeName(),
+            status.getBucketName(), status.getBasicKeyInfo()),
+        status.getBlockSize(),
+        status.getIsDirectory());
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (!(o instanceof OzoneFileStatusLight)) {
+      return false;
+    }
+    OzoneFileStatusLight that = (OzoneFileStatusLight) o;
+    return isDirectory == that.isDirectory &&
+        blockSize == that.blockSize &&
+        getTrimmedName().equals(that.getTrimmedName());
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(getTrimmedName());
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append(getClass().getSimpleName());
+    sb.append("{");
+    if (keyInfo == null) {
+      sb.append("<root>");
+    } else {
+      sb.append(getTrimmedName());
+      if (isDirectory) {
+        sb.append(" (dir)");
+      }
+    }
+    sb.append("}");
+    return sb.toString();
+  }
+
+  public static OzoneFileStatusLight fromOzoneFileStatus(
+      OzoneFileStatus ozoneFileStatus) {
+    BasicOmKeyInfo basicKeyInfo =
+        BasicOmKeyInfo.fromOmKeyInfo(ozoneFileStatus.getKeyInfo());
+    return new OzoneFileStatusLight(basicKeyInfo,
+        ozoneFileStatus.getBlockSize(), ozoneFileStatus.isDirectory());
+  }
+}

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java

@@ -50,6 +50,7 @@
 import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
 import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
 import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
 import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
@@ -912,6 +913,23 @@ List<OzoneFileStatus> listStatus(OmKeyArgs keyArgs, boolean recursive,
                                    boolean allowPartialPrefixes)
       throws IOException;
 
+  /**
+   * Lightweight listStatus API.
+   *
+   * @param keyArgs    Key args
+   * @param recursive  For a directory if true all the descendants of a
+   *                   particular directory are listed
+   * @param startKey   Key from which listing needs to start. If startKey exists
+   *                   its status is included in the final list.
+   * @param numEntries Number of entries to list from the start key
+   * @param allowPartialPrefixes if partial prefixes should be allowed,
+   *                             this is needed in context of ListKeys
+   * @return list of file status
+   */
+  List<OzoneFileStatusLight> listStatusLight(OmKeyArgs keyArgs,
+      boolean recursive, String startKey, long numEntries,
+      boolean allowPartialPrefixes) throws IOException;
+
   /**
    * Add acl for Ozone object. Return true if acl is added successfully else
    * false.