HDDS-11414. Key listing for FSO buckets fails with forward client

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java

@@ -44,6 +44,8 @@ public enum OzoneManagerVersion implements ComponentVersion {
 
   ATOMIC_REWRITE_KEY(6, "OzoneManager version that supports rewriting key as atomic operation"),
   HBASE_SUPPORT(7, "OzoneManager version that supports HBase integration"),
+  LIGHTWEIGHT_LIST_STATUS(8, "OzoneManager version that supports lightweight"
+      + " listStatus API."),
 
   FUTURE_VERSION(-1, "Used internally in the client when the server side is "
       + " newer and an unknown server version has arrived to the client.");

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java

@@ -38,13 +38,11 @@
 import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
 import org.apache.hadoop.ozone.OzoneAcl;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
-import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.ErrorInfo;
 import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
 import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
-import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
 import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
 import org.apache.hadoop.ozone.om.helpers.WithMetadata;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
@@ -1357,10 +1355,9 @@ List<OzoneKey> getNextShallowListOfKeys(String prevKey)
         }
       }
 
-      // 2. Get immediate children by listStatusLight method
-      List<OzoneFileStatusLight> statuses =
-          proxy.listStatusLight(volumeName, name, delimiterKeyPrefix, false,
-              startKey, listCacheSize, false);
+      // 2. Get immediate children by listStatus method
+      List<OzoneFileStatus> statuses = proxy.listStatus(volumeName, name,
+          delimiterKeyPrefix, false, startKey, listCacheSize, false);
 
       if (addedKeyPrefix && statuses.size() > 0) {
         // previous round already include the startKey, so remove it
@@ -1383,7 +1380,7 @@ protected void initDelimiterKeyPrefix() {
     }
 
     protected List<OzoneKey> buildKeysWithKeyPrefix(
-        List<OzoneFileStatusLight> statuses) {
+        List<OzoneFileStatus> statuses) {
       return statuses.stream()
           .map(OzoneBucket::toOzoneKey)
           .filter(key -> StringUtils.startsWith(key.getName(), getKeyPrefix()))
@@ -1392,8 +1389,8 @@ protected List<OzoneKey> buildKeysWithKeyPrefix(
 
   }
 
-  private static OzoneKey toOzoneKey(OzoneFileStatusLight status) {
-    BasicOmKeyInfo keyInfo = status.getKeyInfo();
+  private static OzoneKey toOzoneKey(OzoneFileStatus status) {
+    OmKeyInfo keyInfo = status.getKeyInfo();
     String keyName = keyInfo.getKeyName();
     final Map<String, String> metadata;
     if (status.isDirectory()) {
@@ -1600,8 +1597,8 @@ List<OzoneKey> getNextShallowListOfKeys(String prevKey)
       }
 
       // 2. Get immediate children by listStatus method.
-      List<OzoneFileStatusLight> statuses =
-          proxy.listStatusLight(volumeName, name, getDelimiterKeyPrefix(),
+      List<OzoneFileStatus> statuses =
+          proxy.listStatus(volumeName, name, getDelimiterKeyPrefix(),
               false, startKey, listCacheSize, false);
 
       if (!statuses.isEmpty()) {
@@ -1760,7 +1757,7 @@ private boolean getChildrenKeys(String keyPrefix, String startKey,
       startKey = startKey == null ? "" : startKey;
 
       // 1. Get immediate children of keyPrefix, starting with startKey
-      List<OzoneFileStatusLight> statuses = proxy.listStatusLight(volumeName,
+      List<OzoneFileStatus> statuses = proxy.listStatus(volumeName,
           name, keyPrefix, false, startKey, listCacheSize, true);
       boolean reachedLimitCacheSize = statuses.size() == listCacheSize;
 
@@ -1779,8 +1776,8 @@ private boolean getChildrenKeys(String keyPrefix, String startKey,
       // 4. Iterating over the resultStatuses list and add each key to the
       // resultList.
       for (int indx = 0; indx < statuses.size(); indx++) {
-        OzoneFileStatusLight status = statuses.get(indx);
-        BasicOmKeyInfo keyInfo = status.getKeyInfo();
+        OzoneFileStatus status = statuses.get(indx);
+        OmKeyInfo keyInfo = status.getKeyInfo();
         OzoneKey ozoneKey = toOzoneKey(status);
         keysResultList.add(ozoneKey);
 
@@ -1808,7 +1805,7 @@ private boolean getChildrenKeys(String keyPrefix, String startKey,
     }
 
     private void removeStartKeyIfExistsInStatusList(String startKey,
-        List<OzoneFileStatusLight> statuses) {
+        List<OzoneFileStatus> statuses) {
 
       if (!statuses.isEmpty()) {
         String firstElement = statuses.get(0).getKeyInfo().getKeyName();

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java

@@ -58,7 +58,6 @@
 import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
 import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
-import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
 import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
 import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
 import org.apache.hadoop.ozone.om.helpers.TenantStateList;
@@ -989,25 +988,6 @@ List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
       String keyName, boolean recursive, String startKey,
       long numEntries, boolean allowPartialPrefixes) throws IOException;
 
-  /**
-   * Lightweight listStatus API.
-   *
-   * @param volumeName Volume name
-   * @param bucketName Bucket name
-   * @param keyName    Absolute path of the entry to be listed
-   * @param recursive  For a directory if true all the descendants of a
-   *                   particular directory are listed
-   * @param startKey   Key from which listing needs to start. If startKey exists
-   *                   its status is included in the final list.
-   * @param numEntries Number of entries to list from the start key
-   * @param allowPartialPrefixes if partial prefixes should be allowed,
-   *                             this is needed in context of ListKeys
-   * @return list of file status
-   */
-  List<OzoneFileStatusLight> listStatusLight(String volumeName,
-      String bucketName, String keyName, boolean recursive, String startKey,
-      long numEntries, boolean allowPartialPrefixes) throws IOException;
-
   /**
    * Add acl for Ozone object. Return true if acl is added successfully else
    * false.

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -122,7 +122,6 @@
 import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
 import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
-import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
 import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
 import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
 import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
@@ -2305,19 +2304,16 @@ public List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
       String keyName, boolean recursive, String startKey,
       long numEntries, boolean allowPartialPrefixes) throws IOException {
     OmKeyArgs keyArgs = prepareOmKeyArgs(volumeName, bucketName, keyName);
-    return ozoneManagerClient
-        .listStatus(keyArgs, recursive, startKey, numEntries,
-            allowPartialPrefixes);
-  }
-
-  @Override
-  public List<OzoneFileStatusLight> listStatusLight(String volumeName,
-      String bucketName, String keyName, boolean recursive, String startKey,
-      long numEntries, boolean allowPartialPrefixes) throws IOException {
-    OmKeyArgs keyArgs = prepareOmKeyArgs(volumeName, bucketName, keyName);
-    return ozoneManagerClient
-        .listStatusLight(keyArgs, recursive, startKey, numEntries,
-            allowPartialPrefixes);
+    if (omVersion.compareTo(OzoneManagerVersion.LIGHTWEIGHT_LIST_STATUS) >= 0) {
+      return ozoneManagerClient.listStatusLight(keyArgs, recursive, startKey,
+              numEntries, allowPartialPrefixes)
+          .stream()
+          .map(OzoneFileStatus::new)
+          .collect(Collectors.toList());
+    } else {
+      return ozoneManagerClient.listStatus(keyArgs, recursive, startKey,
+          numEntries, allowPartialPrefixes);
+    }
   }
 
   /**

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java

@@ -88,6 +88,7 @@ public static Codec<OmKeyInfo> getCodec(boolean ignorePipeline) {
    * Support OFS use-case to identify if the key is a file or a directory.
    */
   private boolean isFile;
+  private final String eTag;
 
   /**
    * Represents leaf node name. This also will be used when the keyName is
@@ -133,6 +134,22 @@ private OmKeyInfo(Builder b) {
     this.ownerName = b.ownerName;
     this.tags = b.tags;
     this.expectedDataGeneration = b.expectedDataGeneration;
+    this.eTag = b.eTag;
+  }
+
+  public OmKeyInfo(BasicOmKeyInfo b) {
+    this.volumeName = b.getVolumeName();
+    this.bucketName = b.getBucketName();
+    this.keyName = b.getKeyName();
+    this.dataSize = b.getDataSize();
+    this.creationTime = b.getCreationTime();
+    this.modificationTime = b.getModificationTime();
+    this.replicationConfig = b.getReplicationConfig();
+    this.isFile = b.isFile();
+    this.eTag = b.getETag();
+    this.ownerName = b.getOwnerName();
+    this.fileChecksum = null;
+    this.acls = null;
   }
 
   public String getVolumeName() {
@@ -163,6 +180,10 @@ public long getReplicatedSize() {
     return QuotaUtil.getReplicatedSize(getDataSize(), replicationConfig);
   }
 
+  public String getETag() {
+    return eTag;
+  }
+
   public void setDataSize(long size) {
     this.dataSize = size;
   }
@@ -451,6 +472,7 @@ public String toString() {
         ", fileChecksum=" + fileChecksum +
         ", isFile=" + isFile +
         ", fileName='" + fileName + '\'' +
+        ", eTag=" + eTag + '\'' +
         ", acls=" + acls +
         '}';
   }
@@ -476,6 +498,7 @@ public static class Builder extends WithParentObjectId.Builder {
     private FileChecksum fileChecksum;
 
     private boolean isFile;
+    private String eTag;
     private final Map<String, String> tags = new HashMap<>();
     private Long expectedDataGeneration = null;
 
@@ -606,6 +629,11 @@ public Builder setFile(boolean isAFile) {
       return this;
     }
 
+    public Builder setETag(String etag) {
+      this.eTag = etag;
+      return this;
+    }
+
     public Builder addTag(String key, String value) {
       tags.put(key, value);
       return this;
@@ -702,6 +730,9 @@ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName,
     } else {
       kb.setFactor(ReplicationConfig.getLegacyFactor(replicationConfig));
     }
+    if (eTag != null) {
+      kb.setETag(eTag);
+    }
     kb.setLatestVersion(latestVersion)
         .addAllKeyLocationList(keyLocations)
         .setCreationTime(creationTime)
@@ -779,6 +810,9 @@ public static OmKeyInfo getFromProtobuf(KeyInfo keyInfo) throws IOException {
     if (keyInfo.hasIsFile()) {
       builder.setFile(keyInfo.getIsFile());
     }
+    if (keyInfo.hasETag()) {
+      builder.setETag(keyInfo.getETag());
+    }
     if (keyInfo.hasExpectedDataGeneration()) {
       builder.setExpectedDataGeneration(keyInfo.getExpectedDataGeneration());
     }
@@ -903,6 +937,9 @@ public OmKeyInfo copyObject() {
     if (expectedDataGeneration != null) {
       builder.setExpectedDataGeneration(expectedDataGeneration);
     }
+    if (eTag != null) {
+      builder.setETag(eTag);
+    }
 
     return builder.build();
   }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatus.java

@@ -53,6 +53,12 @@ public OzoneFileStatus(OmKeyInfo keyInfo,
     this.blockSize = blockSize;
   }
 
+  public OzoneFileStatus(OzoneFileStatusLight b) {
+    this.keyInfo = new OmKeyInfo(b.getKeyInfo());
+    this.isDirectory = b.isDirectory();
+    this.blockSize = b.getBlockSize();
+  }
+
   public OmKeyInfo getKeyInfo() {
     return keyInfo;
   }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java

@@ -2459,7 +2459,7 @@ public void testKeyDeleteLegacyWithEnableFileSystemPath() throws IOException {
     // Check number of keys
     OzoneVolume volume = client.getObjectStore().getVolume(volumeName);
     OzoneBucket bucket = volume.getBucket(bucketName);
-    List<OzoneFileStatus> files = bucket.listStatus("", true, "", 5);
+    List<OzoneFileStatus> files = bucket.listStatus("", true, "", 5, false);
     // Two keys should still exist, dirPath and keyPath
     assertEquals(2, files.size());
 

hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto

@@ -1152,6 +1152,7 @@ message KeyInfo {
   // This allows a key to be created an committed atomically if the original has not
   // been modified.
     optional uint64 expectedDataGeneration = 22;
+    optional string eTag = 23;
 }
 
 message BasicKeyInfo {

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java

@@ -42,7 +42,6 @@
 import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
 import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
-import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
 import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
 import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
 import org.apache.hadoop.ozone.om.helpers.TenantStateList;
@@ -564,13 +563,6 @@ public List<OzoneFileStatus> listStatus(String volumeName, String bucketName,
     return null;
   }
 
-  @Override
-  public List<OzoneFileStatusLight> listStatusLight(String volumeName,
-      String bucketName, String keyName, boolean recursive, String startKey,
-      long numEntries, boolean allowPartialPrefixes) throws IOException {
-    return null;
-  }
-
   @Override
   public boolean addAcl(OzoneObj obj, OzoneAcl acl) throws IOException {
     return false;

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketReadWriteKeyOps.java

@@ -130,7 +130,7 @@ protected int getReadCount(int readCount, String readPath)
       throws IOException {
     List<OzoneFileStatus> ozoneFileStatusList = bucket.listStatus(
         OzoneConsts.OM_KEY_PREFIX + readPath + OzoneConsts.OM_KEY_PREFIX, true,
-        "/", keyCountForRead);
+        "/", keyCountForRead, false);
     readCount += ozoneFileStatusList.size();
     return readCount;
   }

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/DeleteKeyHandler.java

@@ -106,7 +106,7 @@ private void deleteFSOKey(OzoneBucket bucket, String keyName)
 
       if (bucket.getFileStatus(keyName).isDirectory()) {
         List<OzoneFileStatus> ozoneFileStatusList =
-            bucket.listStatus(keyName, false, "", 1);
+            bucket.listStatus(keyName, false, "", 1, false);
         if (ozoneFileStatusList != null && !ozoneFileStatusList.isEmpty()) {
           out().printf("Directory is not empty %n");
           return;