From e6af4363813913cec07f3147661177f77dcbb069 Mon Sep 17 00:00:00 2001 From: tanvipenumudy Date: Wed, 27 Sep 2023 10:17:23 +0530 Subject: [PATCH 1/5] HDDS-7715. Implement a lightweight listStatus API --- .../hadoop/ozone/client/OzoneBucket.java | 21 ++- .../ozone/client/protocol/ClientProtocol.java | 37 +++++ .../hadoop/ozone/client/rpc/RpcClient.java | 20 +++ .../java/org/apache/hadoop/ozone/OmUtils.java | 1 + .../hadoop/ozone/om/IOmMetadataReader.java | 27 +++ .../ozone/om/helpers/BasicOmKeyInfo.java | 24 +++ .../om/helpers/OzoneFileStatusLight.java | 157 ++++++++++++++++++ .../om/protocol/OzoneManagerProtocol.java | 32 ++++ ...ManagerProtocolClientSideTranslatorPB.java | 46 +++++ .../src/main/proto/OmClientProtocol.proto | 18 ++ .../hadoop/ozone/om/OmMetadataReader.java | 13 ++ .../apache/hadoop/ozone/om/OmSnapshot.java | 14 ++ .../apache/hadoop/ozone/om/OzoneManager.java | 26 +++ .../OzoneManagerRequestHandler.java | 33 ++++ .../ozone/client/ClientProtocolStub.java | 15 ++ 15 files changed, 475 insertions(+), 9 deletions(-) create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatusLight.java diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index bc8dcdd0e5cc..be2da37afb3b 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -38,10 +38,12 @@ import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.WithMetadata; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -1226,8 +1228,9 @@ protected List getNextShallowListOfKeys(String prevKey) // Elements in statuses must be sorted after startKey, // which means they come after the keyPrefix. - List statuses = proxy.listStatus(volumeName, name, - delimiterKeyPrefix, false, startKey, listCacheSize); + List statuses = + proxy.listStatusLight(volumeName, name, delimiterKeyPrefix, false, + startKey, listCacheSize); if (addedKeyPrefix) { // previous round already include the startKey, so remove it @@ -1246,10 +1249,10 @@ protected List getNextShallowListOfKeys(String prevKey) } private List buildOzoneKeysFromFileStatus( - List statuses) { + List statuses) { return statuses.stream() .map(status -> { - OmKeyInfo keyInfo = status.getKeyInfo(); + BasicOmKeyInfo keyInfo = status.getKeyInfo(); String keyName = keyInfo.getKeyName(); if (status.isDirectory()) { // add trailing slash to represent directory @@ -1497,8 +1500,8 @@ private boolean getChildrenKeys(String keyPrefix, String startKey, startKey = startKey == null ? "" : startKey; // 1. Get immediate children of keyPrefix, starting with startKey - List statuses = proxy.listStatus(volumeName, name, - keyPrefix, false, startKey, listCacheSize, true); + List statuses = proxy.listStatusLight(volumeName, + name, keyPrefix, false, startKey, listCacheSize, true); boolean reachedLimitCacheSize = statuses.size() == listCacheSize; // 2. Special case: ListKey expects keyPrefix element should present in @@ -1516,8 +1519,8 @@ private boolean getChildrenKeys(String keyPrefix, String startKey, // 4. Iterating over the resultStatuses list and add each key to the // resultList. for (int indx = 0; indx < statuses.size(); indx++) { - OzoneFileStatus status = statuses.get(indx); - OmKeyInfo keyInfo = status.getKeyInfo(); + OzoneFileStatusLight status = statuses.get(indx); + BasicOmKeyInfo keyInfo = status.getKeyInfo(); String keyName = keyInfo.getKeyName(); OzoneKey ozoneKey; @@ -1559,7 +1562,7 @@ private boolean getChildrenKeys(String keyPrefix, String startKey, } private void removeStartKeyIfExistsInStatusList(String startKey, - List statuses) { + List statuses) { if (!statuses.isEmpty()) { String firstElement = statuses.get(0).getKeyInfo().getKeyName(); diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 3d51341ec07c..f7f43f24283f 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -54,6 +54,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; @@ -881,6 +882,23 @@ List listStatus(String volumeName, String bucketName, String keyName, boolean recursive, String startKey, long numEntries) throws IOException; + /** + * Lightweight listStatus API. + * + * @param volumeName Volume name + * @param bucketName Bucket name + * @param keyName Absolute path of the entry to be listed + * @param recursive For a directory if true all the descendants of a + * particular directory are listed + * @param startKey Key from which listing needs to start. If startKey exists + * its status is included in the final list. + * @param numEntries Number of entries to list from the start key + * @return list of file status + */ + List listStatusLight(String volumeName, + String bucketName, String keyName, boolean recursive, String startKey, + long numEntries) throws IOException; + /** * List the status for a file or a directory and its contents. @@ -901,6 +919,25 @@ List listStatus(String volumeName, String bucketName, String keyName, boolean recursive, String startKey, long numEntries, boolean allowPartialPrefixes) throws IOException; + /** + * Lightweight listStatus API. + * + * @param volumeName Volume name + * @param bucketName Bucket name + * @param keyName Absolute path of the entry to be listed + * @param recursive For a directory if true all the descendants of a + * particular directory are listed + * @param startKey Key from which listing needs to start. If startKey exists + * its status is included in the final list. + * @param numEntries Number of entries to list from the start key + * @param allowPartialPrefixes if partial prefixes should be allowed, + * this is needed in context of ListKeys + * @return list of file status + */ + List listStatusLight(String volumeName, + String bucketName, String keyName, boolean recursive, String startKey, + long numEntries, boolean allowPartialPrefixes) throws IOException; + /** * Add acl for Ozone object. Return true if acl is added successfully else * false. diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index d3a3106934e8..4cf8fec4e9f5 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -131,6 +131,7 @@ import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; @@ -2064,6 +2065,15 @@ public List listStatus(String volumeName, String bucketName, .listStatus(keyArgs, recursive, startKey, numEntries); } + @Override + public List listStatusLight(String volumeName, + String bucketName, String keyName, boolean recursive, String startKey, + long numEntries) throws IOException { + OmKeyArgs keyArgs = prepareOmKeyArgs(volumeName, bucketName, keyName); + return ozoneManagerClient + .listStatusLight(keyArgs, recursive, startKey, numEntries); + } + @Override public List listStatus(String volumeName, String bucketName, String keyName, boolean recursive, String startKey, @@ -2074,6 +2084,16 @@ public List listStatus(String volumeName, String bucketName, allowPartialPrefixes); } + @Override + public List listStatusLight(String volumeName, + String bucketName, String keyName, boolean recursive, String startKey, + long numEntries, boolean allowPartialPrefixes) throws IOException { + OmKeyArgs keyArgs = prepareOmKeyArgs(volumeName, bucketName, keyName); + return ozoneManagerClient + .listStatusLight(keyArgs, recursive, startKey, numEntries, + allowPartialPrefixes); + } + /** * Add acl for Ozone object. Return true if acl is added successfully else * false. diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 0e0a89a7a2db..09377539abbf 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -248,6 +248,7 @@ public static boolean isReadOnly( case GetFileStatus: case LookupFile: case ListStatus: + case ListStatusLight: case GetAcl: case DBUpdates: case ListMultipartUploads: diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java index c0a920aca948..576a241bac70 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java @@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult; import org.apache.hadoop.ozone.om.helpers.ListKeysResult; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.security.acl.OzoneObj; import java.io.IOException; @@ -64,12 +65,38 @@ List listStatus(OmKeyArgs args, boolean recursive, boolean allowPartialPrefixes) throws IOException; + /** + * Lightweight listStatus API. + * + * @param args Key args + * @param recursive For a directory if true all the descendants of a + * particular directory are listed + * @param startKey Key from which listing needs to start. If startKey exists + * its status is included in the final list. + * @param numEntries Number of entries to list from the start key + * @param allowPartialPrefixes if partial prefixes should be allowed, + * this is needed in context of ListKeys + * @return list of file status + */ + List listStatusLight(OmKeyArgs args, boolean recursive, + String startKey, long numEntries, + boolean allowPartialPrefixes) + throws IOException; + default List listStatus(OmKeyArgs args, boolean recursive, String startKey, long numEntries) throws IOException { return listStatus(args, recursive, startKey, numEntries, false); } + default List listStatusLight(OmKeyArgs args, + boolean recursive, + String startKey, + long numEntries) + throws IOException { + return listStatusLight(args, recursive, startKey, numEntries, false); + } + /** * OzoneFS api to get file status for an entry. * diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java index 2d7d85983209..9c9a5027774f 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java @@ -186,6 +186,30 @@ public static BasicOmKeyInfo getFromProtobuf(BasicKeyInfo basicKeyInfo, return builder.build(); } + public static BasicOmKeyInfo getFromProtobuf(String volumeName, + String bucketName, BasicKeyInfo basicKeyInfo) throws IOException { + if (basicKeyInfo == null) { + return null; + } + + String keyName = basicKeyInfo.getKeyName(); + + Builder builder = new Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setDataSize(basicKeyInfo.getDataSize()) + .setCreationTime(basicKeyInfo.getCreationTime()) + .setModificationTime(basicKeyInfo.getModificationTime()) + .setReplicationConfig(ReplicationConfig.fromProto( + basicKeyInfo.getType(), + basicKeyInfo.getFactor(), + basicKeyInfo.getEcReplicationConfig())) + .setIsFile(!keyName.endsWith("/")); + + return builder.build(); + } + public boolean equals(Object o) { if (this == o) { return true; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatusLight.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatusLight.java new file mode 100644 index 000000000000..8956905bf756 --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatusLight.java @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.helpers; + +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneFileStatusProtoLight; + +import java.io.IOException; +import java.util.Objects; + +import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; + +/** + * Lightweight OzoneFileStatus class. + */ +public class OzoneFileStatusLight { + + private BasicOmKeyInfo keyInfo; + + private boolean isDirectory; + + private long blockSize; + + public OzoneFileStatusLight() { + isDirectory = true; + } + + public OzoneFileStatusLight(BasicOmKeyInfo keyInfo, + long blockSize, boolean isDirectory) { + this.keyInfo = keyInfo; + this.isDirectory = isDirectory; + this.blockSize = blockSize; + } + + public BasicOmKeyInfo getKeyInfo() { + return keyInfo; + } + + public long getBlockSize() { + return blockSize; + } + + public String getTrimmedName() { + String keyName = keyInfo.getKeyName(); + if (keyName.endsWith(OZONE_URI_DELIMITER)) { + return keyName.substring(0, keyName.length() - 1); + } else { + return keyName; + } + } + + public String getPath() { + if (keyInfo == null) { + return OZONE_URI_DELIMITER; + } else { + String path = OZONE_URI_DELIMITER + keyInfo.getKeyName(); + if (path.endsWith(OZONE_URI_DELIMITER)) { + return path.substring(0, path.length() - 1); + } else { + return path; + } + } + } + + public boolean isDirectory() { + if (keyInfo == null) { + return true; + } + return isDirectory; + } + + public boolean isFile() { + return !isDirectory(); + } + + public OzoneFileStatusProtoLight getProtobuf() { + OzoneFileStatusProtoLight.Builder builder = + OzoneFileStatusProtoLight.newBuilder() + .setBlockSize(blockSize) + .setIsDirectory(isDirectory); + //key info can be null for the fake root entry. + if (keyInfo != null) { + builder.setBasicKeyInfo(keyInfo.getProtobuf()) + .setVolumeName(keyInfo.getVolumeName()) + .setBucketName(keyInfo.getBucketName()); + } + return builder.build(); + } + + public static OzoneFileStatusLight getFromProtobuf( + OzoneFileStatusProtoLight status) throws IOException { + return new OzoneFileStatusLight( + BasicOmKeyInfo.getFromProtobuf(status.getVolumeName(), + status.getBucketName(), status.getBasicKeyInfo()), + status.getBlockSize(), + status.getIsDirectory()); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof OzoneFileStatusLight)) { + return false; + } + OzoneFileStatusLight that = (OzoneFileStatusLight) o; + return isDirectory == that.isDirectory && + blockSize == that.blockSize && + getTrimmedName().equals(that.getTrimmedName()); + } + + @Override + public int hashCode() { + return Objects.hash(getTrimmedName()); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()); + sb.append("{"); + if (keyInfo == null) { + sb.append(""); + } else { + sb.append(getTrimmedName()); + if (isDirectory) { + sb.append(" (dir)"); + } + } + sb.append("}"); + return sb.toString(); + } + + public static OzoneFileStatusLight fromOzoneFileStatus( + OzoneFileStatus ozoneFileStatus) { + BasicOmKeyInfo basicKeyInfo = + BasicOmKeyInfo.fromOmKeyInfo(ozoneFileStatus.getKeyInfo()); + return new OzoneFileStatusLight(basicKeyInfo, + ozoneFileStatus.getBlockSize(), ozoneFileStatus.isDirectory()); + } +} diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index eddba4f96f48..bae3ac931dce 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -50,6 +50,7 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; @@ -894,6 +895,20 @@ default OpenKeySession createFile(OmKeyArgs keyArgs, boolean overWrite, List listStatus(OmKeyArgs keyArgs, boolean recursive, String startKey, long numEntries) throws IOException; + /** + * Lightweight listStatus API. + * + * @param keyArgs Key args + * @param recursive For a directory if true all the descendants of a + * particular directory are listed + * @param startKey Key from which listing needs to start. If startKey exists + * its status is included in the final list. + * @param numEntries Number of entries to list from the start key + * @return list of file status + */ + List listStatusLight(OmKeyArgs keyArgs, + boolean recursive, String startKey, long numEntries) throws IOException; + /** * List the status for a file or a directory and its contents. * @@ -912,6 +927,23 @@ List listStatus(OmKeyArgs keyArgs, boolean recursive, boolean allowPartialPrefixes) throws IOException; + /** + * Lightweight listStatus API. + * + * @param keyArgs Key args + * @param recursive For a directory if true all the descendants of a + * particular directory are listed + * @param startKey Key from which listing needs to start. If startKey exists + * its status is included in the final list. + * @param numEntries Number of entries to list from the start key + * @param allowPartialPrefixes if partial prefixes should be allowed, + * this is needed in context of ListKeys + * @return list of file status + */ + List listStatusLight(OmKeyArgs keyArgs, + boolean recursive, String startKey, long numEntries, + boolean allowPartialPrefixes) throws IOException; + /** * Add acl for Ozone object. Return true if acl is added successfully else * false. diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 7133ae03e475..c923eead76cf 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -65,6 +65,7 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; @@ -134,6 +135,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListMultipartUploadsResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusLightResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashRequest; @@ -157,6 +159,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneFileStatusProto; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneFileStatusProtoLight; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrintCompactionLogDagRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RangerBGSyncRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RangerBGSyncResponse; @@ -2222,12 +2225,55 @@ public List listStatus(OmKeyArgs args, boolean recursive, return statusList; } + @Override + public List listStatusLight(OmKeyArgs args, + boolean recursive, String startKey, long numEntries, + boolean allowPartialPrefixes) throws IOException { + KeyArgs keyArgs = KeyArgs.newBuilder() + .setVolumeName(args.getVolumeName()) + .setBucketName(args.getBucketName()) + .setKeyName(args.getKeyName()) + .setSortDatanodes(args.getSortDatanodes()) + .setLatestVersionLocation(args.getLatestVersionLocation()) + .build(); + ListStatusRequest.Builder listStatusRequestBuilder = + ListStatusRequest.newBuilder() + .setKeyArgs(keyArgs) + .setRecursive(recursive) + .setStartKey(startKey) + .setNumEntries(numEntries); + + if (allowPartialPrefixes) { + listStatusRequestBuilder.setAllowPartialPrefix(allowPartialPrefixes); + } + + OMRequest omRequest = createOMRequest(Type.ListStatusLight) + .setListStatusRequest(listStatusRequestBuilder.build()) + .build(); + ListStatusLightResponse listStatusLightResponse = + handleError(submitRequest(omRequest)).getListStatusLightResponse(); + List statusList = + new ArrayList<>(listStatusLightResponse.getStatusesCount()); + for (OzoneFileStatusProtoLight fileStatus : listStatusLightResponse + .getStatusesList()) { + + statusList.add(OzoneFileStatusLight.getFromProtobuf(fileStatus)); + } + return statusList; + } + @Override public List listStatus(OmKeyArgs args, boolean recursive, String startKey, long numEntries) throws IOException { return listStatus(args, recursive, startKey, numEntries, false); } + @Override + public List listStatusLight(OmKeyArgs args, + boolean recursive, String startKey, long numEntries) throws IOException { + return listStatusLight(args, recursive, startKey, numEntries, false); + } + @Override public List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 01d384d5c2dc..b6d2a1a8b3d3 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -142,6 +142,7 @@ enum Type { SetSafeMode = 124; PrintCompactionLogDag = 125; ListKeysLight = 126; + ListStatusLight = 127; } enum SafeMode { @@ -390,6 +391,7 @@ message OMResponse { optional SetSafeModeResponse SetSafeModeResponse = 124; optional PrintCompactionLogDagResponse PrintCompactionLogDagResponse = 125; optional ListKeysLightResponse listKeysLightResponse = 126; + optional ListStatusLightResponse listStatusLightResponse = 127; } enum Status { @@ -1119,6 +1121,14 @@ message OzoneFileStatusProto { } +message OzoneFileStatusProtoLight { + optional string volumeName = 1; + optional string bucketName = 2; + optional BasicKeyInfo basicKeyInfo = 3; + optional uint64 blockSize = 4; + optional bool isDirectory = 5; +} + message GetFileStatusRequest { required KeyArgs keyArgs = 1; } @@ -1127,6 +1137,10 @@ message GetFileStatusResponse { required OzoneFileStatusProto status = 1; } +message GetFileStatusLightResponse { + required OzoneFileStatusProtoLight status = 1; +} + message CreateDirectoryRequest { required KeyArgs keyArgs = 1; } @@ -1172,6 +1186,10 @@ message ListStatusResponse { repeated OzoneFileStatusProto statuses = 1; } +message ListStatusLightResponse { + repeated OzoneFileStatusProtoLight statuses = 1; +} + message CreateKeyRequest { required KeyArgs keyArgs = 1; // Set in OM HA during preExecute step. This way all OM's use same ID in diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java index 467b23df520b..af67d40c77dc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java @@ -37,6 +37,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.security.acl.OzoneObjInfo; import org.apache.hadoop.ozone.security.acl.RequestContext; @@ -251,6 +252,18 @@ public List listStatus(OmKeyArgs args, boolean recursive, } } } + + @Override + public List listStatusLight(OmKeyArgs args, + boolean recursive, String startKey, long numEntries, + boolean allowPartialPrefixes) throws IOException { + List ozoneFileStatuses = + listStatus(args, recursive, startKey, numEntries, allowPartialPrefixes); + + return ozoneFileStatuses.stream() + .map(OzoneFileStatusLight::fromOzoneFileStatus) + .collect(Collectors.toList()); + } @Override public OzoneFileStatus getFileStatus(OmKeyArgs args) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java index a8e3f5c5d33e..5839c61cf31f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java @@ -31,6 +31,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.apache.hadoop.ozone.security.acl.OzoneAuthorizerFactory; @@ -123,6 +124,19 @@ recursive, normalizeKeyName(startKey), numEntries, .collect(Collectors.toList()); } + @Override + public List listStatusLight(OmKeyArgs args, + boolean recursive, String startKey, long numEntries, + boolean allowPartialPrefixes) throws IOException { + + List ozoneFileStatuses = + listStatus(args, recursive, startKey, numEntries, allowPartialPrefixes); + + return ozoneFileStatuses.stream() + .map(OzoneFileStatusLight::fromOzoneFileStatus) + .collect(Collectors.toList()); + } + @Override public OzoneFileStatus getFileStatus(OmKeyArgs args) throws IOException { return denormalizeOzoneFileStatus( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index e627d97bc6bd..d44ccd37f78b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -162,6 +162,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; @@ -3574,6 +3575,20 @@ public List listStatus(OmKeyArgs args, boolean recursive, return listStatus(args, recursive, startKey, numEntries, false); } + @Override + public List listStatusLight(OmKeyArgs args, + boolean recursive, + String startKey, + long numEntries) + throws IOException { + List ozoneFileStatuses = + listStatus(args, recursive, startKey, numEntries); + + return ozoneFileStatuses.stream() + .map(OzoneFileStatusLight::fromOzoneFileStatus) + .collect(Collectors.toList()); + } + public List listStatus(OmKeyArgs args, boolean recursive, String startKey, long numEntries, boolean allowPartialPrefixes) throws IOException { @@ -3584,6 +3599,17 @@ public List listStatus(OmKeyArgs args, boolean recursive, } } + public List listStatusLight(OmKeyArgs args, + boolean recursive, String startKey, long numEntries, + boolean allowPartialPrefixes) throws IOException { + List ozoneFileStatuses = + listStatus(args, recursive, startKey, numEntries, allowPartialPrefixes); + + return ozoneFileStatuses.stream() + .map(OzoneFileStatusLight::fromOzoneFileStatus) + .collect(Collectors.toList()); + } + /** * {@inheritDoc} */ diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 080f48cfcc15..b0c9193f5e40 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -55,6 +55,7 @@ import org.apache.hadoop.ozone.om.helpers.OmPartInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx; @@ -146,6 +147,7 @@ import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListMultipartUploadsResponse; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusRequest; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusResponse; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusLightResponse; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupFileRequest; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupFileResponse; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartUploadInfo; @@ -270,6 +272,12 @@ public OMResponse handleReadRequest(OMRequest request) { listStatus(request.getListStatusRequest(), request.getVersion()); responseBuilder.setListStatusResponse(listStatusResponse); break; + case ListStatusLight: + ListStatusLightResponse listStatusLightResponse = + listStatusLight(request.getListStatusRequest(), + request.getVersion()); + responseBuilder.setListStatusLightResponse(listStatusLightResponse); + break; case GetAcl: GetAclResponse getAclResponse = getAcl(request.getGetAclRequest()); @@ -1179,6 +1187,31 @@ private ListStatusResponse listStatus( return listStatusResponseBuilder.build(); } + private ListStatusLightResponse listStatusLight( + ListStatusRequest request, int clientVersion) throws IOException { + KeyArgs keyArgs = request.getKeyArgs(); + OmKeyArgs omKeyArgs = new OmKeyArgs.Builder() + .setVolumeName(keyArgs.getVolumeName()) + .setBucketName(keyArgs.getBucketName()) + .setKeyName(keyArgs.getKeyName()) + .setLatestVersionLocation(keyArgs.getLatestVersionLocation()) + .setHeadOp(keyArgs.getHeadOp()) + .build(); + boolean allowPartialPrefixes = + request.hasAllowPartialPrefix() && request.getAllowPartialPrefix(); + List statuses = + impl.listStatusLight(omKeyArgs, request.getRecursive(), + request.getStartKey(), request.getNumEntries(), + allowPartialPrefixes); + ListStatusLightResponse.Builder + listStatusLightResponseBuilder = + ListStatusLightResponse.newBuilder(); + for (OzoneFileStatusLight status : statuses) { + listStatusLightResponseBuilder.addStatuses(status.getProtobuf()); + } + return listStatusLightResponseBuilder.build(); + } + @RequestFeatureValidator( conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.POST_PROCESS, diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index 4eeb2ecaa1f0..1adafd2fb7f3 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -38,6 +38,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; @@ -510,6 +511,13 @@ public List listStatus(String volumeName, String bucketName, return null; } + @Override + public List listStatusLight(String volumeName, + String bucketName, String keyName, boolean recursive, String startKey, + long numEntries) throws IOException { + return null; + } + @Override public List listStatus(String volumeName, String bucketName, String keyName, boolean recursive, @@ -519,6 +527,13 @@ public List listStatus(String volumeName, String bucketName, return null; } + @Override + public List listStatusLight(String volumeName, + String bucketName, String keyName, boolean recursive, String startKey, + long numEntries, boolean allowPartialPrefixes) throws IOException { + return null; + } + @Override public boolean addAcl(OzoneObj obj, OzoneAcl acl) throws IOException { return false; From a2832c929aa859e7a39d5c71f53915bba33102d6 Mon Sep 17 00:00:00 2001 From: tanvipenumudy <46785609+tanvipenumudy@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:55:02 +0530 Subject: [PATCH 2/5] Update hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java Co-authored-by: Doroszlai, Attila <6454655+adoroszlai@users.noreply.github.com> --- .../src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index d91b78316f18..c1549c3056b0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3637,6 +3637,7 @@ public List listStatus(OmKeyArgs args, boolean recursive, } } + @Override public List listStatusLight(OmKeyArgs args, boolean recursive, String startKey, long numEntries, boolean allowPartialPrefixes) throws IOException { From 7832a4016cf8d796f4a3d2d5d39c0f467639954e Mon Sep 17 00:00:00 2001 From: tanvipenumudy Date: Tue, 17 Oct 2023 15:49:09 +0530 Subject: [PATCH 3/5] Remove the redundant listStatusLight API implementations --- .../apache/hadoop/ozone/client/OzoneBucket.java | 2 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 9 --------- .../apache/hadoop/ozone/om/IOmMetadataReader.java | 8 -------- .../ozone/om/protocol/OzoneManagerProtocol.java | 14 -------------- ...OzoneManagerProtocolClientSideTranslatorPB.java | 6 ------ .../org/apache/hadoop/ozone/om/OzoneManager.java | 14 -------------- .../hadoop/ozone/client/ClientProtocolStub.java | 7 ------- 7 files changed, 1 insertion(+), 59 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index be2da37afb3b..c4c68c00fdb7 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -1230,7 +1230,7 @@ protected List getNextShallowListOfKeys(String prevKey) // which means they come after the keyPrefix. List statuses = proxy.listStatusLight(volumeName, name, delimiterKeyPrefix, false, - startKey, listCacheSize); + startKey, listCacheSize, false); if (addedKeyPrefix) { // previous round already include the startKey, so remove it diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 4cf8fec4e9f5..fc91335771e1 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -2065,15 +2065,6 @@ public List listStatus(String volumeName, String bucketName, .listStatus(keyArgs, recursive, startKey, numEntries); } - @Override - public List listStatusLight(String volumeName, - String bucketName, String keyName, boolean recursive, String startKey, - long numEntries) throws IOException { - OmKeyArgs keyArgs = prepareOmKeyArgs(volumeName, bucketName, keyName); - return ozoneManagerClient - .listStatusLight(keyArgs, recursive, startKey, numEntries); - } - @Override public List listStatus(String volumeName, String bucketName, String keyName, boolean recursive, String startKey, diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java index 576a241bac70..7d4e769365fd 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java @@ -89,14 +89,6 @@ default List listStatus(OmKeyArgs args, boolean recursive, return listStatus(args, recursive, startKey, numEntries, false); } - default List listStatusLight(OmKeyArgs args, - boolean recursive, - String startKey, - long numEntries) - throws IOException { - return listStatusLight(args, recursive, startKey, numEntries, false); - } - /** * OzoneFS api to get file status for an entry. * diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index bae3ac931dce..7d710ae8717d 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -895,20 +895,6 @@ default OpenKeySession createFile(OmKeyArgs keyArgs, boolean overWrite, List listStatus(OmKeyArgs keyArgs, boolean recursive, String startKey, long numEntries) throws IOException; - /** - * Lightweight listStatus API. - * - * @param keyArgs Key args - * @param recursive For a directory if true all the descendants of a - * particular directory are listed - * @param startKey Key from which listing needs to start. If startKey exists - * its status is included in the final list. - * @param numEntries Number of entries to list from the start key - * @return list of file status - */ - List listStatusLight(OmKeyArgs keyArgs, - boolean recursive, String startKey, long numEntries) throws IOException; - /** * List the status for a file or a directory and its contents. * diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index c923eead76cf..0cbb3203b65c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -2268,12 +2268,6 @@ public List listStatus(OmKeyArgs args, boolean recursive, return listStatus(args, recursive, startKey, numEntries, false); } - @Override - public List listStatusLight(OmKeyArgs args, - boolean recursive, String startKey, long numEntries) throws IOException { - return listStatusLight(args, recursive, startKey, numEntries, false); - } - @Override public List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index c1549c3056b0..3be9e6319d65 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3613,20 +3613,6 @@ public List listStatus(OmKeyArgs args, boolean recursive, return listStatus(args, recursive, startKey, numEntries, false); } - @Override - public List listStatusLight(OmKeyArgs args, - boolean recursive, - String startKey, - long numEntries) - throws IOException { - List ozoneFileStatuses = - listStatus(args, recursive, startKey, numEntries); - - return ozoneFileStatuses.stream() - .map(OzoneFileStatusLight::fromOzoneFileStatus) - .collect(Collectors.toList()); - } - public List listStatus(OmKeyArgs args, boolean recursive, String startKey, long numEntries, boolean allowPartialPrefixes) throws IOException { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index 1adafd2fb7f3..5505688b26c1 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -511,13 +511,6 @@ public List listStatus(String volumeName, String bucketName, return null; } - @Override - public List listStatusLight(String volumeName, - String bucketName, String keyName, boolean recursive, String startKey, - long numEntries) throws IOException { - return null; - } - @Override public List listStatus(String volumeName, String bucketName, String keyName, boolean recursive, From 143a8a1016570394554d722503b6287f3a242223 Mon Sep 17 00:00:00 2001 From: tanvipenumudy Date: Tue, 17 Oct 2023 16:33:49 +0530 Subject: [PATCH 4/5] Remove redundant listStatusLight under ClientProtocol --- .../ozone/client/protocol/ClientProtocol.java | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index f7f43f24283f..b45a3209f418 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -882,24 +882,6 @@ List listStatus(String volumeName, String bucketName, String keyName, boolean recursive, String startKey, long numEntries) throws IOException; - /** - * Lightweight listStatus API. - * - * @param volumeName Volume name - * @param bucketName Bucket name - * @param keyName Absolute path of the entry to be listed - * @param recursive For a directory if true all the descendants of a - * particular directory are listed - * @param startKey Key from which listing needs to start. If startKey exists - * its status is included in the final list. - * @param numEntries Number of entries to list from the start key - * @return list of file status - */ - List listStatusLight(String volumeName, - String bucketName, String keyName, boolean recursive, String startKey, - long numEntries) throws IOException; - - /** * List the status for a file or a directory and its contents. * From 6aa370994a7d603880ca4fd5cf08794dd1ddd4df Mon Sep 17 00:00:00 2001 From: tanvipenumudy Date: Thu, 19 Oct 2023 03:46:38 +0530 Subject: [PATCH 5/5] Hard-code sortDatanodes and latestVersionLocation --- .../OzoneManagerProtocolClientSideTranslatorPB.java | 4 ++-- .../hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 0cbb3203b65c..567fbdad1717 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -2233,8 +2233,8 @@ public List listStatusLight(OmKeyArgs args, .setVolumeName(args.getVolumeName()) .setBucketName(args.getBucketName()) .setKeyName(args.getKeyName()) - .setSortDatanodes(args.getSortDatanodes()) - .setLatestVersionLocation(args.getLatestVersionLocation()) + .setSortDatanodes(false) + .setLatestVersionLocation(true) .build(); ListStatusRequest.Builder listStatusRequestBuilder = ListStatusRequest.newBuilder() diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index b0c9193f5e40..68f42e718ca9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -1194,7 +1194,8 @@ private ListStatusLightResponse listStatusLight( .setVolumeName(keyArgs.getVolumeName()) .setBucketName(keyArgs.getBucketName()) .setKeyName(keyArgs.getKeyName()) - .setLatestVersionLocation(keyArgs.getLatestVersionLocation()) + .setSortDatanodesInPipeline(false) + .setLatestVersionLocation(true) .setHeadOp(keyArgs.getHeadOp()) .build(); boolean allowPartialPrefixes =