diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java index a91885e25dde..00f2e55f97f4 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java @@ -37,6 +37,9 @@ public enum OzoneManagerVersion implements ComponentVersion { OPTIMIZED_GET_KEY_INFO(3, "OzoneManager version that supports optimized" + " key lookups using cached container locations."), + LIGHTWEIGHT_LIST_KEYS(4, "OzoneManager version that supports lightweight" + + " listKeys API."), + FUTURE_VERSION(-1, "Used internally in the client when the server side is " + " newer and an unknown server version has arrived to the client."); diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 984c6e70b160..6f1c0f6493b0 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -107,6 +107,7 @@ import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo; import org.apache.hadoop.ozone.om.helpers.DeleteTenantState; @@ -1558,18 +1559,35 @@ public List listKeys(String volumeName, String bucketName, String keyPrefix, String prevKey, int maxListResult) throws IOException { - List keys = ozoneManagerClient.listKeys( - volumeName, bucketName, prevKey, keyPrefix, maxListResult); - return keys.stream().map(key -> new OzoneKey( - key.getVolumeName(), - key.getBucketName(), - key.getKeyName(), - key.getDataSize(), - key.getCreationTime(), - key.getModificationTime(), - key.getReplicationConfig(), - key.isFile())) - .collect(Collectors.toList()); + + if (omVersion.compareTo(OzoneManagerVersion.LIGHTWEIGHT_LIST_KEYS) >= 0) { + List keys = ozoneManagerClient.listKeysLight( + volumeName, bucketName, prevKey, keyPrefix, maxListResult).getKeys(); + + return keys.stream().map(key -> new OzoneKey( + key.getVolumeName(), + key.getBucketName(), + key.getKeyName(), + key.getDataSize(), + key.getCreationTime(), + key.getModificationTime(), + key.getReplicationConfig(), + key.isFile())) + .collect(Collectors.toList()); + } else { + List keys = ozoneManagerClient.listKeys( + volumeName, bucketName, prevKey, keyPrefix, maxListResult).getKeys(); + return keys.stream().map(key -> new OzoneKey( + key.getVolumeName(), + key.getBucketName(), + key.getKeyName(), + key.getDataSize(), + key.getCreationTime(), + key.getModificationTime(), + key.getReplicationConfig(), + key.isFile())) + .collect(Collectors.toList()); + } } @Override diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 5a293d325621..0e0a89a7a2db 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -241,6 +241,7 @@ public static boolean isReadOnly( case ListBuckets: case LookupKey: case ListKeys: + case ListKeysLight: case ListTrash: case ServiceList: case ListMultiPartUploadParts: diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java index 9bdc4772aa22..c0a920aca948 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java @@ -22,6 +22,8 @@ import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult; +import org.apache.hadoop.ozone.om.helpers.ListKeysResult; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.security.acl.OzoneObj; @@ -109,8 +111,32 @@ default List listStatus(OmKeyArgs args, boolean recursive, * the size of the result will not exceed this limit. * @return a list of keys. */ - List listKeys(String volumeName, String bucketName, - String startKey, String keyPrefix, int maxKeys) + ListKeysResult listKeys(String volumeName, String bucketName, + String startKey, String keyPrefix, int maxKeys) + throws IOException; + + /** + * Lightweight listKeys implementation. + * + * @param volumeName + * the name of the volume. + * @param bucketName + * the name of the bucket. + * @param startKey + * the start key name, only the keys whose name is + * after this value will be included in the result. + * @param keyPrefix + * key name prefix, only the keys whose name has + * this prefix will be included in the result. + * @param maxKeys + * the maximum number of keys to return. It ensures + * the size of the result will not exceed this limit. + * @return a list of keys. + * @throws IOException + */ + ListKeysLightResult listKeysLight(String volumeName, String bucketName, + String startKey, String keyPrefix, + int maxKeys) throws IOException; /** diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java new file mode 100644 index 000000000000..2d7d85983209 --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java @@ -0,0 +1,222 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.om.helpers; + +import java.io.IOException; +import java.util.Objects; + +import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BasicKeyInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysRequest; + +/** + * Lightweight OmKeyInfo class. + */ +public class BasicOmKeyInfo { + + private String volumeName; + private String bucketName; + private String keyName; + private long dataSize; + private long creationTime; + private long modificationTime; + private ReplicationConfig replicationConfig; + private boolean isFile; + + @SuppressWarnings("parameternumber") + public BasicOmKeyInfo(String volumeName, String bucketName, String keyName, + long dataSize, long creationTime, long modificationTime, + ReplicationConfig replicationConfig, boolean isFile) { + this.volumeName = volumeName; + this.bucketName = bucketName; + this.keyName = keyName; + this.dataSize = dataSize; + this.creationTime = creationTime; + this.modificationTime = modificationTime; + this.replicationConfig = replicationConfig; + this.isFile = isFile; + } + + public String getVolumeName() { + return volumeName; + } + + public String getBucketName() { + return bucketName; + } + + public String getKeyName() { + return keyName; + } + + public long getDataSize() { + return dataSize; + } + + public long getCreationTime() { + return creationTime; + } + + public long getModificationTime() { + return modificationTime; + } + + public ReplicationConfig getReplicationConfig() { + return replicationConfig; + } + + public boolean isFile() { + return isFile; + } + + /** + * Builder of BasicOmKeyInfo. + */ + public static class Builder { + private String volumeName; + private String bucketName; + private String keyName; + private long dataSize; + private long creationTime; + private long modificationTime; + private ReplicationConfig replicationConfig; + private boolean isFile; + + public Builder setVolumeName(String volumeName) { + this.volumeName = volumeName; + return this; + } + + public Builder setBucketName(String bucketName) { + this.bucketName = bucketName; + return this; + } + + public Builder setKeyName(String keyName) { + this.keyName = keyName; + return this; + } + + public Builder setDataSize(long dataSize) { + this.dataSize = dataSize; + return this; + } + + public Builder setCreationTime(long creationTime) { + this.creationTime = creationTime; + return this; + } + + public Builder setModificationTime(long modificationTime) { + this.modificationTime = modificationTime; + return this; + } + + public Builder setReplicationConfig(ReplicationConfig replicationConfig) { + this.replicationConfig = replicationConfig; + return this; + } + + public Builder setIsFile(boolean isFile) { + this.isFile = isFile; + return this; + } + + public BasicOmKeyInfo build() { + return new BasicOmKeyInfo(volumeName, bucketName, keyName, dataSize, + creationTime, modificationTime, replicationConfig, isFile); + } + } + + public BasicKeyInfo getProtobuf() { + BasicKeyInfo.Builder builder = BasicKeyInfo.newBuilder() + .setKeyName(keyName) + .setDataSize(dataSize) + .setCreationTime(creationTime) + .setModificationTime(modificationTime) + .setType(replicationConfig.getReplicationType()); + if (replicationConfig instanceof ECReplicationConfig) { + builder.setEcReplicationConfig( + ((ECReplicationConfig) replicationConfig).toProto()); + } else { + builder.setFactor(ReplicationConfig.getLegacyFactor(replicationConfig)); + } + + return builder.build(); + } + + public static BasicOmKeyInfo getFromProtobuf(BasicKeyInfo basicKeyInfo, + ListKeysRequest request) + throws IOException { + if (basicKeyInfo == null || request == null) { + return null; + } + + String keyName = basicKeyInfo.getKeyName(); + + Builder builder = new Builder() + .setVolumeName(request.getVolumeName()) + .setBucketName(request.getBucketName()) + .setKeyName(keyName) + .setDataSize(basicKeyInfo.getDataSize()) + .setCreationTime(basicKeyInfo.getCreationTime()) + .setModificationTime(basicKeyInfo.getModificationTime()) + .setReplicationConfig(ReplicationConfig.fromProto( + basicKeyInfo.getType(), + basicKeyInfo.getFactor(), + basicKeyInfo.getEcReplicationConfig())) + .setIsFile(!keyName.endsWith("/")); + + return builder.build(); + } + + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + BasicOmKeyInfo basicOmKeyInfo = (BasicOmKeyInfo) o; + return volumeName.equals(basicOmKeyInfo.volumeName) && + bucketName.equals(basicOmKeyInfo.bucketName) && + keyName.equals(basicOmKeyInfo.keyName) && + dataSize == basicOmKeyInfo.dataSize && + creationTime == basicOmKeyInfo.creationTime && + modificationTime == basicOmKeyInfo.modificationTime && + replicationConfig.equals(basicOmKeyInfo.replicationConfig) && + isFile == basicOmKeyInfo.isFile; + } + + public int hashCode() { + return Objects.hash(volumeName, bucketName, keyName); + } + + public static BasicOmKeyInfo fromOmKeyInfo(OmKeyInfo omKeyInfo) { + return new BasicOmKeyInfo( + omKeyInfo.getVolumeName(), + omKeyInfo.getBucketName(), + omKeyInfo.getKeyName(), + omKeyInfo.getDataSize(), + omKeyInfo.getCreationTime(), + omKeyInfo.getModificationTime(), + omKeyInfo.getReplicationConfig(), + omKeyInfo.isFile()); + } +} diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ListKeysLightResult.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ListKeysLightResult.java new file mode 100644 index 000000000000..47a0669cc339 --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ListKeysLightResult.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.om.helpers; + +import java.util.List; + +/** + * Encapsulates the result of listKeys. It contains a list of + * {@link BasicOmKeyInfo} and a boolean flag indicating if the listing is + * truncated. + */ +public class ListKeysLightResult { + private List keys; + private boolean isTruncated; + + public ListKeysLightResult(List keys, boolean isTruncated) { + this.keys = keys; + this.isTruncated = isTruncated; + } + + public List getKeys() { + return keys; + } + + public boolean isTruncated() { + return isTruncated; + } +} diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ListKeysResult.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ListKeysResult.java new file mode 100644 index 000000000000..679e1a25e9b0 --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ListKeysResult.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.om.helpers; + +import java.util.List; + +/** + * Encapsulates the result of listKeys. It contains a list of {@link OmKeyInfo} + * and a boolean flag indicating if the listing is truncated. + */ +public class ListKeysResult { + private List keys; + private boolean isTruncated; + + public ListKeysResult(List keys, boolean isTruncated) { + this.keys = keys; + this.isTruncated = isTruncated; + } + + public List getKeys() { + return keys; + } + + public boolean isTruncated() { + return isTruncated; + } +} diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index c53ea15617ce..5d5de0b89950 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -40,6 +40,9 @@ import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult; +import org.apache.hadoop.ozone.om.helpers.ListKeysResult; import org.apache.hadoop.ozone.om.helpers.DBUpdates; import org.apache.hadoop.ozone.om.helpers.DeleteTenantState; import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext; @@ -126,6 +129,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListBucketsResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysLightResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListMultipartUploadsRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListMultipartUploadsResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusRequest; @@ -971,7 +975,7 @@ public void deleteBucket(String volume, String bucket) throws IOException { * List keys in a bucket. */ @Override - public List listKeys(String volumeName, String bucketName, + public ListKeysResult listKeys(String volumeName, String bucketName, String startKey, String prefix, int maxKeys) throws IOException { List keys = new ArrayList<>(); ListKeysRequest.Builder reqBuilder = ListKeysRequest.newBuilder(); @@ -1001,10 +1005,49 @@ public List listKeys(String volumeName, String bucketName, list.add(fromProtobuf); } keys.addAll(list); - return keys; + return new ListKeysResult(keys, resp.getIsTruncated()); } + /** + * Lightweight listKeys implementation. + */ + @Override + public ListKeysLightResult listKeysLight(String volumeName, + String bucketName, String startKey, + String prefix, + int maxKeys) throws IOException { + List keys = new ArrayList<>(); + ListKeysRequest.Builder reqBuilder = ListKeysRequest.newBuilder(); + reqBuilder.setVolumeName(volumeName); + reqBuilder.setBucketName(bucketName); + reqBuilder.setCount(maxKeys); + + if (StringUtils.isNotEmpty(startKey)) { + reqBuilder.setStartKey(startKey); + } + + if (prefix != null) { + reqBuilder.setPrefix(prefix); + } + + ListKeysRequest req = reqBuilder.build(); + + OMRequest omRequest = createOMRequest(Type.ListKeysLight) + .setListKeysRequest(req) + .build(); + + ListKeysLightResponse resp = + handleError(submitRequest(omRequest)).getListKeysLightResponse(); + for (OzoneManagerProtocolProtos.BasicKeyInfo + basicKeyInfo : resp.getBasicKeyInfoList()) { + BasicOmKeyInfo fromProtobuf = + BasicOmKeyInfo.getFromProtobuf(basicKeyInfo, req); + keys.add(fromProtobuf); + } + return new ListKeysLightResult(keys, resp.getIsTruncated()); + } + @Override @Nonnull public S3SecretValue getS3Secret(String kerberosID) throws IOException { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java index f9d83c83906c..e2ce4e9ba570 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java @@ -953,8 +953,9 @@ public void testLatestLocationVersion() throws IOException { assertEquals(key.getKeyLocationVersions().size(), 2); // Test ListKeys (latestLocationVersion is always true for ListKeys) - List keyInfos = keyManager.listKeys(keyArgs.getVolumeName(), - keyArgs.getBucketName(), "", keyArgs.getKeyName(), 100); + List keyInfos = + keyManager.listKeys(keyArgs.getVolumeName(), keyArgs.getBucketName(), + "", keyArgs.getKeyName(), 100).getKeys(); assertEquals(keyInfos.size(), 1); assertEquals(keyInfos.get(0).getKeyLocationVersions().size(), 1); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java index 9a80af67d8b8..1529720ecd83 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java @@ -411,7 +411,7 @@ private void assertKeysWritten(String volumeName, Set expectedKeys, // Wait for a potentially slow follower to apply all key writes. LambdaTestUtils.await(WAIT_TIMEOUT_MILLIS, 1000, () -> { List keys = om.getMetadataManager().listKeys(volumeName, - BUCKET, null, KEY_PREFIX, 100); + BUCKET, null, KEY_PREFIX, 100).getKeys(); boolean allKeysFound = (expectedKeys.size() == keys.size()); if (!allKeysFound) { diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index fc4844302bca..df41d1ae8392 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -141,6 +141,7 @@ enum Type { CancelSnapshotDiff = 123; SetSafeMode = 124; PrintCompactionLogDag = 125; + ListKeysLight = 126; } enum SafeMode { @@ -388,6 +389,7 @@ message OMResponse { optional CancelSnapshotDiffResponse cancelSnapshotDiffResponse = 123; optional SetSafeModeResponse SetSafeModeResponse = 124; optional PrintCompactionLogDagResponse PrintCompactionLogDagResponse = 125; + optional ListKeysLightResponse listKeysLightResponse = 126; } enum Status { @@ -1082,6 +1084,16 @@ message KeyInfo { optional bool isFile = 19; } +message BasicKeyInfo { + optional string keyName = 1; + optional uint64 dataSize = 2; + optional uint64 creationTime = 3; + optional uint64 modificationTime = 4; + optional hadoop.hdds.ReplicationType type = 5; + optional hadoop.hdds.ReplicationFactor factor = 6; + optional hadoop.hdds.ECReplicationConfig ecReplicationConfig = 7; +} + message DirectoryInfo { required string name = 1; required uint64 creationTime = 2; @@ -1355,6 +1367,12 @@ message ListKeysRequest { message ListKeysResponse { repeated KeyInfo keyInfo = 2; + optional bool isTruncated = 3; +} + +message ListKeysLightResponse { + repeated BasicKeyInfo basicKeyInfo = 1; + optional bool isTruncated = 2; } message CommitKeyRequest { diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index 3dcabf088e6d..96c55bc451be 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.common.BlockGroup; +import org.apache.hadoop.ozone.om.helpers.ListKeysResult; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDBAccessIdInfo; import org.apache.hadoop.ozone.om.helpers.OmDBUserPrincipalInfo; @@ -201,8 +202,9 @@ List listBuckets(String volumeName, String startBucket, * @return a list of keys. * @throws IOException */ - List listKeys(String volumeName, - String bucketName, String startKey, String keyPrefix, int maxKeys) + ListKeysResult listKeys(String volumeName, + String bucketName, String startKey, String keyPrefix, + int maxKeys) throws IOException; /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index 0bcc7f823127..6ba98cbde09e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -21,6 +21,7 @@ import org.apache.hadoop.ozone.common.BlockGroup; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.ListKeysResult; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList; @@ -98,8 +99,8 @@ public interface KeyManager extends OzoneManagerFS, IOzoneAcl { * @return a list of keys. * @throws IOException */ - List listKeys(String volumeName, - String bucketName, String startKey, String keyPrefix, int maxKeys) + ListKeysResult listKeys(String volumeName, String bucketName, String startKey, + String keyPrefix, int maxKeys) throws IOException; /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 8bce4eb70f59..625893534a69 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -63,6 +63,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo; +import org.apache.hadoop.ozone.om.helpers.ListKeysResult; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; @@ -556,7 +557,7 @@ protected Map refreshPipeline( } @Override - public List listKeys(String volumeName, String bucketName, + public ListKeysResult listKeys(String volumeName, String bucketName, String startKey, String keyPrefix, int maxKeys) throws IOException { Preconditions.checkNotNull(volumeName); @@ -572,15 +573,17 @@ public List listKeys(String volumeName, String bucketName, keyPrefix = OmUtils.normalizeKey(keyPrefix, true); } - List keyList = metadataManager.listKeys(volumeName, bucketName, - startKey, keyPrefix, maxKeys); + ListKeysResult listKeysResult = + metadataManager.listKeys(volumeName, bucketName, startKey, keyPrefix, + maxKeys); + List keyList = listKeysResult.getKeys(); // For listKeys, we return the latest Key Location by default for (OmKeyInfo omKeyInfo : keyList) { slimLocationVersion(omKeyInfo); } - return keyList; + return listKeysResult; } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 1bbe8bbb5f63..6c411adb0c5a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -60,6 +60,7 @@ import org.apache.hadoop.ozone.om.codec.TokenIdentifierCodec; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; +import org.apache.hadoop.ozone.om.helpers.ListKeysResult; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDBAccessIdInfo; import org.apache.hadoop.ozone.om.helpers.OmDBUserPrincipalInfo; @@ -1153,12 +1154,13 @@ public List listBuckets(final String volumeName, } @Override - public List listKeys(String volumeName, String bucketName, - String startKey, String keyPrefix, int maxKeys) throws IOException { + public ListKeysResult listKeys(String volumeName, String bucketName, + String startKey, String keyPrefix, int maxKeys) + throws IOException { List result = new ArrayList<>(); if (maxKeys <= 0) { - return result; + return new ListKeysResult(result, false); } if (Strings.isNullOrEmpty(volumeName)) { @@ -1253,6 +1255,8 @@ public List listKeys(String volumeName, String bucketName, } } + boolean isTruncated = cacheKeyMap.size() > maxKeys; + // Finally DB entries and cache entries are merged, then return the count // of maxKeys from the sorted map. currentCount = 0; @@ -1273,7 +1277,7 @@ public List listKeys(String volumeName, String bucketName, // Clear map and set. cacheKeyMap.clear(); - return result; + return new ListKeysResult(result, isTruncated); } // TODO: HDDS-2419 - Complete stub below for core logic diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java index 72506d982154..467b23df520b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java @@ -30,6 +30,9 @@ import org.apache.hadoop.ozone.audit.Auditor; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult; +import org.apache.hadoop.ozone.om.helpers.ListKeysResult; import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; @@ -43,6 +46,7 @@ import java.net.InetAddress; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getRemoteUser; @@ -305,7 +309,7 @@ public OmKeyInfo lookupFile(OmKeyArgs args) throws IOException { } @Override - public List listKeys(String volumeName, String bucketName, + public ListKeysResult listKeys(String volumeName, String bucketName, String startKey, String keyPrefix, int maxKeys) throws IOException { long startNanos = Time.monotonicNowNanos(); ResolvedBucket bucket = ozoneManager.resolveBucketLink( @@ -340,6 +344,21 @@ public List listKeys(String volumeName, String bucketName, } } + @Override + public ListKeysLightResult listKeysLight(String volumeName, + String bucketName, + String startKey, String keyPrefix, + int maxKeys) throws IOException { + ListKeysResult listKeysResult = + listKeys(volumeName, bucketName, startKey, keyPrefix, maxKeys); + List keys = listKeysResult.getKeys(); + List basicKeysList = + keys.stream().map(BasicOmKeyInfo::fromOmKeyInfo) + .collect(Collectors.toList()); + + return new ListKeysLightResult(basicKeysList, listKeysResult.isTruncated()); + } + /** * Returns list of ACLs for given Ozone object. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java index 8fa9bc25f1ff..a8e3f5c5d33e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java @@ -23,6 +23,9 @@ import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.audit.AuditLoggerType; +import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult; +import org.apache.hadoop.ozone.om.helpers.ListKeysResult; import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; @@ -133,12 +136,29 @@ public OmKeyInfo lookupFile(OmKeyArgs args) throws IOException { } @Override - public List listKeys(String vname, String bname, - String startKey, String keyPrefix, int maxKeys) throws IOException { - List l = omMetadataReader.listKeys(vname, bname, + public ListKeysResult listKeys(String vname, String bname, + String startKey, String keyPrefix, int maxKeys) + throws IOException { + ListKeysResult listKeysResult = omMetadataReader.listKeys(vname, bname, normalizeKeyName(startKey), normalizeKeyName(keyPrefix), maxKeys); - return l.stream().map(this::denormalizeOmKeyInfo) - .collect(Collectors.toList()); + return new ListKeysResult( + listKeysResult.getKeys().stream().map(this::denormalizeOmKeyInfo) + .collect(Collectors.toList()), listKeysResult.isTruncated()); + } + + @Override + public ListKeysLightResult listKeysLight(String volName, + String buckName, + String startKey, String keyPrefix, + int maxKeys) throws IOException { + ListKeysResult listKeysResult = + listKeys(volumeName, bucketName, startKey, keyPrefix, maxKeys); + List keys = listKeysResult.getKeys(); + List basicKeysList = + keys.stream().map(BasicOmKeyInfo::fromOmKeyInfo) + .collect(Collectors.toList()); + + return new ListKeysLightResult(basicKeysList, listKeysResult.isTruncated()); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index b30ff26de3e8..f097e0d57438 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -154,6 +154,9 @@ import org.apache.hadoop.ozone.om.helpers.OmDBUserPrincipalInfo; import org.apache.hadoop.ozone.om.helpers.OmDBTenantState; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.ListKeysResult; +import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts; @@ -2832,15 +2835,31 @@ public KeyInfoWithVolumeContext getKeyInfo(final OmKeyArgs args, * {@inheritDoc} */ @Override - public List listKeys(String volumeName, String bucketName, - String startKey, String keyPrefix, int maxKeys) throws IOException { + public ListKeysResult listKeys(String volumeName, String bucketName, + String startKey, String keyPrefix, int maxKeys) + throws IOException { try (ReferenceCounted rcReader = - getReader(volumeName, bucketName, keyPrefix)) { + getReader(volumeName, bucketName, keyPrefix)) { return rcReader.get().listKeys( volumeName, bucketName, startKey, keyPrefix, maxKeys); } } + @Override + public ListKeysLightResult listKeysLight(String volumeName, + String bucketName, + String startKey, String keyPrefix, + int maxKeys) throws IOException { + ListKeysResult listKeysResult = + listKeys(volumeName, bucketName, startKey, keyPrefix, maxKeys); + List keys = listKeysResult.getKeys(); + List basicKeysList = + keys.stream().map(BasicOmKeyInfo::fromOmKeyInfo) + .collect(Collectors.toList()); + + return new ListKeysLightResult(basicKeysList, listKeysResult.isTruncated()); + } + @Override public List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 08ba70549575..080f48cfcc15 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -42,6 +42,9 @@ import org.apache.hadoop.ozone.om.OzoneManagerPrepareState; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult; +import org.apache.hadoop.ozone.om.helpers.ListKeysResult; import org.apache.hadoop.ozone.om.helpers.DBUpdates; import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -98,6 +101,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListBucketsResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysLightResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashRequest; @@ -216,6 +220,11 @@ public OMResponse handleReadRequest(OMRequest request) { request.getListKeysRequest(), request.getVersion()); responseBuilder.setListKeysResponse(listKeysResponse); break; + case ListKeysLight: + ListKeysLightResponse listKeysLightResponse = listKeysLight( + request.getListKeysRequest()); + responseBuilder.setListKeysLightResponse(listKeysLightResponse); + break; case ListTrash: ListTrashResponse listTrashResponse = listTrash( request.getListTrashRequest(), request.getVersion()); @@ -671,16 +680,34 @@ private ListKeysResponse listKeys(ListKeysRequest request, int clientVersion) ListKeysResponse.Builder resp = ListKeysResponse.newBuilder(); - List keys = impl.listKeys( + ListKeysResult listKeysResult = impl.listKeys( request.getVolumeName(), request.getBucketName(), request.getStartKey(), request.getPrefix(), request.getCount()); - for (OmKeyInfo key : keys) { + for (OmKeyInfo key : listKeysResult.getKeys()) { resp.addKeyInfo(key.getProtobuf(true, clientVersion)); } + resp.setIsTruncated(listKeysResult.isTruncated()); + return resp.build(); + } + private ListKeysLightResponse listKeysLight(ListKeysRequest request) + throws IOException { + ListKeysLightResponse.Builder resp = + ListKeysLightResponse.newBuilder(); + + ListKeysLightResult listKeysLightResult = impl.listKeysLight( + request.getVolumeName(), + request.getBucketName(), + request.getStartKey(), + request.getPrefix(), + request.getCount()); + for (BasicOmKeyInfo key : listKeysLightResult.getKeys()) { + resp.addBasicKeyInfo(key.getProtobuf()); + } + resp.setIsTruncated(listKeysLightResult.isTruncated()); return resp.build(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index 148dc94b2208..9c2163afb36c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -388,7 +388,7 @@ public void testListKeys() throws Exception { // List all keys which have prefix "key-a" List omKeyInfoList = omMetadataManager.listKeys(volumeNameA, ozoneBucket, - null, prefixKeyA, 100); + null, prefixKeyA, 100).getKeys(); assertEquals(omKeyInfoList.size(), 50); @@ -401,7 +401,7 @@ public void testListKeys() throws Exception { String startKey = prefixKeyA + 10; omKeyInfoList = omMetadataManager.listKeys(volumeNameA, ozoneBucket, - startKey, prefixKeyA, 100); + startKey, prefixKeyA, 100).getKeys(); assertEquals(keysASet.tailSet( startKey).size() - 1, omKeyInfoList.size()); @@ -409,7 +409,7 @@ public void testListKeys() throws Exception { startKey = prefixKeyA + 38; omKeyInfoList = omMetadataManager.listKeys(volumeNameA, ozoneBucket, - startKey, prefixKeyA, 100); + startKey, prefixKeyA, 100).getKeys(); assertEquals(keysASet.tailSet( startKey).size() - 1, omKeyInfoList.size()); @@ -424,7 +424,7 @@ public void testListKeys() throws Exception { omKeyInfoList = omMetadataManager.listKeys(volumeNameB, hadoopBucket, - null, prefixKeyB, 100); + null, prefixKeyB, 100).getKeys(); assertEquals(omKeyInfoList.size(), 50); @@ -440,7 +440,7 @@ public void testListKeys() throws Exception { for (int i = 0; i < 5; i++) { omKeyInfoList = omMetadataManager.listKeys(volumeNameB, hadoopBucket, - startKey, prefixKeyB, 10); + startKey, prefixKeyB, 10).getKeys(); assertEquals(10, omKeyInfoList.size()); @@ -458,13 +458,13 @@ public void testListKeys() throws Exception { // As now we have iterated all 50 buckets, calling next time should // return empty list. omKeyInfoList = omMetadataManager.listKeys(volumeNameB, hadoopBucket, - startKey, prefixKeyB, 10); + startKey, prefixKeyB, 10).getKeys(); assertEquals(omKeyInfoList.size(), 0); // List all keys with empty prefix omKeyInfoList = omMetadataManager.listKeys(volumeNameA, ozoneBucket, - null, null, 100); + null, null, 100).getKeys(); assertEquals(50, omKeyInfoList.size()); for (OmKeyInfo omKeyInfo : omKeyInfoList) { assertTrue(omKeyInfo.getKeyName().startsWith( @@ -507,7 +507,7 @@ public void testListKeysWithFewDeleteEntriesInCache() throws Exception { // Now list keys which match with prefixKeyA. List omKeyInfoList = omMetadataManager.listKeys(volumeNameA, ozoneBucket, - null, prefixKeyA, 100); + null, prefixKeyA, 100).getKeys(); // As in total 100, 50 are marked for delete. It should list only 50 keys. assertEquals(50, omKeyInfoList.size()); @@ -528,7 +528,7 @@ public void testListKeysWithFewDeleteEntriesInCache() throws Exception { for (int i = 0; i < 5; i++) { omKeyInfoList = omMetadataManager.listKeys(volumeNameA, ozoneBucket, - startKey, prefixKeyA, 10); + startKey, prefixKeyA, 10).getKeys(); System.out.println(i); assertEquals(10, omKeyInfoList.size()); @@ -547,7 +547,7 @@ public void testListKeysWithFewDeleteEntriesInCache() throws Exception { // As now we have iterated all 50 buckets, calling next time should // return empty list. omKeyInfoList = omMetadataManager.listKeys(volumeNameA, ozoneBucket, - startKey, prefixKeyA, 10); + startKey, prefixKeyA, 10).getKeys(); assertEquals(omKeyInfoList.size(), 0); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java index a8c126f43bd8..7a17c70ef769 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java @@ -482,7 +482,7 @@ private Set getKeysFromDb(OMMetadataManager omMetadataReader, String startKey = null; while (true) { List omKeyInfoList = omMetadataReader.listKeys(volume, bucket, - startKey, null, 1000); + startKey, null, 1000).getKeys(); if (omKeyInfoList.isEmpty()) { break; } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmMetadataGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmMetadataGenerator.java index 6ce460563fda..106bf9d2ec85 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmMetadataGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmMetadataGenerator.java @@ -396,8 +396,9 @@ private void applyOperation(long counter) throws Exception { threadSeqId = getThreadSequenceId(); startKeyName = getPath(threadSeqId * batchSize); getMetrics().timer(operation.name()).time(() -> { - List keyInfoList = ozoneManagerClient.listKeys( - volumeName, bucketName, startKeyName, "", batchSize); + List keyInfoList = + ozoneManagerClient.listKeys(volumeName, bucketName, startKeyName, + "", batchSize).getKeys(); if (keyInfoList.size() + 1 < batchSize) { throw new NoSuchFileException( "There are not enough files for testing you should use "