diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java index b8e2542bdea9..cc6695dc7d68 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java @@ -38,6 +38,10 @@ public enum ClientVersion implements ComponentVersion { ERASURE_CODING_SUPPORT(2, "This client version has support for Erasure" + " Coding."), + BUCKET_LAYOUT_SUPPORT(3, + "This client version has support for Object Store and File " + + "System Optimized Bucket Layouts."), + FUTURE_VERSION(-1, "Used internally when the server side is older and an" + " unknown client version has arrived from the client."); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java index a9803e03e4a9..8a17777b1101 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.om.helpers; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; /** @@ -88,4 +89,20 @@ public static BucketLayout fromString(String value) { // Added safer `isBlank` check for unit test cases. return StringUtils.isBlank(value) ? LEGACY : BucketLayout.valueOf(value); } + + /** + * Helper method for upgrade scenarios. Throws an exception if a bucket layout + * is not supported on an older client. + * + * @throws OMException if bucket layout is not supported on older clients. + */ + public void validateSupportedOperation() throws OMException { + // Older clients do not support any bucket layout other than LEGACY. + if (!isLegacy()) { + throw new OMException("Client is attempting to modify a bucket which" + + " uses non-LEGACY bucket layout features. Please upgrade the client" + + " to a compatible version before performing this operation.", + OMException.ResultCodes.NOT_SUPPORTED_OPERATION); + } + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java index b27ae730aeee..41017f1d868a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java @@ -18,6 +18,7 @@ import com.google.common.base.Preconditions; import com.google.protobuf.RpcController; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.permission.FsPermission; @@ -446,6 +447,7 @@ FileStatus getStatus() { try { omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() .setClientId(CLIENT_ID.toString()) + .setVersion(ClientVersion.CURRENT_VERSION) .setUserInfo(getUserInfo()) .setRenameKeyRequest(renameKeyRequest) .setCmdType(OzoneManagerProtocolProtos.Type.RenameKey) @@ -510,6 +512,7 @@ private OzoneManagerProtocolProtos.OMRequest getDeleteKeyRequest( omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() .setClientId(CLIENT_ID.toString()) + .setVersion(ClientVersion.CURRENT_VERSION) .setUserInfo(getUserInfo()) .setDeleteKeyRequest(deleteKeyRequest) .setCmdType(OzoneManagerProtocolProtos.Type.DeleteKey) @@ -577,6 +580,7 @@ boolean processKeyPath(List keyPathList) { try { omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() .setClientId(CLIENT_ID.toString()) + .setVersion(ClientVersion.CURRENT_VERSION) .setUserInfo(getUserInfo()) .setDeleteKeysRequest(deleteKeysRequest) .setCmdType(OzoneManagerProtocolProtos.Type.DeleteKeys) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java index 4e234aa69ac7..12dfd4d3ca6c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java @@ -22,10 +22,15 @@ import java.util.Map; import com.google.common.base.Optional; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,6 +47,7 @@ import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos .DeleteBucketRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos @@ -192,4 +198,32 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, } } + /** + * Validates bucket delete requests. + * Handles the cases where an older client attempts to delete a bucket + * a new bucket layout. + * We do not want to allow this to happen, since this would cause the client + * to be able to delete buckets it cannot understand. + * + * @param req - the request to validate + * @param ctx - the validation context + * @return the validated request + * @throws OMException if the request is invalid + */ + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.PRE_PROCESS, + requestType = Type.DeleteBucket + ) + public static OMRequest blockBucketDeleteWithBucketLayoutFromOldClient( + OMRequest req, ValidationContext ctx) throws IOException { + DeleteBucketRequest request = req.getDeleteBucketRequest(); + + if (request.hasBucketName() && request.hasVolumeName()) { + BucketLayout bucketLayout = ctx.getBucketLayout( + request.getVolumeName(), request.getBucketName()); + bucketLayout.validateSupportedOperation(); + } + return req; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java index 62e8bc20764d..729cf5a876c2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java @@ -415,4 +415,35 @@ public static OMRequest disallowCreateDirectoryWithECReplicationConfig( } return req; } + + /** + * Validates directory create requests. + * Handles the cases where an older client attempts to create a directory + * inside a bucket with a non LEGACY bucket layout. + * We do not want an older client modifying a bucket that it cannot + * understand. + * + * @param req - the request to validate + * @param ctx - the validation context + * @return the validated request + * @throws OMException if the request is invalid + */ + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.PRE_PROCESS, + requestType = Type.CreateDirectory + ) + public static OMRequest blockCreateDirectoryWithBucketLayoutFromOldClient( + OMRequest req, ValidationContext ctx) throws IOException { + if (req.getCreateDirectoryRequest().hasKeyArgs()) { + KeyArgs keyArgs = req.getCreateDirectoryRequest().getKeyArgs(); + + if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) { + BucketLayout bucketLayout = ctx.getBucketLayout( + keyArgs.getVolumeName(), keyArgs.getBucketName()); + bucketLayout.validateSupportedOperation(); + } + } + return req; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java index fdfb4119bdd9..634ba81f2af0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java @@ -42,6 +42,8 @@ import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.file.OMFileCreateResponse; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,7 +62,6 @@ import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateFileRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateFileResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; @@ -408,4 +409,36 @@ public static OMRequest disallowCreateFileWithECReplicationConfig( } return req; } + + /** + * Validates file create requests. + * Handles the cases where an older client attempts to create a file + * inside a bucket with a non LEGACY bucket layout. + * We do not want an older client modifying a bucket that it cannot + * understand. + * + * @param req - the request to validate + * @param ctx - the validation context + * @return the validated request + * @throws OMException if the request is invalid + */ + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.PRE_PROCESS, + requestType = Type.CreateFile + ) + public static OMRequest blockCreateFileWithBucketLayoutFromOldClient( + OMRequest req, ValidationContext ctx) throws IOException { + if (req.getCreateFileRequest().hasKeyArgs()) { + + KeyArgs keyArgs = req.getCreateFileRequest().getKeyArgs(); + + if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) { + BucketLayout bucketLayout = ctx.getBucketLayout( + keyArgs.getVolumeName(), keyArgs.getBucketName()); + bucketLayout.validateSupportedOperation(); + } + } + return req; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index a2cd19ab19f0..ecc2bf06c547 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -284,4 +284,33 @@ public static OMRequest disallowAllocateBlockWithECReplicationConfig( } return req; } + + /** + * Validates block allocation requests. + * We do not want to allow older clients to create block allocation requests + * for keys that are present in buckets which use non LEGACY layouts. + * + * @param req - the request to validate + * @param ctx - the validation context + * @return the validated request + * @throws OMException if the request is invalid + */ + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.PRE_PROCESS, + requestType = Type.AllocateBlock + ) + public static OMRequest blockAllocateBlockWithBucketLayoutFromOldClient( + OMRequest req, ValidationContext ctx) throws IOException { + if (req.getAllocateBlockRequest().hasKeyArgs()) { + KeyArgs keyArgs = req.getAllocateBlockRequest().getKeyArgs(); + + if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) { + BucketLayout bucketLayout = ctx.getBucketLayout( + keyArgs.getVolumeName(), keyArgs.getBucketName()); + bucketLayout.validateSupportedOperation(); + } + } + return req; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index 6f9cafe4ec68..ab903416e22d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -340,4 +340,33 @@ public static OMRequest disallowCommitKeyWithECReplicationConfig( } return req; } + + /** + * Validates key commit requests. + * We do not want to allow older clients to commit keys associated with + * buckets which use non LEGACY layouts. + * + * @param req - the request to validate + * @param ctx - the validation context + * @return the validated request + * @throws OMException if the request is invalid + */ + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.PRE_PROCESS, + requestType = Type.CommitKey + ) + public static OMRequest blockCommitKeyWithBucketLayoutFromOldClient( + OMRequest req, ValidationContext ctx) throws IOException { + if (req.getCommitKeyRequest().hasKeyArgs()) { + KeyArgs keyArgs = req.getCommitKeyRequest().getKeyArgs(); + + if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) { + BucketLayout bucketLayout = ctx.getBucketLayout( + keyArgs.getVolumeName(), keyArgs.getBucketName()); + bucketLayout.validateSupportedOperation(); + } + } + return req; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index 906ea5975496..99986a7410cd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -395,4 +395,33 @@ public static OMRequest disallowCreateKeyWithECReplicationConfig( } return req; } + + /** + * Validates key create requests. + * We do not want to allow older clients to create keys in buckets which use + * non LEGACY layouts. + * + * @param req - the request to validate + * @param ctx - the validation context + * @return the validated request + * @throws OMException if the request is invalid + */ + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.PRE_PROCESS, + requestType = Type.CreateKey + ) + public static OMRequest blockCreateKeyWithBucketLayoutFromOldClient( + OMRequest req, ValidationContext ctx) throws IOException { + if (req.getCreateKeyRequest().hasKeyArgs()) { + KeyArgs keyArgs = req.getCreateKeyRequest().getKeyArgs(); + + if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) { + BucketLayout bucketLayout = ctx.getBucketLayout( + keyArgs.getVolumeName(), keyArgs.getBucketName()); + bucketLayout.validateSupportedOperation(); + } + } + return req; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java index d54e4ebddd19..1398c0b70796 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java @@ -26,6 +26,10 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.apache.hadoop.ozone.security.acl.OzoneObj; import org.slf4j.Logger; @@ -42,6 +46,8 @@ import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.OMKeyDeleteResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; @@ -209,4 +215,33 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, return omClientResponse; } + + /** + * Validates key delete requests. + * We do not want to allow older clients to delete keys in buckets which use + * non LEGACY layouts. + * + * @param req - the request to validate + * @param ctx - the validation context + * @return the validated request + * @throws OMException if the request is invalid + */ + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.PRE_PROCESS, + requestType = Type.DeleteKey + ) + public static OMRequest blockDeleteKeyWithBucketLayoutFromOldClient( + OMRequest req, ValidationContext ctx) throws IOException { + if (req.getDeleteKeyRequest().hasKeyArgs()) { + KeyArgs keyArgs = req.getDeleteKeyRequest().getKeyArgs(); + + if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) { + BucketLayout bucketLayout = ctx.getBucketLayout( + keyArgs.getVolumeName(), keyArgs.getBucketName()); + bucketLayout.validateSupportedOperation(); + } + } + return req; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java index 2e25b567c7af..033f62278256 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java @@ -29,6 +29,10 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.apache.hadoop.ozone.security.acl.OzoneObj; import org.slf4j.Logger; @@ -44,6 +48,7 @@ import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.OMKeyRenameResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos .KeyArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; @@ -250,4 +255,34 @@ private Map buildAuditMap( auditMap.put(OzoneConsts.DST_KEY, renameKeyRequest.getToKeyName()); return auditMap; } + + + /** + * Validates rename key requests. + * We do not want to allow older clients to rename keys in buckets which use + * non LEGACY layouts. + * + * @param req - the request to validate + * @param ctx - the validation context + * @return the validated request + * @throws OMException if the request is invalid + */ + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.PRE_PROCESS, + requestType = Type.RenameKey + ) + public static OMRequest blockRenameKeyWithBucketLayoutFromOldClient( + OMRequest req, ValidationContext ctx) throws IOException { + if (req.getRenameKeyRequest().hasKeyArgs()) { + KeyArgs keyArgs = req.getRenameKeyRequest().getKeyArgs(); + + if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) { + BucketLayout bucketLayout = ctx.getBucketLayout( + keyArgs.getVolumeName(), keyArgs.getBucketName()); + bucketLayout.validateSupportedOperation(); + } + } + return req; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index 9c34242f2604..1781612d0b7e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -27,15 +27,22 @@ import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.ResolvedBucket; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.OMKeysDeleteResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeysRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeysResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; @@ -295,4 +302,32 @@ protected static void addDeletedKeys(Map auditMap, auditMap.put(UNDELETED_KEYS_LIST, String.join(",", unDeletedKeys)); } + /** + * Validates delete key requests. + * We do not want to allow older clients to delete keys in buckets which use + * non LEGACY layouts. + * + * @param req - the request to validate + * @param ctx - the validation context + * @return the validated request + * @throws OMException if the request is invalid + */ + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.PRE_PROCESS, + requestType = Type.DeleteKeys + ) + public static OMRequest blockDeleteKeysWithBucketLayoutFromOldClient( + OMRequest req, ValidationContext ctx) throws IOException { + if (req.getDeleteKeysRequest().hasDeleteKeys()) { + DeleteKeyArgs keyArgs = req.getDeleteKeysRequest().getDeleteKeys(); + + if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) { + BucketLayout bucketLayout = ctx.getBucketLayout( + keyArgs.getVolumeName(), keyArgs.getBucketName()); + bucketLayout.validateSupportedOperation(); + } + } + return req; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java index 4427e6d64eac..bd6e98cc9129 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java @@ -28,14 +28,20 @@ import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.ResolvedBucket; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmRenameKeys; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.OMKeysRenameResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeysArgs; @@ -277,4 +283,33 @@ private Map buildAuditMap(Map auditMap, auditMap.put(UNRENAMED_KEYS_MAP, unRenameKeysMap.toString()); return auditMap; } + + /** + * Validates rename keys requests. + * We do not want to allow older clients to rename keys in buckets which use + * non LEGACY layouts. + * + * @param req - the request to validate + * @param ctx - the validation context + * @return the validated request + * @throws OMException if the request is invalid + */ + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.PRE_PROCESS, + requestType = Type.RenameKeys + ) + public static OMRequest blockRenameKeysWithBucketLayoutFromOldClient( + OMRequest req, ValidationContext ctx) throws IOException { + if (req.getRenameKeysRequest().hasRenameKeysArgs()) { + RenameKeysArgs keyArgs = req.getRenameKeysRequest().getRenameKeysArgs(); + + if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) { + BucketLayout bucketLayout = ctx.getBucketLayout( + keyArgs.getVolumeName(), keyArgs.getBucketName()); + bucketLayout.validateSupportedOperation(); + } + } + return req; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java index 4aaec04bf361..9be37ecd5228 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java @@ -298,4 +298,34 @@ protected void logResult(OzoneManager ozoneManager, } return req; } + + + /** + * Validates S3 initiate MPU requests. + * We do not want to allow older clients to initiate MPU to buckets which + * use non LEGACY layouts. + * + * @param req - the request to validate + * @param ctx - the validation context + * @return the validated request + * @throws OMException if the request is invalid + */ + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.PRE_PROCESS, + requestType = Type.InitiateMultiPartUpload + ) + public static OMRequest blockInitiateMPUWithBucketLayoutFromOldClient( + OMRequest req, ValidationContext ctx) throws IOException { + if (req.getInitiateMultiPartUploadRequest().hasKeyArgs()) { + KeyArgs keyArgs = req.getInitiateMultiPartUploadRequest().getKeyArgs(); + + if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) { + BucketLayout bucketLayout = ctx.getBucketLayout( + keyArgs.getVolumeName(), keyArgs.getBucketName()); + bucketLayout.validateSupportedOperation(); + } + } + return req; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java index 0959c7198806..abb3a45a3340 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java @@ -282,4 +282,33 @@ public static OMRequest disallowAbortMultiPartUploadWithECReplicationConfig( } return req; } + + /** + * Validates S3 MPU abort requests. + * We do not want to allow older clients to abort MPU operations in + * buckets which use non LEGACY layouts. + * + * @param req - the request to validate + * @param ctx - the validation context + * @return the validated request + * @throws OMException if the request is invalid + */ + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.PRE_PROCESS, + requestType = Type.AbortMultiPartUpload + ) + public static OMRequest blockMPUAbortWithBucketLayoutFromOldClient( + OMRequest req, ValidationContext ctx) throws IOException { + if (req.getAbortMultiPartUploadRequest().hasKeyArgs()) { + KeyArgs keyArgs = req.getAbortMultiPartUploadRequest().getKeyArgs(); + + if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) { + BucketLayout bucketLayout = ctx.getBucketLayout( + keyArgs.getVolumeName(), keyArgs.getBucketName()); + bucketLayout.validateSupportedOperation(); + } + } + return req; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java index 00b8636226d1..0644e1f27447 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java @@ -357,4 +357,33 @@ public static OMRequest disallowCommitMultiPartUploadWithECReplicationConfig( } return req; } + + /** + * Validates S3 MPU commit part requests. + * We do not want to allow older clients to commit MPU keys to buckets which + * use non LEGACY layouts. + * + * @param req - the request to validate + * @param ctx - the validation context + * @return the validated request + * @throws OMException if the request is invalid + */ + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.PRE_PROCESS, + requestType = Type.CommitMultiPartUpload + ) + public static OMRequest blockMPUCommitWithBucketLayoutFromOldClient( + OMRequest req, ValidationContext ctx) throws IOException { + if (req.getCommitMultiPartUploadRequest().hasKeyArgs()) { + KeyArgs keyArgs = req.getCommitMultiPartUploadRequest().getKeyArgs(); + + if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) { + BucketLayout bucketLayout = ctx.getBucketLayout( + keyArgs.getVolumeName(), keyArgs.getBucketName()); + bucketLayout.validateSupportedOperation(); + } + } + return req; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index 90a51180e884..ba73cde1104e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -596,4 +596,33 @@ private void updateCache(OMMetadataManager omMetadataManager, } return req; } + + /** + * Validates S3 MPU complete requests. + * We do not want to allow older clients to upload MPU keys to buckets which + * use non LEGACY layouts. + * + * @param req - the request to validate + * @param ctx - the validation context + * @return the validated request + * @throws OMException if the request is invalid + */ + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.PRE_PROCESS, + requestType = Type.CompleteMultiPartUpload + ) + public static OMRequest blockMPUCompleteWithBucketLayoutFromOldClient( + OMRequest req, ValidationContext ctx) throws IOException { + if (req.getCompleteMultiPartUploadRequest().hasKeyArgs()) { + KeyArgs keyArgs = req.getCompleteMultiPartUploadRequest().getKeyArgs(); + + if (keyArgs.hasVolumeName() && keyArgs.hasBucketName()) { + BucketLayout bucketLayout = ctx.getBucketLayout( + keyArgs.getVolumeName(), keyArgs.getBucketName()); + bucketLayout.validateSupportedOperation(); + } + } + return req; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java index e2e127a4fa40..feb3b299edfc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java @@ -97,7 +97,7 @@ public OMResponse validateResponse(OMRequest request, OMResponse response) m.getName(), m.getDeclaringClass().getPackage().getName(), m.getDeclaringClass().getSimpleName()); validatedResponse = - (OMResponse) m.invoke(null, request, response, context); + (OMResponse) m.invoke(null, request, validatedResponse, context); } } catch (InvocationTargetException | IllegalAccessException e) { throw new ServiceException(e); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index ecec94592d94..0e882a6f414b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -19,10 +19,13 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import com.google.protobuf.ServiceException; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.UpgradeFinalizationStatus; @@ -44,6 +47,7 @@ import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.ratis.OzoneManagerDoubleBuffer; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.OMClientRequest; @@ -433,6 +437,39 @@ public static OMResponse disallowLookupKeyResponseWithECReplicationConfig( return resp; } + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.POST_PROCESS, + requestType = Type.LookupKey + ) + public static OMResponse disallowLookupKeyWithBucketLayout( + OMRequest req, OMResponse resp, ValidationContext ctx) + throws ServiceException, IOException { + if (!resp.hasLookupKeyResponse()) { + return resp; + } + KeyInfo keyInfo = resp.getLookupKeyResponse().getKeyInfo(); + // If the key is present inside a bucket using a non LEGACY bucket layout, + // then the client needs to be upgraded before proceeding. + if (keyInfo.hasVolumeName() && keyInfo.hasBucketName() && + !ctx.getBucketLayout(keyInfo.getVolumeName(), keyInfo.getBucketName()) + .equals(BucketLayout.LEGACY)) { + resp = resp.toBuilder() + .setStatus(Status.NOT_SUPPORTED_OPERATION) + .setMessage("Key is present inside a bucket with bucket layout " + + "features, which the client can not understand. Please upgrade" + + " the client to a compatible version before trying to read the" + + " key info for " + + req.getLookupKeyRequest().getKeyArgs().getVolumeName() + + "/" + req.getLookupKeyRequest().getKeyArgs().getBucketName() + + "/" + req.getLookupKeyRequest().getKeyArgs().getKeyName() + + ".") + .clearLookupKeyResponse() + .build(); + } + return resp; + } + private ListBucketsResponse listBuckets(ListBucketsRequest request) throws IOException { ListBucketsResponse.Builder resp = @@ -495,6 +532,47 @@ public static OMResponse disallowListKeysResponseWithECReplicationConfig( return resp; } + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.POST_PROCESS, + requestType = Type.ListKeys + ) + public static OMResponse disallowListKeysWithBucketLayout( + OMRequest req, OMResponse resp, ValidationContext ctx) + throws ServiceException, IOException { + if (!resp.hasListKeysResponse()) { + return resp; + } + + // Put volume and bucket pairs into a set to avoid duplicates. + HashSet> volumeBucketSet = new HashSet<>(); + List keys = resp.getListKeysResponse().getKeyInfoList(); + for (KeyInfo key : keys) { + if (key.hasVolumeName() && key.hasBucketName()) { + volumeBucketSet.add( + new ImmutablePair<>(key.getVolumeName(), key.getBucketName())); + } + } + + for (Pair volumeBucket : volumeBucketSet) { + // If any of the buckets have a non legacy layout, then the client is + // not compatible with the response. + if (!ctx.getBucketLayout(volumeBucket.getLeft(), volumeBucket.getRight()) + .isLegacy()) { + resp = resp.toBuilder() + .setStatus(Status.NOT_SUPPORTED_OPERATION) + .setMessage("The list of keys contains keys present inside bucket" + + " with bucket layout features, hence the client is not able " + + "to understand all the keys returned. Please upgrade the" + + " client to get the list of keys.") + .clearListKeysResponse() + .build(); + break; + } + } + return resp; + } + private ListTrashResponse listTrash(ListTrashRequest request, int clientVersion) throws IOException { @@ -545,6 +623,51 @@ public static OMResponse disallowListTrashWithECReplicationConfig( return resp; } + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.POST_PROCESS, + requestType = Type.ListTrash + ) + public static OMResponse disallowListTrashWithBucketLayout( + OMRequest req, OMResponse resp, ValidationContext ctx) + throws ServiceException, IOException { + if (!resp.hasListTrashResponse()) { + return resp; + } + + // Add the volume and bucket pairs to a set to avoid duplicates. + List repeatedKeys = + resp.getListTrashResponse().getDeletedKeysList(); + HashSet> volumeBucketSet = new HashSet<>(); + + for (RepeatedKeyInfo repeatedKey : repeatedKeys) { + for (KeyInfo key : repeatedKey.getKeyInfoList()) { + if (key.hasVolumeName() && key.hasBucketName()) { + volumeBucketSet.add( + new ImmutablePair<>(key.getVolumeName(), key.getBucketName())); + } + } + } + + // If any of the keys is present inside a bucket using a non LEGACY bucket + // layout, then the client needs to be upgraded before proceeding. + for (Pair volumeBucket : volumeBucketSet) { + if (!ctx.getBucketLayout(volumeBucket.getLeft(), volumeBucket.getRight()) + .isLegacy()) { + resp = resp.toBuilder() + .setStatus(Status.NOT_SUPPORTED_OPERATION) + .setMessage("The list of keys contains keys present in buckets " + + " using bucket layout features, hence the client is not able to" + + " understand all the keys returned. Please upgrade the" + + " client to get the list of keys.") + .clearListTrashResponse() + .build(); + break; + } + } + return resp; + } + private ServiceListResponse getServiceList(ServiceListRequest request) throws IOException { ServiceListResponse.Builder resp = ServiceListResponse.newBuilder(); @@ -701,6 +824,40 @@ public static OMResponse disallowGetFileStatusWithECReplicationConfig( return resp; } + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.POST_PROCESS, + requestType = Type.GetFileStatus + ) + public static OMResponse disallowGetFileStatusWithBucketLayout( + OMRequest req, OMResponse resp, ValidationContext ctx) + throws ServiceException, IOException { + if (!resp.hasGetFileStatusResponse()) { + return resp; + } + + // If the File is present inside a bucket with non LEGACY layout, + // then the client should be upgraded before proceeding. + KeyInfo keyInfo = resp.getGetFileStatusResponse().getStatus().getKeyInfo(); + if (keyInfo.hasVolumeName() && keyInfo.hasBucketName() && + !ctx.getBucketLayout(keyInfo.getVolumeName(), keyInfo.getBucketName()) + .isLegacy()) { + resp = resp.toBuilder() + .setStatus(Status.NOT_SUPPORTED_OPERATION) + .setMessage("Key is present in a bucket using bucket layout features" + + " which the client can not understand." + + " Please upgrade the client before trying to read the key info" + + " for " + + req.getGetFileStatusRequest().getKeyArgs().getVolumeName() + + "/" + req.getGetFileStatusRequest().getKeyArgs().getBucketName() + + "/" + req.getGetFileStatusRequest().getKeyArgs().getKeyName() + + ".") + .clearGetFileStatusResponse() + .build(); + } + return resp; + } + private LookupFileResponse lookupFile(LookupFileRequest request, int clientVersion) throws IOException { KeyArgs keyArgs = request.getKeyArgs(); @@ -745,6 +902,37 @@ public static OMResponse disallowLookupFileWithECReplicationConfig( return resp; } + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.POST_PROCESS, + requestType = Type.LookupFile + ) + public static OMResponse disallowLookupFileWithBucketLayout( + OMRequest req, OMResponse resp, ValidationContext ctx) + throws ServiceException, IOException { + if (!resp.hasLookupFileResponse()) { + return resp; + } + KeyInfo keyInfo = resp.getLookupFileResponse().getKeyInfo(); + if (keyInfo.hasVolumeName() && keyInfo.hasBucketName() && + !ctx.getBucketLayout(keyInfo.getVolumeName(), keyInfo.getBucketName()) + .equals(BucketLayout.LEGACY)) { + resp = resp.toBuilder() + .setStatus(Status.NOT_SUPPORTED_OPERATION) + .setMessage("File is present inside a bucket with bucket layout " + + "features, which the client can not understand. Please upgrade" + + " the client to a compatible version before trying to read the" + + " key info for " + + req.getLookupFileRequest().getKeyArgs().getVolumeName() + + "/" + req.getLookupFileRequest().getKeyArgs().getBucketName() + + "/" + req.getLookupFileRequest().getKeyArgs().getKeyName() + + ".") + .clearLookupFileResponse() + .build(); + } + return resp; + } + private ListStatusResponse listStatus( ListStatusRequest request, int clientVersion) throws IOException { KeyArgs keyArgs = request.getKeyArgs(); @@ -796,6 +984,52 @@ public static OMResponse disallowListStatusResponseWithECReplicationConfig( return resp; } + @RequestFeatureValidator( + conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + processingPhase = RequestProcessingPhase.POST_PROCESS, + requestType = Type.ListStatus + ) + public static OMResponse disallowListStatusResponseWithBucketLayout( + OMRequest req, OMResponse resp, ValidationContext ctx) + throws ServiceException, IOException { + if (!resp.hasListStatusResponse()) { + return resp; + } + + // Add the volume and bucket pairs to a set to avoid duplicate entries. + List statuses = + resp.getListStatusResponse().getStatusesList(); + HashSet> volumeBucketSet = new HashSet<>(); + + for (OzoneFileStatusProto status : statuses) { + KeyInfo keyInfo = status.getKeyInfo(); + if (keyInfo.hasVolumeName() && keyInfo.hasBucketName()) { + volumeBucketSet.add( + new ImmutablePair<>(keyInfo.getVolumeName(), + keyInfo.getBucketName())); + } + } + + // If any of the keys are present in a bucket with a non LEGACY bucket + // layout, then the client needs to be upgraded before proceeding. + for (Pair volumeBucket : volumeBucketSet) { + if (!ctx.getBucketLayout(volumeBucket.getLeft(), + volumeBucket.getRight()).isLegacy()) { + resp = resp.toBuilder() + .setStatus(Status.NOT_SUPPORTED_OPERATION) + .setMessage("The list of keys is present in a bucket using bucket" + + " layout features, hence the client is not able to" + + " represent all the keys returned." + + " Please upgrade the client to get the list of keys.") + .clearListStatusResponse() + .build(); + break; + } + } + + return resp; + } + private FinalizeUpgradeProgressResponse reportUpgradeProgress( FinalizeUpgradeProgressRequest request) throws IOException { String upgradeClientId = request.getUpgradeClientId();