HDDS-11132. Update the OLDER_CLIENT_REQUEST validations so that validations don't fail

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java

@@ -290,11 +290,13 @@ public void close() throws IOException {
     transport.close();
   }
 
+
   /**
    * Returns a OMRequest builder with specified type.
    * @param cmdType type of the request
    */
-  private OMRequest.Builder createOMRequest(Type cmdType) {
+  @VisibleForTesting
+  public OMRequest.Builder createOMRequest(Type cmdType) {
     return OMRequest.newBuilder()
         .setCmdType(cmdType)
         .setVersion(ClientVersion.CURRENT_VERSION)
@@ -307,7 +309,8 @@ private OMRequest.Builder createOMRequest(Type cmdType) {
    * @return response from OM
    * @throws IOException thrown if any Protobuf service exception occurs
    */
-  private OMResponse submitRequest(OMRequest omRequest)
+  @VisibleForTesting
+  public OMResponse submitRequest(OMRequest omRequest)
       throws IOException {
     OMRequest.Builder  builder = OMRequest.newBuilder(omRequest);
     // Insert S3 Authentication information for each request.

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketLayoutWithOlderClient.java → hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketLayoutWithAllClients.java

@@ -16,36 +16,47 @@
  */
 package org.apache.hadoop.ozone.om;
 
+import com.google.protobuf.ServiceException;
 import org.apache.hadoop.hdds.utils.IOUtils;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.ClientVersion;
 import org.apache.hadoop.ozone.TestDataUtil;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.rpc.RpcClient;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
 import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.UUID;
+import java.util.function.Function;
+import java.util.stream.Stream;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 
 /**
- * Tests to verify bucket ops with older version client.
+ * Tests to verify bucket ops with different client versions.
  */
 @Timeout(1200)
-public class TestBucketLayoutWithOlderClient {
+public class TestBucketLayoutWithAllClients {
 
   private static MiniOzoneCluster cluster = null;
   private static OzoneConfiguration conf;
   private static OzoneClient client;
+  private static OzoneManagerProtocolClientSideTranslatorPB ozoneManagerClient;
 
   /**
    * Create a MiniDFSCluster for testing.
@@ -61,6 +72,21 @@ public static void init() throws Exception {
     cluster = MiniOzoneCluster.newBuilder(conf).build();
     cluster.waitForClusterToBeReady();
     client = cluster.newClient();
+    ozoneManagerClient =
+        (OzoneManagerProtocolClientSideTranslatorPB)
+            ((RpcClient) client.getObjectStore().getClientProxy()).getOzoneManagerClient();
+  }
+
+  private static Stream<Arguments> allClientVersions() {
+    return Arrays.stream(ClientVersion.values()).flatMap(
+        clientVersion -> Stream.of(Arguments.of(clientVersion, BucketLayout.FILE_SYSTEM_OPTIMIZED),
+            Arguments.of(clientVersion, BucketLayout.OBJECT_STORE)));
+  }
+
+  private OzoneManagerProtocolProtos.OMRequest.Builder getOmRequest(OzoneManagerProtocolProtos.Type type,
+                                                                    ClientVersion clientVersion) {
+    return ozoneManagerClient.createOMRequest(type)
+        .setVersion(clientVersion.toProtoValue());
   }
 
   @Test
@@ -109,6 +135,57 @@ public void testCreateBucketWithOlderClient() throws Exception {
     assertEquals(BucketLayout.LEGACY, bucketInfo.getBucketLayout());
   }
 
+  private OzoneManagerProtocolProtos.KeyArgs getKeyArgs(String volume, String bucket) {
+    return OzoneManagerProtocolProtos.KeyArgs.newBuilder()
+            .setVolumeName(volume)
+            .setBucketName(bucket).setKeyName("a/b/c/key")
+            .build();
+  }
+
+  private OzoneManagerProtocolProtos.Status createKey(String volume, String bucket, ClientVersion clientVersion,
+                            OzoneManagerProtocolProtos.Status expectedStatus) throws IOException {
+    OzoneManagerProtocolProtos.CreateFileRequest createFileRequest =
+        OzoneManagerProtocolProtos.CreateFileRequest.newBuilder()
+        .setKeyArgs(getKeyArgs(volume, bucket))
+        .setIsOverwrite(true)
+        .setIsRecursive(true).build();
+    OzoneManagerProtocolProtos.OMRequest request = getOmRequest(OzoneManagerProtocolProtos.Type.CreateFile,
+        clientVersion)
+        .setCreateFileRequest(createFileRequest).build();
+    OzoneManagerProtocolProtos.OMResponse
+    omResponse = ozoneManagerClient.submitRequest(request);
+    return expectedStatus;
+  }
+
+  private void performOpWithLatestClientVersion(ClientVersion clientVersion,
+                                                Function<ClientVersion, OzoneManagerProtocolProtos.Status> operation,
+                                                OzoneManagerProtocolProtos.Status expectedStatusWithClientVersion) {
+    Assertions.assertEquals(expectedStatusWithClientVersion, operation.apply(clientVersion));
+    if (!operation.apply(clientVersion).equals(OzoneManagerProtocolProtos.Status.OK)) {
+      operation.apply(ClientVersion.CURRENT);
+    }
+
+  }
+  @ParameterizedTest
+  @MethodSource("allClientVersions")
+  public void testBucketOperationsOnNonLegacyBucket(ClientVersion version, BucketLayout bucketLayout)
+      throws IOException, ServiceException {
+    String suffix = version.toProtoValue() + "-" +
+        bucketLayout.toString().toLowerCase().replaceAll("_","-");
+    String volume = "volume-" + suffix;
+    String bucket = "bucket-" + suffix;
+    OzoneBucket ozoneBucket = TestDataUtil.createVolumeAndBucket(client, volume, bucket, bucketLayout);
+    Assertions.assertEquals(bucketLayout,
+        client.getObjectStore().getVolume(volume).getBucket(bucket).getBucketLayout());
+    // Create key
+    OzoneManagerProtocolProtos.Status expectedStatus =
+        version.toProtoValue() >= ClientVersion.BUCKET_LAYOUT_SUPPORT.toProtoValue() ?
+            OzoneManagerProtocolProtos.Status.OK : OzoneManagerProtocolProtos.Status.NOT_SUPPORTED_OPERATION;
+    performOpWithLatestClientVersion(createKey(volume, bucket, version, expectedStatus));
+
+
+  }
+
   /**
    * Shutdown MiniDFSCluster.
    */

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java

@@ -23,6 +23,7 @@
 import java.util.Iterator;
 import java.util.Map;
 
+import org.apache.hadoop.ozone.ClientVersion;
 import org.apache.ratis.server.protocol.TermIndex;
 import org.apache.hadoop.hdds.utils.db.Table;
 import org.apache.hadoop.hdds.utils.db.TableIterator;
@@ -287,6 +288,10 @@ private boolean bucketContainsSnapshotInCache(
   )
   public static OMRequest blockBucketDeleteWithBucketLayoutFromOldClient(
       OMRequest req, ValidationContext ctx) throws IOException {
+    if (ClientVersion.fromProtoValue(req.getVersion())
+        .compareTo(ClientVersion.BUCKET_LAYOUT_SUPPORT) >= 0) {
+      return req;
+    }
     DeleteBucketRequest request = req.getDeleteBucketRequest();
 
     if (request.hasBucketName() && request.hasVolumeName()) {

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java

@@ -31,6 +31,7 @@
 import org.apache.hadoop.hdds.client.ECReplicationConfig;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.ozone.ClientVersion;
 import org.apache.ratis.server.protocol.TermIndex;
 import org.apache.hadoop.ozone.OmUtils;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
@@ -438,6 +439,10 @@ public static OMRequest disallowCreateDirectoryWithECReplicationConfig(
   )
   public static OMRequest blockCreateDirectoryWithBucketLayoutFromOldClient(
       OMRequest req, ValidationContext ctx) throws IOException {
+    if (ClientVersion.fromProtoValue(req.getVersion())
+        .compareTo(ClientVersion.BUCKET_LAYOUT_SUPPORT) >= 0) {
+      return req;
+    }
     if (req.getCreateDirectoryRequest().hasKeyArgs()) {
       KeyArgs keyArgs = req.getCreateDirectoryRequest().getKeyArgs();
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java

@@ -29,6 +29,7 @@
 import com.google.common.base.Preconditions;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.ozone.ClientVersion;
 import org.apache.ratis.server.protocol.TermIndex;
 import org.apache.hadoop.ozone.OmUtils;
 import org.apache.hadoop.ozone.OzoneConsts;
@@ -431,6 +432,10 @@ public static OMRequest disallowCreateFileWithECReplicationConfig(
   )
   public static OMRequest blockCreateFileWithBucketLayoutFromOldClient(
       OMRequest req, ValidationContext ctx) throws IOException {
+    if (ClientVersion.fromProtoValue(req.getVersion())
+        .compareTo(ClientVersion.BUCKET_LAYOUT_SUPPORT) >= 0) {
+      return req;
+    }
     if (req.getCreateFileRequest().hasKeyArgs()) {
 
       KeyArgs keyArgs = req.getCreateFileRequest().getKeyArgs();

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java

@@ -26,6 +26,7 @@
 
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.ozone.ClientVersion;
 import org.apache.ratis.server.protocol.TermIndex;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
@@ -304,6 +305,11 @@ public static OMRequest disallowAllocateBlockWithECReplicationConfig(
   )
   public static OMRequest blockAllocateBlockWithBucketLayoutFromOldClient(
       OMRequest req, ValidationContext ctx) throws IOException {
+    if (ClientVersion.fromProtoValue(req.getVersion())
+        .compareTo(ClientVersion.BUCKET_LAYOUT_SUPPORT) >= 0) {
+      return req;
+    }
+
     if (req.getAllocateBlockRequest().hasKeyArgs()) {
       KeyArgs keyArgs = req.getAllocateBlockRequest().getKeyArgs();
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java

@@ -28,6 +28,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.ozone.ClientVersion;
 import org.apache.ratis.server.protocol.TermIndex;
 import org.apache.hadoop.ozone.OmUtils;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -469,6 +470,10 @@ public static OMRequest disallowCommitKeyWithECReplicationConfig(
   )
   public static OMRequest blockCommitKeyWithBucketLayoutFromOldClient(
       OMRequest req, ValidationContext ctx) throws IOException {
+    if (ClientVersion.fromProtoValue(req.getVersion())
+        .compareTo(ClientVersion.BUCKET_LAYOUT_SUPPORT) >= 0) {
+      return req;
+    }
     if (req.getCommitKeyRequest().hasKeyArgs()) {
       KeyArgs keyArgs = req.getCommitKeyRequest().getKeyArgs();
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java

@@ -28,6 +28,7 @@
 
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.ozone.ClientVersion;
 import org.apache.ratis.server.protocol.TermIndex;
 import org.apache.hadoop.ozone.OmUtils;
 import org.apache.hadoop.ozone.om.OMConfigKeys;
@@ -430,6 +431,10 @@ public static OMRequest disallowCreateKeyWithECReplicationConfig(
   )
   public static OMRequest blockCreateKeyWithBucketLayoutFromOldClient(
       OMRequest req, ValidationContext ctx) throws IOException {
+    if (ClientVersion.fromProtoValue(req.getVersion())
+        .compareTo(ClientVersion.BUCKET_LAYOUT_SUPPORT) >= 0) {
+      return req;
+    }
     if (req.getCreateKeyRequest().hasKeyArgs()) {
       KeyArgs keyArgs = req.getCreateKeyRequest().getKeyArgs();
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java

@@ -23,6 +23,7 @@
 import java.util.Map;
 
 import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.ozone.ClientVersion;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.ratis.server.protocol.TermIndex;
 import org.apache.hadoop.ozone.OmUtils;
@@ -239,6 +240,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn
   )
   public static OMRequest blockDeleteKeyWithBucketLayoutFromOldClient(
       OMRequest req, ValidationContext ctx) throws IOException {
+    if (ClientVersion.fromProtoValue(req.getVersion())
+        .compareTo(ClientVersion.BUCKET_LAYOUT_SUPPORT) >= 0) {
+      return req;
+    }
     if (req.getDeleteKeyRequest().hasKeyArgs()) {
       KeyArgs keyArgs = req.getDeleteKeyRequest().getKeyArgs();
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java

@@ -23,6 +23,7 @@
 import java.util.Map;
 
 import com.google.common.base.Preconditions;
+import org.apache.hadoop.ozone.ClientVersion;
 import org.apache.ratis.server.protocol.TermIndex;
 import org.apache.hadoop.ozone.OmUtils;
 import org.apache.hadoop.ozone.OzoneConsts;
@@ -276,6 +277,10 @@ private Map<String, String> buildAuditMap(
   )
   public static OMRequest blockRenameKeyWithBucketLayoutFromOldClient(
       OMRequest req, ValidationContext ctx) throws IOException {
+    if (ClientVersion.fromProtoValue(req.getVersion())
+        .compareTo(ClientVersion.BUCKET_LAYOUT_SUPPORT) >= 0) {
+      return req;
+    }
     if (req.getRenameKeyRequest().hasKeyArgs()) {
       KeyArgs keyArgs = req.getRenameKeyRequest().getKeyArgs();
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java

@@ -20,6 +20,7 @@
 
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.ozone.ClientVersion;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.ratis.server.protocol.TermIndex;
 import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
@@ -357,6 +358,10 @@ protected static void addDeletedKeys(Map<String, String> auditMap,
   )
   public static OMRequest blockDeleteKeysWithBucketLayoutFromOldClient(
       OMRequest req, ValidationContext ctx) throws IOException {
+    if (ClientVersion.fromProtoValue(req.getVersion())
+        .compareTo(ClientVersion.BUCKET_LAYOUT_SUPPORT) >= 0) {
+      return req;
+    }
     if (req.getDeleteKeysRequest().hasDeleteKeys()) {
       DeleteKeyArgs keyArgs = req.getDeleteKeysRequest().getDeleteKeys();
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.ozone.om.request.key;
 
 import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.ozone.ClientVersion;
 import org.apache.ratis.server.protocol.TermIndex;
 import org.apache.hadoop.hdds.utils.db.Table;
 import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
@@ -303,6 +304,10 @@ private Map<String, String> buildAuditMap(Map<String, String> auditMap,
   )
   public static OMRequest blockRenameKeysWithBucketLayoutFromOldClient(
       OMRequest req, ValidationContext ctx) throws IOException {
+    if (ClientVersion.fromProtoValue(req.getVersion())
+        .compareTo(ClientVersion.BUCKET_LAYOUT_SUPPORT) >= 0) {
+      return req;
+    }
     if (req.getRenameKeysRequest().hasRenameKeysArgs()) {
       RenameKeysArgs keyArgs = req.getRenameKeysRequest().getRenameKeysArgs();
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java

@@ -23,6 +23,7 @@
 import java.util.Map;
 
 import com.google.common.collect.Lists;
+import org.apache.hadoop.ozone.ClientVersion;
 import org.apache.ratis.server.protocol.TermIndex;
 import org.apache.hadoop.ozone.OzoneAcl;
 import org.apache.hadoop.ozone.OzoneConsts;
@@ -171,6 +172,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn
   )
   public static OMRequest blockAddAclWithBucketLayoutFromOldClient(
       OMRequest req, ValidationContext ctx) throws IOException {
+    if (ClientVersion.fromProtoValue(req.getVersion())
+        .compareTo(ClientVersion.BUCKET_LAYOUT_SUPPORT) >= 0) {
+      return req;
+    }
     if (req.getAddAclRequest().hasObj()) {
       OzoneObj obj = OzoneObjInfo.fromProtobuf(req.getAddAclRequest().getObj());
       String path = obj.getPath();

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java

@@ -23,6 +23,7 @@
 import java.util.Map;
 
 import com.google.common.collect.Lists;
+import org.apache.hadoop.ozone.ClientVersion;
 import org.apache.ratis.server.protocol.TermIndex;
 import org.apache.hadoop.ozone.OzoneAcl;
 import org.apache.hadoop.ozone.OzoneConsts;
@@ -172,6 +173,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn
   )
   public static OMRequest blockRemoveAclWithBucketLayoutFromOldClient(
       OMRequest req, ValidationContext ctx) throws IOException {
+    if (ClientVersion.fromProtoValue(req.getVersion())
+        .compareTo(ClientVersion.BUCKET_LAYOUT_SUPPORT) >= 0) {
+      return req;
+    }
     if (req.getRemoveAclRequest().hasObj()) {
       OzoneObj obj =
           OzoneObjInfo.fromProtobuf(req.getRemoveAclRequest().getObj());