HDDS-3903. OzoneRpcClient support batch rename keys.

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -293,6 +293,8 @@ private OzoneConsts() {
   public static final String MAX_PARTS = "maxParts";
   public static final String S3_BUCKET = "s3Bucket";
   public static final String S3_GETSECRET_USER = "S3GetSecretUser";
+  public static final String RENAMED_KEYS_MAP = "renamedKeysMap";
+  public static final String UNRENAMED_KEYS_MAP = "unRenamedKeysMap";
   public static final String MULTIPART_UPLOAD_PART_NUMBER = "partNumber";
   public static final String MULTIPART_UPLOAD_PART_NAME = "partName";
   public static final String BUCKET_ENCRYPTION_KEY = "bucketEncryptionKey";

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java

@@ -472,6 +472,16 @@ public void renameKey(String fromKeyName, String toKeyName)
     proxy.renameKey(volumeName, name, fromKeyName, toKeyName);
   }
 
+  /**
+   * Rename the key by keyMap, The key is fromKeyName and value is toKeyName.
+   * @param keyMap The key is original key name nad value is new key name.
+   * @throws IOException
+   */
+  public void renameKeys(Map<String, String> keyMap)
+      throws IOException {
+    proxy.renameKeys(volumeName, name, keyMap);
+  }
+
   /**
    * Initiate multipart upload for a specified key.
    * @param keyName

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java

@@ -314,7 +314,17 @@ void deleteKeys(String volumeName, String bucketName,
    * @throws IOException
    */
   void renameKey(String volumeName, String bucketName, String fromKeyName,
-      String toKeyName) throws IOException;
+                 String toKeyName) throws IOException;
+
+  /**
+   * Renames existing keys within a bucket.
+   * @param volumeName Name of the Volume
+   * @param bucketName Name of the Bucket
+   * @param keyMap The key is original key name nad value is new key name.
+   * @throws IOException
+   */
+  void renameKeys(String volumeName, String bucketName,
+                  Map<String, String> keyMap) throws IOException;
 
   /**
    * Returns list of Keys in {Volume/Bucket} that matches the keyPrefix,

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -27,6 +27,7 @@
 import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
@@ -760,6 +761,25 @@ public void renameKey(String volumeName, String bucketName,
     ozoneManagerClient.renameKey(keyArgs, toKeyName);
   }
 
+  @Override
+  public void renameKeys(String volumeName, String bucketName,
+                         Map<String, String> keyMap) throws IOException {
+    verifyVolumeName(volumeName);
+    verifyBucketName(bucketName);
+    HddsClientUtils.checkNotNull(keyMap);
+    Map <OmKeyArgs, String> keyArgsMap = new HashMap<>();
+    for (Map.Entry< String, String > entry : keyMap.entrySet()) {
+      OmKeyArgs keyArgs = new OmKeyArgs.Builder()
+          .setVolumeName(volumeName)
+          .setBucketName(bucketName)
+          .setKeyName(entry.getKey())
+          .build();
+      keyArgsMap.put(keyArgs, entry.getValue());
+    }
+    ozoneManagerClient.renameKeys(keyArgsMap);
+  }
+
+
   @Override
   public List<OzoneKey> listKeys(String volumeName, String bucketName,
                                  String keyPrefix, String prevKey,

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java

@@ -246,6 +246,7 @@ public static boolean isReadOnly(
     case DeleteBucket:
     case CreateKey:
     case RenameKey:
+    case RenameKeys:
     case DeleteKey:
     case DeleteKeys:
     case CommitKey:

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java

@@ -32,6 +32,7 @@ public enum OMAction implements AuditAction {
   DELETE_BUCKET,
   DELETE_KEY,
   RENAME_KEY,
+  RENAME_KEYS,
   SET_OWNER,
   SET_QUOTA,
   UPDATE_VOLUME,

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java

@@ -227,6 +227,9 @@ public enum ResultCodes {
 
     DETECTED_LOOP_IN_BUCKET_LINKS,
 
-    NOT_SUPPORTED_OPERATION
+    NOT_SUPPORTED_OPERATION,
+
+    PARTIAL_RENAME
+
   }
 }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java

@@ -21,6 +21,7 @@
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
 import org.apache.hadoop.ozone.OzoneAcl;
@@ -217,6 +218,13 @@ OmKeyLocationInfo allocateBlock(OmKeyArgs args, long clientID,
    */
   void renameKey(OmKeyArgs args, String toKeyName) throws IOException;
 
+  /**
+   * Rename existing keys within a bucket.
+   * @param keyMap The key is original key OmKeyArgs and value is new key name.
+   * @throws IOException
+   */
+  void renameKeys(Map<OmKeyArgs, String> keyMap) throws IOException;
+
   /**
    * Deletes an existing key.
    *

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java

@@ -21,6 +21,7 @@
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 import org.apache.hadoop.hdds.annotation.InterfaceAudience;
@@ -122,6 +123,7 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RemoveAclRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RemoveAclResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeyRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeysRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenewDelegationTokenResponseProto;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListResponse;
@@ -675,6 +677,30 @@ public OmKeyInfo lookupKey(OmKeyArgs args) throws IOException {
     return OmKeyInfo.getFromProtobuf(resp.getKeyInfo());
   }
 
+  @Override
+  public void renameKeys(Map<OmKeyArgs, String> keyMap) throws IOException {
+    RenameKeysRequest.Builder reqKeys = RenameKeysRequest.newBuilder();
+    List<RenameKeyRequest> renameKeyRequestList  = new ArrayList<>();
+    for (Map.Entry< OmKeyArgs, String> entry : keyMap.entrySet()) {
+      RenameKeyRequest.Builder reqKey = RenameKeyRequest.newBuilder();
+      OmKeyArgs args = entry.getKey();
+      KeyArgs keyArgs = KeyArgs.newBuilder()
+          .setVolumeName(args.getVolumeName())
+          .setBucketName(args.getBucketName())
+          .setKeyName(args.getKeyName()).build();
+      reqKey.setKeyArgs(keyArgs);
+      reqKey.setToKeyName(entry.getValue());
+      renameKeyRequestList.add(reqKey.build());
+    }
+    reqKeys.addAllRenameKeyRequest(renameKeyRequestList);
+
+    OMRequest omRequest = createOMRequest(Type.RenameKeys)
+        .setRenameKeysRequest(reqKeys)
+        .build();
+
+    handleError(submitRequest(omRequest));
+  }
+
   @Override
   public void renameKey(OmKeyArgs args, String toKeyName) throws IOException {
     RenameKeyRequest.Builder req = RenameKeyRequest.newBuilder();

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java

@@ -110,6 +110,8 @@
 import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
 import static org.apache.hadoop.ozone.OzoneAcl.AclScope.DEFAULT;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PARTIAL_RENAME;
 import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.GROUP;
 import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.USER;
 import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.READ;
@@ -121,6 +123,7 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+
 import org.junit.Test;
 
 /**
@@ -1202,7 +1205,7 @@ public void testDeleteKey()
     Assert.assertEquals(keyName, key.getName());
     bucket.deleteKey(keyName);
 
-    OzoneTestUtils.expectOmException(ResultCodes.KEY_NOT_FOUND,
+    OzoneTestUtils.expectOmException(KEY_NOT_FOUND,
         () -> bucket.getKey(keyName));
   }
 
@@ -1217,13 +1220,7 @@ public void testRenameKey()
     OzoneVolume volume = store.getVolume(volumeName);
     volume.createBucket(bucketName);
     OzoneBucket bucket = volume.getBucket(bucketName);
-    OzoneOutputStream out = bucket.createKey(fromKeyName,
-        value.getBytes().length, STAND_ALONE,
-        ONE, new HashMap<>());
-    out.write(value.getBytes());
-    out.close();
-    OzoneKey key = bucket.getKey(fromKeyName);
-    Assert.assertEquals(fromKeyName, key.getName());
+    createTestKey(bucket, fromKeyName, value);
 
     // Rename to empty string should fail.
     OMException oe = null;
@@ -1244,12 +1241,79 @@ public void testRenameKey()
     } catch (OMException e) {
       oe = e;
     }
-    Assert.assertEquals(ResultCodes.KEY_NOT_FOUND, oe.getResult());
+    Assert.assertEquals(KEY_NOT_FOUND, oe.getResult());
 
-    key = bucket.getKey(toKeyName);
+    OzoneKey key = bucket.getKey(toKeyName);
     Assert.assertEquals(toKeyName, key.getName());
   }
 
+  @Test
+  public void testKeysRename() throws Exception {
+    String volumeName = UUID.randomUUID().toString();
+    String bucketName = UUID.randomUUID().toString();
+    String keyName1 = "dir/file1";
+    String keyName2 = "dir/file2";
+
+    String newKeyName1 = "dir/key1";
+    String newKeyName2 = "dir/key2";
+
+    String value = "sample value";
+    store.createVolume(volumeName);
+    OzoneVolume volume = store.getVolume(volumeName);
+    volume.createBucket(bucketName);
+    OzoneBucket bucket = volume.getBucket(bucketName);
+    createTestKey(bucket, keyName1, value);
+    createTestKey(bucket, keyName2, value);
+
+    Map<String, String> keyMap = new HashMap();
+    keyMap.put(keyName1, newKeyName1);
+    keyMap.put(keyName2, newKeyName2);
+    bucket.renameKeys(keyMap);
+
+    // new key should exist
+    Assert.assertEquals(newKeyName1, bucket.getKey(newKeyName1).getName());
+    Assert.assertEquals(newKeyName2, bucket.getKey(newKeyName2).getName());
+
+    // old key should not exist
+    assertKeyRenamedEx(bucket, keyName1);
+    assertKeyRenamedEx(bucket, keyName2);
+  }
+
+  @Test
+  public void testKeysRenameFail() throws Exception {
+    String volumeName = UUID.randomUUID().toString();
+    String bucketName = UUID.randomUUID().toString();
+    String keyName1 = "dir/file1";
+    String keyName2 = "dir/file2";
+
+    String newKeyName1 = "dir/key1";
+    String newKeyName2 = "dir/key2";
+
+    String value = "sample value";
+    store.createVolume(volumeName);
+    OzoneVolume volume = store.getVolume(volumeName);
+    volume.createBucket(bucketName);
+    OzoneBucket bucket = volume.getBucket(bucketName);
+
+    // Create only keyName1 to test the partial failure of renameKeys.
+    createTestKey(bucket, keyName1, value);
+
+    Map<String, String> keyMap = new HashMap();
+    keyMap.put(keyName1, newKeyName1);
+    keyMap.put(keyName2, newKeyName2);
+
+    try {
+      bucket.renameKeys(keyMap);
+    } catch (OMException ex) {
+      Assert.assertEquals(PARTIAL_RENAME, ex.getResult());
+    }
+
+    // newKeyName1 should exist
+    Assert.assertEquals(newKeyName1, bucket.getKey(newKeyName1).getName());
+    // newKeyName2 should not exist
+    assertKeyRenamedEx(bucket, keyName2);
+  }
+
   @Test
   public void testListVolume() throws IOException {
     String volBase = "vol-" + RandomStringUtils.randomNumeric(3);
@@ -2685,6 +2749,28 @@ private void completeMultipartUpload(OzoneBucket bucket, String keyName,
     Assert.assertNotNull(omMultipartUploadCompleteInfo.getHash());
   }
 
+  private void createTestKey(OzoneBucket bucket, String keyName,
+                             String keyValue) throws IOException {
+    OzoneOutputStream out = bucket.createKey(keyName,
+        keyValue.getBytes().length, STAND_ALONE,
+        ONE, new HashMap<>());
+    out.write(keyValue.getBytes());
+    out.close();
+    OzoneKey key = bucket.getKey(keyName);
+    Assert.assertEquals(keyName, key.getName());
+  }
+
+  private void assertKeyRenamedEx(OzoneBucket bucket, String keyName)
+      throws Exception {
+    OMException oe = null;
+    try {
+      bucket.getKey(keyName);
+    } catch (OMException e) {
+      oe = e;
+    }
+    Assert.assertEquals(KEY_NOT_FOUND, oe.getResult());
+  }
+
   /**
    * Tests GDPR encryption/decryption.
    * 1. Create GDPR Enabled bucket.

hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto

@@ -60,6 +60,7 @@ enum Type {
   CommitKey = 36;
   AllocateBlock = 37;
   DeleteKeys = 38;
+  RenameKeys = 39;
 
   InitiateMultiPartUpload = 45;
   CommitMultiPartUpload = 46;
@@ -126,6 +127,7 @@ message OMRequest {
   optional CommitKeyRequest                 commitKeyRequest               = 36;
   optional AllocateBlockRequest             allocateBlockRequest           = 37;
   optional DeleteKeysRequest                deleteKeysRequest              = 38;
+  optional RenameKeysRequest                renameKeysRequest              = 39;
 
   optional MultipartInfoInitiateRequest     initiateMultiPartUploadRequest = 45;
   optional MultipartCommitUploadPartRequest commitMultiPartUploadRequest   = 46;
@@ -198,6 +200,7 @@ message OMResponse {
   optional CommitKeyResponse                 commitKeyResponse             = 36;
   optional AllocateBlockResponse             allocateBlockResponse         = 37;
   optional DeleteKeysResponse                deleteKeysResponse            = 38;
+  optional RenameKeysResponse                renameKeysResponse            = 39;
 
   optional MultipartInfoInitiateResponse   initiateMultiPartUploadResponse = 45;
   optional MultipartCommitUploadPartResponse commitMultiPartUploadResponse = 46;
@@ -308,6 +311,9 @@ enum Status {
     DETECTED_LOOP_IN_BUCKET_LINKS = 63;
 
     NOT_SUPPORTED_OPERATION = 64;
+
+    PARTIAL_RENAME = 65;
+
 }
 
 /**
@@ -839,6 +845,22 @@ message LookupKeyResponse {
     optional uint64 openVersion = 4;
 }
 
+message RenameKeysRequest{
+    repeated RenameKeyRequest renameKeyRequest = 1;
+}
+
+message RenameKeyArgs{
+    required string volumeName = 1;
+    required string bucketName = 2;
+    optional string fromKeyName = 3;
+    optional string toKeyName = 4;
+}
+
+message RenameKeysResponse{
+    repeated RenameKeyArgs unRenamedKeys = 1;
+    optional bool status = 2;
+}
+
 message RenameKeyRequest{
     required KeyArgs keyArgs = 1;
     required string toKeyName = 2;