HDDS-3930. Fix OMKeyDeletesRequest.

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -296,6 +296,8 @@ private OzoneConsts() {
   public static final String MULTIPART_UPLOAD_PART_NUMBER = "partNumber";
   public static final String MULTIPART_UPLOAD_PART_NAME = "partName";
   public static final String BUCKET_ENCRYPTION_KEY = "bucketEncryptionKey";
+  public static final String DELETED_KEYS_LIST = "deletedKeysList";
+  public static final String UNDELETED_KEYS_LIST = "unDeletedKeysList";
 
 
 

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -76,6 +76,7 @@
 import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
 import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmDeleteKeys;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
@@ -730,16 +731,9 @@ public void deleteKeys(
           throws IOException {
     HddsClientUtils.verifyResourceName(volumeName, bucketName);
     Preconditions.checkNotNull(keyNameList);
-    List<OmKeyArgs> keyArgsList = new ArrayList<>();
-    for (String keyName: keyNameList) {
-      OmKeyArgs keyArgs = new OmKeyArgs.Builder()
-          .setVolumeName(volumeName)
-          .setBucketName(bucketName)
-          .setKeyName(keyName)
-          .build();
-      keyArgsList.add(keyArgs);
-    }
-    ozoneManagerClient.deleteKeys(keyArgsList);
+    OmDeleteKeys omDeleteKeys = new OmDeleteKeys(volumeName, bucketName,
+        keyNameList);
+    ozoneManagerClient.deleteKeys(omDeleteKeys);
   }
 
   @Override

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java

@@ -38,6 +38,7 @@ public enum OMAction implements AuditAction {
   UPDATE_BUCKET,
   UPDATE_KEY,
   PURGE_KEYS,
+  DELETE_KEYS,
 
   // S3 Bucket
   CREATE_S3_BUCKET,

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java

@@ -223,6 +223,8 @@ public enum ResultCodes {
 
     INVALID_VOLUME_NAME,
 
+    PARTIAL_DELETE,
+
     REPLAY // When ratis logs are replayed.
   }
 }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDeleteKeys.java

@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+import java.util.List;
+
+/**
+ * Represent class which has info of Keys to be deleted from Client.
+ */
+public class OmDeleteKeys {
+
+  private String volume;
+  private String bucket;
+
+  private List<String> keyNames;
+
+
+  public OmDeleteKeys(String volume, String bucket, List<String> keyNames) {
+    this.volume = volume;
+    this.bucket = bucket;
+    this.keyNames = keyNames;
+  }
+
+  public String getVolume() {
+    return volume;
+  }
+
+  public String getBucket() {
+    return bucket;
+  }
+
+  public List< String > getKeyNames() {
+    return keyNames;
+  }
+}

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java

@@ -29,6 +29,7 @@
 import org.apache.hadoop.ozone.om.helpers.DBUpdates;
 import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
 import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmDeleteKeys;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -229,10 +230,10 @@ OmKeyLocationInfo allocateBlock(OmKeyArgs args, long clientID,
    * multiple keys and a single key. Used by deleting files
    * through OzoneFileSystem.
    *
-   * @param args the list args of the key.
+   * @param deleteKeys
    * @throws IOException
    */
-  void deleteKeys(List<OmKeyArgs> args) throws IOException;
+  void deleteKeys(OmDeleteKeys deleteKeys) throws IOException;
 
   /**
    * Deletes an existing empty bucket from volume.

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java

@@ -33,6 +33,7 @@
 import org.apache.hadoop.ozone.om.helpers.KeyValueUtil;
 import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
 import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmDeleteKeys;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -71,8 +72,9 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteBucketRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeysRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyArgs;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeysRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteVolumeRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetAclRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetAclResponse;
@@ -141,6 +143,7 @@
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import com.google.protobuf.ByteString;
+
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_ERROR_OTHER;
 import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.ACCESS_DENIED;
@@ -717,22 +720,17 @@ public void deleteKey(OmKeyArgs args) throws IOException {
    * Deletes existing key/keys. This interface supports delete
    * multiple keys and a single key.
    *
-   * @param args the list args of the key.
+   * @param deleteKeys
    * @throws IOException
    */
   @Override
-  public void deleteKeys(List<OmKeyArgs> args) throws IOException {
+  public void deleteKeys(OmDeleteKeys deleteKeys) throws IOException {
     DeleteKeysRequest.Builder req = DeleteKeysRequest.newBuilder();
-    List <KeyArgs> keyArgsList = new ArrayList<KeyArgs>();
-    for (OmKeyArgs omKeyArgs : args) {
-      KeyArgs keyArgs = KeyArgs.newBuilder()
-          .setVolumeName(omKeyArgs.getVolumeName())
-          .setBucketName(omKeyArgs.getBucketName())
-          .setKeyName(omKeyArgs.getKeyName()).build();
-      keyArgsList.add(keyArgs);
-    }
-    req.addAllKeyArgs(keyArgsList);
-
+    DeleteKeyArgs deletedKeys = DeleteKeyArgs.newBuilder()
+        .setBucketName(deleteKeys.getBucket())
+        .setVolumeName(deleteKeys.getVolume())
+        .addAllKeys(deleteKeys.getKeyNames()).build();
+    req.setDeleteKeys(deletedKeys);
     OMRequest omRequest = createOMRequest(Type.DeleteKeys)
         .setDeleteKeysRequest(req)
         .build();

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java

@@ -72,8 +72,8 @@
 public class TestOzoneFileSystem {
 
   /**
-    * Set a timeout for each test.
-    */
+   * Set a timeout for each test.
+   */
   @Rule
   public Timeout timeout = new Timeout(300000);
 
@@ -89,7 +89,7 @@ public class TestOzoneFileSystem {
 
   @Test(timeout = 300_000)
   public void testCreateFileShouldCheckExistenceOfDirWithSameName()
-          throws Exception {
+      throws Exception {
     /*
      * Op 1. create file -> /d1/d2/d3/d4/key2
      * Op 2. create dir -> /d1/d2/d3/d4/key2
@@ -195,20 +195,20 @@ public void tearDown() {
   }
 
   private void setupOzoneFileSystem()
-          throws IOException, TimeoutException, InterruptedException {
+      throws IOException, TimeoutException, InterruptedException {
     OzoneConfiguration conf = new OzoneConfiguration();
     cluster = MiniOzoneCluster.newBuilder(conf)
-            .setNumDatanodes(3)
-            .build();
+        .setNumDatanodes(3)
+        .build();
     cluster.waitForClusterToBeReady();
     // create a volume and a bucket to be used by OzoneFileSystem
     OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
     volumeName = bucket.getVolumeName();
     bucketName = bucket.getName();
 
     String rootPath = String.format("%s://%s.%s/",
-            OzoneConsts.OZONE_URI_SCHEME, bucket.getName(),
-            bucket.getVolumeName());
+        OzoneConsts.OZONE_URI_SCHEME, bucket.getName(),
+        bucket.getVolumeName());
 
     // Set the fs.defaultFS and start the filesystem
     conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithData.java

@@ -50,7 +50,7 @@
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.DIRECTORY_NOT_FOUND;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_ALREADY_EXISTS;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE;
-import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PARTIAL_DELETE;
 import static org.junit.Assert.fail;
 
 /**
@@ -187,8 +187,9 @@ public void testKeysDelete() throws Exception {
       ozoneBucket.deleteKeys(keyList2);
       fail("testFilesDelete");
     } catch (OMException ex) {
-      // The expected exception KEY_NOT_FOUND.
-      Assert.assertEquals(KEY_NOT_FOUND, ex.getResult());
+      // The expected exception PARTIAL_DELETE, as if not able to delete, we
+      // return error codee PARTIAL_DElETE.
+      Assert.assertEquals(PARTIAL_DELETE, ex.getResult());
     }
   }
 

hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto

@@ -303,6 +303,8 @@ enum Status {
 
     INVALID_VOLUME_NAME = 61;
 
+    PARTIAL_DELETE = 62;
+
     // When transactions are replayed
     REPLAY = 100;
 }
@@ -848,7 +850,18 @@ message DeleteKeyRequest {
 }
 
 message DeleteKeysRequest {
-    repeated KeyArgs keyArgs = 1;
+    optional DeleteKeyArgs deleteKeys = 1;
+}
+
+message DeleteKeyArgs {
+    required string volumeName = 1;
+    required string bucketName = 2;
+    repeated string keys = 3;
+}
+
+message DeleteKeysResponse {
+    optional DeleteKeyArgs unDeletedKeys = 1;
+    optional bool status = 2;
 }
 
 message DeleteKeyResponse {
@@ -866,10 +879,7 @@ message DeletedKeys {
     repeated string keys = 3;
 }
 
-message DeleteKeysResponse {
-    repeated KeyInfo deletedKeys = 1;
-    repeated KeyInfo unDeletedKeys = 2;
-}
+
 
 message PurgeKeysRequest {
     repeated DeletedKeys deletedKeys = 1;

hadoop-ozone/interface-client/src/main/proto/proto.lock

@@ -416,6 +416,10 @@
                 "name": "INVALID_VOLUME_NAME",
                 "integer": 61
               },
+              {
+                "name": "PARTIAL_DELETE",
+                "integer": 62
+              },
               {
                 "name": "REPLAY",
                 "integer": 100
@@ -2434,12 +2438,47 @@
             "fields": [
               {
                 "id": 1,
-                "name": "keyArgs",
-                "type": "KeyArgs",
+                "name": "deleteKeys",
+                "type": "DeleteKeyArgs"
+              }
+            ]
+          },
+          {
+            "name": "DeleteKeyArgs",
+            "fields": [
+              {
+                "id": 1,
+                "name": "volumeName",
+                "type": "string"
+              },
+              {
+                "id": 2,
+                "name": "bucketName",
+                "type": "string"
+              },
+              {
+                "id": 3,
+                "name": "keys",
+                "type": "string",
                 "is_repeated": true
               }
             ]
           },
+          {
+            "name": "DeleteKeysResponse",
+            "fields": [
+              {
+                "id": 1,
+                "name": "unDeletedKeys",
+                "type": "DeleteKeyArgs"
+              },
+              {
+                "id": 2,
+                "name": "status",
+                "type": "bool"
+              }
+            ]
+          },
           {
             "name": "DeleteKeyResponse",
             "fields": [
@@ -2481,23 +2520,6 @@
               }
             ]
           },
-          {
-            "name": "DeleteKeysResponse",
-            "fields": [
-              {
-                "id": 1,
-                "name": "deletedKeys",
-                "type": "KeyInfo",
-                "is_repeated": true
-              },
-              {
-                "id": 2,
-                "name": "unDeletedKeys",
-                "type": "KeyInfo",
-                "is_repeated": true
-              }
-            ]
-          },
           {
             "name": "PurgeKeysRequest",
             "fields": [

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java

@@ -214,6 +214,10 @@ public void setNumKeys(long val) {
     this.numKeys.incr(val- oldVal);
   }
 
+  public void decNumKeys(long val) {
+    this.numKeys.incr(-val);
+  }
+
   public long getNumVolumes() {
     return numVolumes.value();
   }