HDDS-2424. Add the recover-trash command server side handling. (#399)

cxorm · web-flow · commit f20cb4e5b288 · 2020-05-05T17:44:23.000+05:30
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -342,7 +342,7 @@ List<RepeatedOmKeyInfo> listTrash(String volumeName, String bucketName,
    * @param bucketName - The bucket name.
    * @param keyName - The key user want to recover.
    * @param destinationBucket - The bucket user want to recover to.
-   * @return The recoverTrash
+   * @return The result of recovering operation is success or not.
    * @throws IOException
    */
   boolean recoverTrash(String volumeName, String bucketName, String keyName,
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -229,7 +229,6 @@ public static boolean isReadOnly(
     case LookupKey:
     case ListKeys:
     case ListTrash:
-    case RecoverTrash:
     case ServiceList:
     case ListMultiPartUploadParts:
     case GetFileStatus:
@@ -264,6 +263,7 @@ public static boolean isReadOnly(
     case SetAcl:
     case AddAcl:
     case PurgeKeys:
+    case RecoverTrash:
       return false;
     default:
       LOG.error("CmdType {} is not categorized as readOnly or not.", cmdType);
@@ -502,4 +502,18 @@ public static long getOMClientRpcTimeOut(Configuration configuration) {
     return OzoneConfiguration.of(configuration)
         .getObject(OMClientConfig.class).getRpcTimeOut();
   }
+
+  /**
+   * Return OmKeyInfo that would be recovered.
+   */
+  public static OmKeyInfo prepareKeyForRecover(OmKeyInfo keyInfo,
+      RepeatedOmKeyInfo repeatedOmKeyInfo) {
+
+    /* TODO: HDDS-2425. HDDS-2426.*/
+    if (repeatedOmKeyInfo.getOmKeyInfoList().contains(keyInfo)) {
+      return keyInfo;
+    } else {
+      return null;
+    }
+  }
 }
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -500,10 +500,12 @@ List<RepeatedOmKeyInfo> listTrash(String volumeName, String bucketName,
    * @param bucketName - The bucket name.
    * @param keyName - The key user want to recover.
    * @param destinationBucket - The bucket user want to recover to.
-   * @return The recoverTrash
+   * @return The result of recovering operation is success or not.
    * @throws IOException
    */
-  boolean recoverTrash(String volumeName, String bucketName, String keyName,
-      String destinationBucket) throws IOException;
+  default boolean recoverTrash(String volumeName, String bucketName,
+      String keyName, String destinationBucket) throws IOException {
+    return false;
+  }
 
 }
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -1639,15 +1639,14 @@ public boolean recoverTrash(String volumeName, String bucketName,
         "The destination bucket name cannot be null or empty. " +
         "Please enter a valid destination bucket name.");
 
-    RecoverTrashRequest recoverRequest = RecoverTrashRequest.newBuilder()
+    RecoverTrashRequest.Builder req = RecoverTrashRequest.newBuilder()
         .setVolumeName(volumeName)
         .setBucketName(bucketName)
         .setKeyName(keyName)
-        .setDestinationBucket(destinationBucket)
-        .build();
+        .setDestinationBucket(destinationBucket);
 
     OMRequest omRequest = createOMRequest(Type.RecoverTrash)
-        .setRecoverTrashRequest(recoverRequest)
+        .setRecoverTrashRequest(req)
         .build();
 
     RecoverTrashResponse recoverResponse =
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
@@ -202,6 +202,18 @@ List<OmKeyInfo> listKeys(String volumeName,
   List<RepeatedOmKeyInfo> listTrash(String volumeName, String bucketName,
       String startKeyName, String keyPrefix, int maxKeys) throws IOException;
 
+  /**
+   * Recover trash allows the user to recover the keys
+   * that were marked as deleted, but not actually deleted by Ozone Manager.
+   * @param volumeName - The volume name.
+   * @param bucketName - The bucket name.
+   * @param keyName - The key user want to recover.
+   * @param destinationBucket - The bucket user want to recover to.
+   * @return The result of recovering operation is success or not.
+   */
+  boolean recoverTrash(String volumeName, String bucketName,
+      String keyName, String destinationBucket) throws IOException;
+
   /**
    * Returns a list of volumes owned by a given user; if user is null, returns
    * all volumes.
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -804,6 +804,18 @@ public List<RepeatedOmKeyInfo> listTrash(String volumeName, String bucketName,
     return deletedKeys;
   }
 
+  @Override
+  public boolean recoverTrash(String volumeName, String bucketName,
+      String keyName, String destinationBucket) throws IOException {
+
+    /* TODO: HDDS-2425 and HDDS-2426
+        core logic stub would be added in later patch.
+     */
+
+    boolean recoverOperation = true;
+    return recoverOperation;
+  }
+
   /**
    * @param userName volume owner, null for listing all volumes.
    */
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -2319,15 +2319,6 @@ public List<RepeatedOmKeyInfo> listTrash(String volumeName,
     }
   }
 
-  // TODO: HDDS-2424. recover-trash command server side handling.
-  @Override
-  public boolean recoverTrash(String volumeName, String bucketName,
-      String keyName, String destinationBucket) throws IOException {
-
-    boolean recoverOperation = true;
-    return recoverOperation;
-  }
-
   /**
    * Sets bucket property from args.
    *
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
@@ -34,6 +34,7 @@
 import org.apache.hadoop.ozone.om.request.key.OMKeyDeleteRequest;
 import org.apache.hadoop.ozone.om.request.key.OMKeyPurgeRequest;
 import org.apache.hadoop.ozone.om.request.key.OMKeyRenameRequest;
+import org.apache.hadoop.ozone.om.request.key.OMTrashRecoverRequest;
 import org.apache.hadoop.ozone.om.request.key.acl.OMKeyAddAclRequest;
 import org.apache.hadoop.ozone.om.request.key.acl.OMKeyRemoveAclRequest;
 import org.apache.hadoop.ozone.om.request.key.acl.OMKeySetAclRequest;
@@ -140,6 +141,8 @@ public static OMClientRequest createClientRequest(OMRequest omRequest) {
       return new OMRenewDelegationTokenRequest(omRequest);
     case GetS3Secret:
       return new S3GetSecretRequest(omRequest);
+    case RecoverTrash:
+      return new OMTrashRecoverRequest(omRequest);
     default:
       throw new IllegalStateException("Unrecognized write command " +
           "type request" + cmdType);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMTrashRecoverRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMTrashRecoverRequest.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.request.key;
+
+import java.io.IOException;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper;
+import org.apache.hadoop.ozone.om.response.key.OMTrashRecoverResponse;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+    .RecoverTrashRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+    .OMResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+    .OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status;
+
+import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK;
+
+/**
+ * Handles RecoverTrash request.
+ */
+public class OMTrashRecoverRequest extends OMKeyRequest {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(OMTrashRecoverRequest.class);
+
+  public OMTrashRecoverRequest(OMRequest omRequest) {
+    super(omRequest);
+  }
+
+  @Override
+  public OMRequest preExecute(OzoneManager ozoneManager) {
+    RecoverTrashRequest recoverTrashRequest = getOmRequest()
+        .getRecoverTrashRequest();
+    Preconditions.checkNotNull(recoverTrashRequest);
+
+    return getOmRequest().toBuilder().build();
+  }
+
+  @Override
+  public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
+      long transactionLogIndex,
+      OzoneManagerDoubleBufferHelper ozoneManagerDoubleBufferHelper) {
+    RecoverTrashRequest recoverTrashRequest = getOmRequest()
+        .getRecoverTrashRequest();
+    Preconditions.checkNotNull(recoverTrashRequest);
+
+    String volumeName = recoverTrashRequest.getVolumeName();
+    String bucketName = recoverTrashRequest.getBucketName();
+    String keyName = recoverTrashRequest.getKeyName();
+    String destinationBucket = recoverTrashRequest.getDestinationBucket();
+
+    /** TODO: HDDS-2818. New Metrics for Trash Key Recover and Fails.
+     *  OMMetrics omMetrics = ozoneManager.getMetrics();
+     */
+
+    OMResponse.Builder omResponse = OMResponse.newBuilder()
+        .setCmdType(Type.RecoverTrash).setStatus(Status.OK)
+        .setSuccess(true);
+
+    OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
+    boolean acquireLock = false;
+    OMClientResponse omClientResponse = null;
+    try {
+      // Check acl for the destination bucket.
+      checkBucketAcls(ozoneManager, volumeName, destinationBucket, keyName,
+          IAccessAuthorizer.ACLType.WRITE);
+
+      acquireLock = omMetadataManager.getLock()
+          .acquireWriteLock(BUCKET_LOCK, volumeName, destinationBucket);
+
+      // Validate.
+      validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
+      validateBucketAndVolume(omMetadataManager, volumeName, destinationBucket);
+
+
+      /** TODO: HDDS-2425. HDDS-2426.
+       *  Update cache.
+       *    omMetadataManager.getKeyTable().addCacheEntry(
+       *    new CacheKey<>(),
+       *    new CacheValue<>()
+       *    );
+       *
+       *  Execute recovering trash in non-existing bucket.
+       *  Execute recovering trash in existing bucket.
+       *    omClientResponse = new OMTrashRecoverResponse(omKeyInfo,
+       *    omResponse.setRecoverTrashResponse(
+       *    RecoverTrashResponse.newBuilder())
+       *    .build());
+       */
+      omClientResponse = null;
+
+    } catch (IOException ex) {
+      LOG.error("Fail for recovering trash.", ex);
+      omClientResponse = new OMTrashRecoverResponse(null,
+          createErrorOMResponse(omResponse, ex));
+    } finally {
+      if (omClientResponse != null) {
+        omClientResponse.setFlushFuture(
+            ozoneManagerDoubleBufferHelper.add(omClientResponse,
+                transactionLogIndex));
+      }
+      if (acquireLock) {
+        omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName,
+            destinationBucket);
+      }
+    }
+
+    return omClientResponse;
+  }
+
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMTrashRecoverResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMTrashRecoverResponse.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.response.key;
+
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
+    .OMResponse;
+import org.apache.hadoop.hdds.utils.db.BatchOperation;
+
+import java.io.IOException;
+import javax.annotation.Nullable;
+import javax.annotation.Nonnull;
+
+/**
+ * Response for RecoverTrash request.
+ */
+public class OMTrashRecoverResponse extends OMClientResponse {
+  private OmKeyInfo omKeyInfo;
+
+  public OMTrashRecoverResponse(@Nullable OmKeyInfo omKeyInfo,
+      @Nonnull OMResponse omResponse) {
+    super(omResponse);
+    this.omKeyInfo = omKeyInfo;
+  }
+
+  @Override
+  public void addToDBBatch(OMMetadataManager omMetadataManager,
+      BatchOperation batchOperation) throws IOException {
+
+      /* TODO: HDDS-2425. HDDS-2426. */
+    String trashKey = omMetadataManager
+        .getOzoneKey(omKeyInfo.getVolumeName(),
+            omKeyInfo.getBucketName(), omKeyInfo.getKeyName());
+    RepeatedOmKeyInfo repeatedOmKeyInfo = omMetadataManager
+        .getDeletedTable().get(trashKey);
+    omKeyInfo = OmUtils.prepareKeyForRecover(omKeyInfo, repeatedOmKeyInfo);
+    omMetadataManager.getDeletedTable()
+        .deleteWithBatch(batchOperation, omKeyInfo.getKeyName());
+    /* TODO: trashKey should be updated to destinationBucket. */
+    omMetadataManager.getKeyTable()
+        .putWithBatch(batchOperation, trashKey, omKeyInfo);
+  }
+
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java
@@ -55,6 +55,7 @@ public class TestTrashService {
   public TemporaryFolder tempFolder = new TemporaryFolder();
 
   private KeyManager keyManager;
+  private OmMetadataManagerImpl omMetadataManager;
   private String volumeName;
   private String bucketName;
 
@@ -69,8 +70,8 @@ public void setup() throws IOException {
     System.setProperty(DBConfigFromFile.CONFIG_DIR, "/");
     ServerUtils.setOzoneMetaDirPath(configuration, folder.toString());
 
-    OmMetadataManagerImpl omMetadataManager =
-        new OmMetadataManagerImpl(configuration);
+    omMetadataManager = new OmMetadataManagerImpl(configuration);
+
     keyManager = new KeyManagerImpl(
         new ScmBlockLocationTestingClient(null, null, 0),
         omMetadataManager, configuration, UUID.randomUUID().toString(), null);
@@ -86,11 +87,9 @@ public void testRecoverTrash() throws IOException {
     String destinationBucket = "destBucket";
     createAndDeleteKey(keyName);
 
-    /* TODO:HDDS-2424. */
-    // boolean recoverOperation =
-    //     ozoneManager.recoverTrash(
-    //         volumeName, bucketName, keyName, destinationBucket);
-    // Assert.assertTrue(recoverOperation);
+    boolean recoverOperation = omMetadataManager
+        .recoverTrash(volumeName, bucketName, keyName, destinationBucket);
+    Assert.assertTrue(recoverOperation);
   }
 
   private void createAndDeleteKey(String keyName) throws IOException {

Original file line number	Diff line number	Diff line change
`@@ -2319,15 +2319,6 @@ public List<RepeatedOmKeyInfo> listTrash(String volumeName,`
`2319`	`2319`	`}`
`2320`	`2320`	`}`
`2321`	`2321`
`2322`		`- // TODO: HDDS-2424. recover-trash command server side handling.`
`2323`		`- @Override`
`2324`		`- public boolean recoverTrash(String volumeName, String bucketName,`
`2325`		`- String keyName, String destinationBucket) throws IOException {`
`2326`		`-`
`2327`		`- boolean recoverOperation = true;`
`2328`		`- return recoverOperation;`
`2329`		`- }`
`2330`		`-`
`2331`	`2322`	`/**`
`2332`	`2323`	`* Sets bucket property from args.`
`2333`	`2324`	`*`