diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 23308e83d574..6d12ff99a89d 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -342,7 +342,7 @@ List listTrash(String volumeName, String bucketName, * @param bucketName - The bucket name. * @param keyName - The key user want to recover. * @param destinationBucket - The bucket user want to recover to. - * @return The recoverTrash + * @return The result of recovering operation is success or not. * @throws IOException */ boolean recoverTrash(String volumeName, String bucketName, String keyName, diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index ff615fb5c1c2..12220cd7b528 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -229,7 +229,6 @@ public static boolean isReadOnly( case LookupKey: case ListKeys: case ListTrash: - case RecoverTrash: case ServiceList: case ListMultiPartUploadParts: case GetFileStatus: @@ -264,6 +263,7 @@ public static boolean isReadOnly( case SetAcl: case AddAcl: case PurgeKeys: + case RecoverTrash: return false; default: LOG.error("CmdType {} is not categorized as readOnly or not.", cmdType); @@ -502,4 +502,18 @@ public static long getOMClientRpcTimeOut(Configuration configuration) { return OzoneConfiguration.of(configuration) .getObject(OMClientConfig.class).getRpcTimeOut(); } + + /** + * Return OmKeyInfo that would be recovered. + */ + public static OmKeyInfo prepareKeyForRecover(OmKeyInfo keyInfo, + RepeatedOmKeyInfo repeatedOmKeyInfo) { + + /* TODO: HDDS-2425. HDDS-2426.*/ + if (repeatedOmKeyInfo.getOmKeyInfoList().contains(keyInfo)) { + return keyInfo; + } else { + return null; + } + } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 4a5e8a19c40d..e434ef8fc532 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -500,10 +500,12 @@ List listTrash(String volumeName, String bucketName, * @param bucketName - The bucket name. * @param keyName - The key user want to recover. * @param destinationBucket - The bucket user want to recover to. - * @return The recoverTrash + * @return The result of recovering operation is success or not. * @throws IOException */ - boolean recoverTrash(String volumeName, String bucketName, String keyName, - String destinationBucket) throws IOException; + default boolean recoverTrash(String volumeName, String bucketName, + String keyName, String destinationBucket) throws IOException { + return false; + } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 0b951fa8a340..c75f7dec9dca 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -1639,15 +1639,14 @@ public boolean recoverTrash(String volumeName, String bucketName, "The destination bucket name cannot be null or empty. " + "Please enter a valid destination bucket name."); - RecoverTrashRequest recoverRequest = RecoverTrashRequest.newBuilder() + RecoverTrashRequest.Builder req = RecoverTrashRequest.newBuilder() .setVolumeName(volumeName) .setBucketName(bucketName) .setKeyName(keyName) - .setDestinationBucket(destinationBucket) - .build(); + .setDestinationBucket(destinationBucket); OMRequest omRequest = createOMRequest(Type.RecoverTrash) - .setRecoverTrashRequest(recoverRequest) + .setRecoverTrashRequest(req) .build(); RecoverTrashResponse recoverResponse = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index 9a9cf8070f3d..e5432daf3d61 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -202,6 +202,18 @@ List listKeys(String volumeName, List listTrash(String volumeName, String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException; + /** + * Recover trash allows the user to recover the keys + * that were marked as deleted, but not actually deleted by Ozone Manager. + * @param volumeName - The volume name. + * @param bucketName - The bucket name. + * @param keyName - The key user want to recover. + * @param destinationBucket - The bucket user want to recover to. + * @return The result of recovering operation is success or not. + */ + boolean recoverTrash(String volumeName, String bucketName, + String keyName, String destinationBucket) throws IOException; + /** * Returns a list of volumes owned by a given user; if user is null, returns * all volumes. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index bcc26fc6622f..54f1452ed64d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -804,6 +804,18 @@ public List listTrash(String volumeName, String bucketName, return deletedKeys; } + @Override + public boolean recoverTrash(String volumeName, String bucketName, + String keyName, String destinationBucket) throws IOException { + + /* TODO: HDDS-2425 and HDDS-2426 + core logic stub would be added in later patch. + */ + + boolean recoverOperation = true; + return recoverOperation; + } + /** * @param userName volume owner, null for listing all volumes. */ diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 347f55e5d36f..f1318fd0d1cf 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -2319,15 +2319,6 @@ public List listTrash(String volumeName, } } - // TODO: HDDS-2424. recover-trash command server side handling. - @Override - public boolean recoverTrash(String volumeName, String bucketName, - String keyName, String destinationBucket) throws IOException { - - boolean recoverOperation = true; - return recoverOperation; - } - /** * Sets bucket property from args. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index d2057c99e29e..9f199076fbee 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -34,6 +34,7 @@ import org.apache.hadoop.ozone.om.request.key.OMKeyDeleteRequest; import org.apache.hadoop.ozone.om.request.key.OMKeyPurgeRequest; import org.apache.hadoop.ozone.om.request.key.OMKeyRenameRequest; +import org.apache.hadoop.ozone.om.request.key.OMTrashRecoverRequest; import org.apache.hadoop.ozone.om.request.key.acl.OMKeyAddAclRequest; import org.apache.hadoop.ozone.om.request.key.acl.OMKeyRemoveAclRequest; import org.apache.hadoop.ozone.om.request.key.acl.OMKeySetAclRequest; @@ -140,6 +141,8 @@ public static OMClientRequest createClientRequest(OMRequest omRequest) { return new OMRenewDelegationTokenRequest(omRequest); case GetS3Secret: return new S3GetSecretRequest(omRequest); + case RecoverTrash: + return new OMTrashRecoverRequest(omRequest); default: throw new IllegalStateException("Unrecognized write command " + "type request" + cmdType); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMTrashRecoverRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMTrashRecoverRequest.java new file mode 100644 index 000000000000..eac7842f84e2 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMTrashRecoverRequest.java @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.request.key; + +import java.io.IOException; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; +import org.apache.hadoop.ozone.om.response.key.OMTrashRecoverResponse; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .RecoverTrashRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status; + +import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; + +/** + * Handles RecoverTrash request. + */ +public class OMTrashRecoverRequest extends OMKeyRequest { + private static final Logger LOG = + LoggerFactory.getLogger(OMTrashRecoverRequest.class); + + public OMTrashRecoverRequest(OMRequest omRequest) { + super(omRequest); + } + + @Override + public OMRequest preExecute(OzoneManager ozoneManager) { + RecoverTrashRequest recoverTrashRequest = getOmRequest() + .getRecoverTrashRequest(); + Preconditions.checkNotNull(recoverTrashRequest); + + return getOmRequest().toBuilder().build(); + } + + @Override + public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, + long transactionLogIndex, + OzoneManagerDoubleBufferHelper ozoneManagerDoubleBufferHelper) { + RecoverTrashRequest recoverTrashRequest = getOmRequest() + .getRecoverTrashRequest(); + Preconditions.checkNotNull(recoverTrashRequest); + + String volumeName = recoverTrashRequest.getVolumeName(); + String bucketName = recoverTrashRequest.getBucketName(); + String keyName = recoverTrashRequest.getKeyName(); + String destinationBucket = recoverTrashRequest.getDestinationBucket(); + + /** TODO: HDDS-2818. New Metrics for Trash Key Recover and Fails. + * OMMetrics omMetrics = ozoneManager.getMetrics(); + */ + + OMResponse.Builder omResponse = OMResponse.newBuilder() + .setCmdType(Type.RecoverTrash).setStatus(Status.OK) + .setSuccess(true); + + OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); + boolean acquireLock = false; + OMClientResponse omClientResponse = null; + try { + // Check acl for the destination bucket. + checkBucketAcls(ozoneManager, volumeName, destinationBucket, keyName, + IAccessAuthorizer.ACLType.WRITE); + + acquireLock = omMetadataManager.getLock() + .acquireWriteLock(BUCKET_LOCK, volumeName, destinationBucket); + + // Validate. + validateBucketAndVolume(omMetadataManager, volumeName, bucketName); + validateBucketAndVolume(omMetadataManager, volumeName, destinationBucket); + + + /** TODO: HDDS-2425. HDDS-2426. + * Update cache. + * omMetadataManager.getKeyTable().addCacheEntry( + * new CacheKey<>(), + * new CacheValue<>() + * ); + * + * Execute recovering trash in non-existing bucket. + * Execute recovering trash in existing bucket. + * omClientResponse = new OMTrashRecoverResponse(omKeyInfo, + * omResponse.setRecoverTrashResponse( + * RecoverTrashResponse.newBuilder()) + * .build()); + */ + omClientResponse = null; + + } catch (IOException ex) { + LOG.error("Fail for recovering trash.", ex); + omClientResponse = new OMTrashRecoverResponse(null, + createErrorOMResponse(omResponse, ex)); + } finally { + if (omClientResponse != null) { + omClientResponse.setFlushFuture( + ozoneManagerDoubleBufferHelper.add(omClientResponse, + transactionLogIndex)); + } + if (acquireLock) { + omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName, + destinationBucket); + } + } + + return omClientResponse; + } + +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMTrashRecoverResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMTrashRecoverResponse.java new file mode 100644 index 000000000000..fb330a309046 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMTrashRecoverResponse.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.response.key; + +import org.apache.hadoop.ozone.OmUtils; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .OMResponse; +import org.apache.hadoop.hdds.utils.db.BatchOperation; + +import java.io.IOException; +import javax.annotation.Nullable; +import javax.annotation.Nonnull; + +/** + * Response for RecoverTrash request. + */ +public class OMTrashRecoverResponse extends OMClientResponse { + private OmKeyInfo omKeyInfo; + + public OMTrashRecoverResponse(@Nullable OmKeyInfo omKeyInfo, + @Nonnull OMResponse omResponse) { + super(omResponse); + this.omKeyInfo = omKeyInfo; + } + + @Override + public void addToDBBatch(OMMetadataManager omMetadataManager, + BatchOperation batchOperation) throws IOException { + + /* TODO: HDDS-2425. HDDS-2426. */ + String trashKey = omMetadataManager + .getOzoneKey(omKeyInfo.getVolumeName(), + omKeyInfo.getBucketName(), omKeyInfo.getKeyName()); + RepeatedOmKeyInfo repeatedOmKeyInfo = omMetadataManager + .getDeletedTable().get(trashKey); + omKeyInfo = OmUtils.prepareKeyForRecover(omKeyInfo, repeatedOmKeyInfo); + omMetadataManager.getDeletedTable() + .deleteWithBatch(batchOperation, omKeyInfo.getKeyName()); + /* TODO: trashKey should be updated to destinationBucket. */ + omMetadataManager.getKeyTable() + .putWithBatch(batchOperation, trashKey, omKeyInfo); + } + +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java index 087bf17f9cbc..cf9e62649deb 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java @@ -55,6 +55,7 @@ public class TestTrashService { public TemporaryFolder tempFolder = new TemporaryFolder(); private KeyManager keyManager; + private OmMetadataManagerImpl omMetadataManager; private String volumeName; private String bucketName; @@ -69,8 +70,8 @@ public void setup() throws IOException { System.setProperty(DBConfigFromFile.CONFIG_DIR, "/"); ServerUtils.setOzoneMetaDirPath(configuration, folder.toString()); - OmMetadataManagerImpl omMetadataManager = - new OmMetadataManagerImpl(configuration); + omMetadataManager = new OmMetadataManagerImpl(configuration); + keyManager = new KeyManagerImpl( new ScmBlockLocationTestingClient(null, null, 0), omMetadataManager, configuration, UUID.randomUUID().toString(), null); @@ -86,11 +87,9 @@ public void testRecoverTrash() throws IOException { String destinationBucket = "destBucket"; createAndDeleteKey(keyName); - /* TODO:HDDS-2424. */ - // boolean recoverOperation = - // ozoneManager.recoverTrash( - // volumeName, bucketName, keyName, destinationBucket); - // Assert.assertTrue(recoverOperation); + boolean recoverOperation = omMetadataManager + .recoverTrash(volumeName, bucketName, keyName, destinationBucket); + Assert.assertTrue(recoverOperation); } private void createAndDeleteKey(String keyName) throws IOException {