HDDS-4122. Implement OM Delete Expired Open Key Request and Response #1435
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,126 @@ | ||
| /** | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * <p> | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * <p> | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.hadoop.ozone.om.response.key; | ||
|
|
||
| import org.apache.hadoop.hdds.utils.db.Table; | ||
| import org.apache.hadoop.ozone.OmUtils; | ||
| import org.apache.hadoop.ozone.om.OMMetadataManager; | ||
| import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; | ||
| import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; | ||
| import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; | ||
| import org.apache.hadoop.ozone.om.response.CleanupTableInfo; | ||
| import org.apache.hadoop.ozone.om.response.OMClientResponse; | ||
| import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos | ||
| .OMResponse; | ||
| import org.apache.hadoop.hdds.utils.db.BatchOperation; | ||
|
|
||
| import java.io.IOException; | ||
| import javax.annotation.Nullable; | ||
| import javax.annotation.Nonnull; | ||
|
|
||
| import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE; | ||
|
|
||
| /** | ||
| * Base class for responses that need to move keys from an arbitrary table to | ||
| * the deleted table. | ||
| */ | ||
| @CleanupTableInfo(cleanupTables = {DELETED_TABLE}) | ||
| public abstract class AbstractOMKeyDeleteResponse extends OMClientResponse { | ||
|
|
||
| private boolean isRatisEnabled; | ||
|
|
||
| public AbstractOMKeyDeleteResponse( | ||
| @Nonnull OMResponse omResponse, boolean isRatisEnabled) { | ||
|
|
||
| super(omResponse); | ||
| this.isRatisEnabled = isRatisEnabled; | ||
| } | ||
|
|
||
| /** | ||
| * For when the request is not successful. | ||
| * For a successful request, the other constructor should be used. | ||
| */ | ||
| public AbstractOMKeyDeleteResponse(@Nonnull OMResponse omResponse) { | ||
| super(omResponse); | ||
| checkStatusNotOK(); | ||
| } | ||
|
|
||
| /** | ||
| * Adds the operation of deleting the {@code keyName omKeyInfo} pair from | ||
| * {@code fromTable} to the batch operation {@code batchOperation}. The | ||
| * batch operation is not committed, so no changes are persisted to disk. | ||
| * The log transaction index used will be retrieved by calling | ||
| * {@link OmKeyInfo#getUpdateID} on {@code omKeyInfo}. | ||
| */ | ||
| protected void addDeletionToBatch( | ||
| OMMetadataManager omMetadataManager, | ||
| BatchOperation batchOperation, | ||
| Table<String, ?> fromTable, | ||
| String keyName, | ||
| OmKeyInfo omKeyInfo) throws IOException { | ||
|
|
||
| // For OmResponse with failure, this should do nothing. This method is | ||
| // not called in failure scenario in OM code. | ||
| fromTable.deleteWithBatch(batchOperation, keyName); | ||
|
|
||
| // If Key is not empty add this to delete table. | ||
| if (!isKeyEmpty(omKeyInfo)) { | ||
| // If a deleted key is put in the table where a key with the same | ||
| // name already exists, then the old deleted key information would be | ||
| // lost. To avoid this, first check if a key with same name exists. | ||
| // deletedTable in OM Metadata stores <KeyName, RepeatedOMKeyInfo>. | ||
| // The RepeatedOmKeyInfo is the structure that allows us to store a | ||
| // list of OmKeyInfo that can be tied to same key name. For a keyName | ||
| // if RepeatedOMKeyInfo structure is null, we create a new instance, | ||
| // if it is not null, then we simply add to the list and store this | ||
| // instance in deletedTable. | ||
| RepeatedOmKeyInfo repeatedOmKeyInfo = | ||
| omMetadataManager.getDeletedTable().get(keyName); | ||
| repeatedOmKeyInfo = OmUtils.prepareKeyForDelete( | ||
| omKeyInfo, repeatedOmKeyInfo, omKeyInfo.getUpdateID(), | ||
| isRatisEnabled); | ||
| omMetadataManager.getDeletedTable().putWithBatch( | ||
| batchOperation, keyName, repeatedOmKeyInfo); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public abstract void addToDBBatch(OMMetadataManager omMetadataManager, | ||
| BatchOperation batchOperation) throws IOException; | ||
|
|
||
| /** | ||
| * Check if the key is empty or not. Key will be empty if it does not have | ||
| * blocks. | ||
| * | ||
| * @param keyInfo | ||
| * @return if empty true, else false. | ||
| */ | ||
| private boolean isKeyEmpty(@Nullable OmKeyInfo keyInfo) { | ||
|
Contributor
There was a problem hiding this comment. Looks some of the logic is common for OMKeyDeleteResponse and OMOpenKeysDeleteResponse like isKeyEmpty and deleteFromTable can be used from OMKeyDeleteResponse.
Contributor
Author
There was a problem hiding this comment. Yes, the idea of creating this abstract class was to eventually consolidate the duplicate code between OMOpenKeysDeleteResponse, OMKeyDeleteResponse, and OMKeysDeleteResponse. I had originally refactored the other classes as well to use this code, but since HDDS-451 (quota support) is moving along at a brisk pace, I could not keep up with the merge conflicts as the other response classes kept changing, and decided it was better to do this in a later PR.
Contributor
Author
There was a problem hiding this comment. Looks like development on the key(s) delete request and response classes has taken a break. I have refactored them to use these shared methods now. |
||
| if (keyInfo == null) { | ||
| return true; | ||
| } | ||
| for (OmKeyLocationInfoGroup keyLocationList : keyInfo | ||
| .getKeyLocationVersions()) { | ||
| if (keyLocationList.getLocationListCount() != 0) { | ||
| return false; | ||
| } | ||
| } | ||
| return true; | ||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Good point to update the used bytes while doing this cleanup. I am wondering what this would mean with multiple key version support in the future. We do not seem to store the "version" of the current open key.