Skip to content

HDDS-2417 Add the list trash command to the client side #138

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dineshchitlangia merged 1 commit into from
Nov 11, 2019
Merged

HDDS-2417 Add the list trash command to the client side #138

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions ...op-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import java.util.Map;

import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
Expand Down Expand Up @@ -314,6 +315,25 @@ List<OzoneKey> listKeys(String volumeName, String bucketName,
String keyPrefix, String prevKey, int maxListResult)
throws IOException;

/**
* List trash allows the user to list the keys that were marked as deleted,
* but not actually deleted by Ozone Manager. This allows a user to recover
* keys within a configurable window.
* @param volumeName - The volume name, which can also be a wild card
* using '*'.
* @param bucketName - The bucket name, which can also be a wild card
* using '*'.
* @param startKeyName - List keys from a specific key name.
* @param keyPrefix - List keys using a specific prefix.
* @param maxKeys - The number of keys to be returned. This must be below
* the cluster level set by admins.
* @return The list of keys that are deleted from the deleted table.
* @throws IOException
*/
List<RepeatedOmKeyInfo> listTrash(String volumeName, String bucketName,
String startKeyName, String keyPrefix,
int maxKeys)
throws IOException;

/**
* Get OzoneKey.
Expand Down
12 changes: 12 additions & 0 deletions hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
Expand Down Expand Up @@ -708,6 +709,17 @@ public List<OzoneKey> listKeys(String volumeName, String bucketName,
.collect(Collectors.toList());
}

@Override
public List<RepeatedOmKeyInfo> listTrash(String volumeName, String bucketName,
String startKeyName, String keyPrefix, int maxKeys) throws IOException {

Preconditions.checkNotNull(volumeName);
Preconditions.checkNotNull(bucketName);

return ozoneManagerClient.listTrash(volumeName, bucketName, startKeyName,
keyPrefix, maxKeys);
}

@Override
public OzoneKeyDetails getKeyDetails(
String volumeName, String bucketName, String keyName)
Expand Down
1 change: 1 addition & 0 deletions hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,7 @@ public static boolean isReadOnly(
case ListBuckets:
case LookupKey:
case ListKeys:
case ListTrash:
case InfoS3Bucket:
case ListS3Buckets:
case ServiceList:
Expand Down
19 changes: 19 additions & 0 deletions ...-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
Expand Down Expand Up @@ -527,4 +528,22 @@ DBUpdatesWrapper getDBUpdates(
OzoneManagerProtocolProtos.DBUpdatesRequest dbUpdatesRequest)
throws IOException;

/**
* List trash allows the user to list the keys that were marked as deleted,
* but not actually deleted by Ozone Manager. This allows a user to recover
* keys within a configurable window.
* @param volumeName - The volume name, which can also be a wild card
* using '*'.
* @param bucketName - The bucket name, which can also be a wild card
* using '*'.
* @param startKeyName - List keys from a specific key name.
* @param keyPrefix - List keys using a specific prefix.
* @param maxKeys - The number of keys to be returned. This must be below
* the cluster level set by admins.
* @return The list of keys that are deleted from the deleted table.
* @throws IOException
*/
List<RepeatedOmKeyInfo> listTrash(String volumeName, String bucketName,
String startKeyName, String keyPrefix, int maxKeys) throws IOException;

}
42 changes: 42 additions & 0 deletions ...ava/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
Expand Down Expand Up @@ -105,6 +106,8 @@
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListBucketsResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupKeyRequest;
Expand Down Expand Up @@ -1592,4 +1595,43 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
}
return statusList;
}

@Override
public List<RepeatedOmKeyInfo> listTrash(String volumeName,
String bucketName, String startKeyName, String keyPrefix, int maxKeys)
throws IOException {

Preconditions.checkArgument(Strings.isNullOrEmpty(volumeName),
"The volume name cannot be null or " +
"empty. Please enter a valid volume name or use '*' as a wild card");

Preconditions.checkArgument(Strings.isNullOrEmpty(bucketName),
"The bucket name cannot be null or " +
"empty. Please enter a valid bucket name or use '*' as a wild card");
Copy link
Contributor

@anuengineer anuengineer Nov 10, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As a future JIRA, I was thinking it might be useful to get just the list of buckets that are deleted. It might be interesting.

Copy link
Contributor Author

@mbsharp mbsharp Nov 10, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just created HDDS-2457 to track that.


ListTrashRequest trashRequest = ListTrashRequest.newBuilder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setStartKeyName(startKeyName)
.setKeyPrefix(keyPrefix)
.setMaxKeys(maxKeys)
.build();

OMRequest omRequest = createOMRequest(Type.ListTrash)
.setListTrashRequest(trashRequest)
.build();

ListTrashResponse trashResponse =
handleError(submitRequest(omRequest)).getListTrashResponse();

List<RepeatedOmKeyInfo> deletedKeyList =
new ArrayList<>(trashResponse.getDeletedKeysCount());

deletedKeyList.addAll(
trashResponse.getDeletedKeysList().stream()
.map(RepeatedOmKeyInfo::getFromProto)
.collect(Collectors.toList()));

return deletedKeyList;
}
}
21 changes: 21 additions & 0 deletions hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ enum Type {
PurgeKeys = 81;

ListMultipartUploads = 82;

ListTrash = 91;
}

message OMRequest {
Expand Down Expand Up @@ -161,6 +163,8 @@ message OMRequest {

optional UpdateGetS3SecretRequest updateGetS3SecretRequest = 82;
optional ListMultipartUploadsRequest listMultipartUploadsRequest = 83;

optional ListTrashRequest listTrashRequest = 91;
}

message OMResponse {
Expand Down Expand Up @@ -230,6 +234,8 @@ message OMResponse {
optional PurgeKeysResponse purgeKeysResponse = 81;

optional ListMultipartUploadsResponse listMultipartUploadsResponse = 82;

optional ListTrashResponse listTrashResponse = 91;
}

enum Status {
Expand Down Expand Up @@ -302,6 +308,21 @@ enum Status {
INVALID_PART_ORDER = 56;
}

/**
This command acts as a list command for deleted keys that are still present
in the deleted table on Ozone Manager.
*/
message ListTrashRequest {
required string volumeName = 1;
required string bucketName = 2;
optional string startKeyName = 3;
optional string keyPrefix = 4;
optional int32 maxKeys = 5;
}

message ListTrashResponse {
repeated RepeatedKeyInfo deletedKeys = 1;
}

message VolumeInfo {
required string adminName = 1;
Expand Down
11 changes: 11 additions & 0 deletions hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@
import org.apache.hadoop.ozone.om.ha.OMHANodeDetails;
import org.apache.hadoop.ozone.om.ha.OMNodeDetails;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerServerProtocol;
import org.apache.hadoop.ozone.om.ratis.OMRatisSnapshotInfo;
Expand Down Expand Up @@ -2239,6 +2240,16 @@ public List<OmKeyInfo> listKeys(String volumeName, String bucketName,
}
}

// TODO: HDDS-2418 - Complete stub below for server logic
@Override
public List<RepeatedOmKeyInfo> listTrash(String volumeName,
String bucketName, String startKeyName, String keyPrefix, int maxKeys)
throws IOException {

List<RepeatedOmKeyInfo> deletedKeys = new ArrayList<>();
return deletedKeys;
}

/**
* Sets bucket property from args.
*
Expand Down