Skip to content

HDDS-4742. Make trash work with FS Optimised Buckets. #1915

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rakeshadr merged 6 commits into from
Feb 16, 2021
Merged

HDDS-4742. Make trash work with FS Optimised Buckets. #1915

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
35d1647
HDDS-4742.Make trash work with FS Optimised Buckets.
sadanand48 Feb 9, 2021
e5dacae
rebase branch
sadanand48 Feb 15, 2021
a3369ea
addressed comment
sadanand48 Feb 15, 2021
011d980
addressed comments
sadanand48 Feb 16, 2021
0cd2a5c
fixed typo
sadanand48 Feb 16, 2021
c655887
fix checkstyle
sadanand48 Feb 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,12 @@
package org.apache.hadoop.ozone.om.helpers;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.util.StringUtils;

import javax.annotation.Nonnull;
import java.nio.file.Paths;
import java.util.Map;

import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
Expand Down Expand Up @@ -205,4 +207,27 @@ public static int getFileCount(String keyName) {
java.nio.file.Path keyPath = Paths.get(keyName);
return keyPath.getNameCount();
}


/**
* Returns true if the bucket is FS Optimised.
* @param bucketMetadata
* @return
*/
public static boolean isFSOptimizedBucket(
Map<String, String> bucketMetadata) {
// layout version V1 represents optimized FS path
boolean layoutVersionEnabled =
org.apache.commons.lang3.StringUtils.equalsIgnoreCase(
OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1,
bucketMetadata
.get(OMConfigKeys.OZONE_OM_LAYOUT_VERSION));

boolean fsEnabled =
Boolean.parseBoolean(bucketMetadata
.get(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS));

return layoutVersionEnabled && fsEnabled;
}

}
12 changes: 0 additions & 12 deletions ...zone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,18 +380,6 @@ public void testRenameDestinationParentDoesntExist() throws Exception {
}
}

@Override
@Test
@Ignore("TODO:HDDS-2939")
public void testTrash() throws Exception {
}

@Override
@Test
@Ignore("TODO:HDDS-2939")
public void testRenameToTrashEnabled() throws Exception {
}

@Override
@Test
@Ignore("TODO:HDDS-2939")
Expand Down
142 changes: 108 additions & 34 deletions ...op-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,11 @@
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.helpers.OMRatisHelper;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.om.helpers.OMRatisHelper;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Progressable;
import org.apache.ratis.protocol.ClientId;
Expand Down Expand Up @@ -156,6 +157,11 @@ public boolean rename(Path src, Path dst) throws IOException {
// check whether the src and dst belong to the same bucket & trashroot.
OFSPath srcPath = new OFSPath(src);
OFSPath dstPath = new OFSPath(dst);
OmBucketInfo bucket = ozoneManager.getBucketInfo(srcPath.getVolumeName(),
srcPath.getBucketName());
if (OzoneFSUtils.isFSOptimizedBucket(bucket.getMetadata())) {
return renameV1(srcPath, dstPath);
}
Preconditions.checkArgument(srcPath.getBucketName().
equals(dstPath.getBucketName()));
Preconditions.checkArgument(srcPath.getTrashRoot().
Expand All @@ -165,14 +171,50 @@ public boolean rename(Path src, Path dst) throws IOException {
return true;
}

private boolean renameV1(OFSPath srcPath, OFSPath dstPath) {
OzoneManagerProtocolProtos.OMRequest omRequest =
getRenameKeyRequest(srcPath, dstPath);
try {
if(omRequest != null) {
submitRequest(omRequest);
return true;
}
return false;
} catch (Exception e){
LOG.error("couldnt send rename requestV1", e);
return false;
}
}

@Override
public boolean delete(Path path, boolean b) throws IOException {
ozoneManager.getMetrics().incNumTrashDeletes();
OFSPath srcPath = new OFSPath(path);
OmBucketInfo bucket = ozoneManager.getBucketInfo(srcPath.getVolumeName(),
srcPath.getBucketName());
if (OzoneFSUtils.isFSOptimizedBucket(bucket.getMetadata())) {
return deleteV1(srcPath);
}
DeleteIterator iterator = new DeleteIterator(path, true);
iterator.iterate();
return true;
}

private boolean deleteV1(OFSPath srcPath) {
OzoneManagerProtocolProtos.OMRequest omRequest =
getDeleteKeyRequest(srcPath);
try {
if(omRequest != null) {
submitRequest(omRequest);
return true;
}
return false;
} catch (Throwable e) {
LOG.error("Couldn't send delete request.", e);
return false;
}
}

@Override
public FileStatus[] listStatus(Path path) throws IOException {
ozoneManager.getMetrics().incNumTrashListStatus();
Expand Down Expand Up @@ -377,6 +419,41 @@ FileStatus getStatus() {
}
}


private OzoneManagerProtocolProtos.OMRequest
getRenameKeyRequest(
OFSPath src, OFSPath dst) {
String volumeName = src.getVolumeName();
String bucketName = src.getBucketName();
String keyName = src.getKeyName();

OzoneManagerProtocolProtos.KeyArgs keyArgs =
OzoneManagerProtocolProtos.KeyArgs.newBuilder()
.setKeyName(keyName)
.setVolumeName(volumeName)
.setBucketName(bucketName)
.build();
String toKeyName = dst.getKeyName();
OzoneManagerProtocolProtos.RenameKeyRequest renameKeyRequest =
OzoneManagerProtocolProtos.RenameKeyRequest.newBuilder()
.setKeyArgs(keyArgs)
.setToKeyName(toKeyName)
.build();
OzoneManagerProtocolProtos.OMRequest omRequest =
null;
try {
omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder()
.setClientId(CLIENT_ID.toString())
.setUserInfo(getUserInfo())
.setRenameKeyRequest(renameKeyRequest)
.setCmdType(OzoneManagerProtocolProtos.Type.RenameKey)
.build();
} catch (IOException e) {
LOG.error("Couldn't get userinfo", e);
}
return omRequest;
}

private class RenameIterator extends OzoneListingIterator {
private final String srcPath;
private final String dstPath;
Expand Down Expand Up @@ -408,40 +485,37 @@ boolean processKeyPath(List<String> keyPathList) {
}
return true;
}
}

private OzoneManagerProtocolProtos.OMRequest
getRenameKeyRequest(
OFSPath src, OFSPath dst) {
String volumeName = src.getVolumeName();
String bucketName = src.getBucketName();
String keyName = src.getKeyName();

OzoneManagerProtocolProtos.KeyArgs keyArgs =
OzoneManagerProtocolProtos.KeyArgs.newBuilder()
.setKeyName(keyName)
.setVolumeName(volumeName)
.setBucketName(bucketName)
.build();
String toKeyName = dst.getKeyName();
OzoneManagerProtocolProtos.RenameKeyRequest renameKeyRequest =
OzoneManagerProtocolProtos.RenameKeyRequest.newBuilder()
.setKeyArgs(keyArgs)
.setToKeyName(toKeyName)
.build();
OzoneManagerProtocolProtos.OMRequest omRequest =
null;
try {
omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder()
.setClientId(CLIENT_ID.toString())
.setUserInfo(getUserInfo())
.setRenameKeyRequest(renameKeyRequest)
.setCmdType(OzoneManagerProtocolProtos.Type.RenameKey)
private OzoneManagerProtocolProtos.OMRequest getDeleteKeyRequest(
OFSPath srcPath) {
String volume = srcPath.getVolumeName();
String bucket = srcPath.getBucketName();
String key = srcPath.getKeyName();
OzoneManagerProtocolProtos.KeyArgs keyArgs =
OzoneManagerProtocolProtos.KeyArgs.newBuilder()
.setKeyName(key)
.setVolumeName(volume)
.setBucketName(bucket)
.setRecursive(true)
.build();
} catch (IOException e) {
LOG.error("Couldn't get userinfo", e);
}
return omRequest;
OzoneManagerProtocolProtos.DeleteKeyRequest deleteKeyRequest =
OzoneManagerProtocolProtos.DeleteKeyRequest.newBuilder()
.setKeyArgs(keyArgs).build();
OzoneManagerProtocolProtos.OMRequest omRequest =
null;
try {
omRequest =
OzoneManagerProtocolProtos.OMRequest.newBuilder()
.setClientId(CLIENT_ID.toString())
.setUserInfo(getUserInfo())
.setDeleteKeyRequest(deleteKeyRequest)
.setCmdType(OzoneManagerProtocolProtos.Type.DeleteKey)
.build();
} catch (IOException e) {
LOG.error("Couldn't get userinfo", e);
}
return omRequest;
}

private class DeleteIterator extends OzoneListingIterator {
Expand All @@ -467,7 +541,7 @@ boolean processKeyPath(List<String> keyPathList) {
for (String keyPath : keyPathList) {
OFSPath path = new OFSPath(keyPath);
OzoneManagerProtocolProtos.OMRequest omRequest =
getDeleteKeyRequest(path);
getDeleteKeysRequest(path);
try {
ozoneManager.getMetrics().incNumTrashFilesDeletes();
submitRequest(omRequest);
Expand All @@ -479,7 +553,7 @@ boolean processKeyPath(List<String> keyPathList) {
}

private OzoneManagerProtocolProtos.OMRequest
getDeleteKeyRequest(
getDeleteKeysRequest(
OFSPath keyPath) {
String volumeName = keyPath.getVolumeName();
String bucketName = keyPath.getBucketName();
Expand Down
15 changes: 2 additions & 13 deletions .../ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,12 @@
import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenRenewer;
Expand Down Expand Up @@ -548,17 +548,6 @@ private BlockLocation[] getBlockLocations(OzoneFileStatus fileStatus) {

@Override
public boolean isFSOptimizedBucket() {
// layout version V1 represents optimized FS path
boolean layoutVersionEnabled =
StringUtils.equalsIgnoreCase(
OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1,
bucket.getMetadata()
.get(OMConfigKeys.OZONE_OM_LAYOUT_VERSION));

boolean fsEnabled =
Boolean.parseBoolean(bucket.getMetadata()
.get(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS));

return layoutVersionEnabled && fsEnabled;
return OzoneFSUtils.isFSOptimizedBucket(bucket.getMetadata());
}
}