From 7cd6768fa3581941be3fd40e6c6c8048655183b5 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 8 Dec 2022 17:03:01 -0700 Subject: [PATCH 1/4] HDDS-7603. Enable Erasure coding and encryption flag on FileStatus for ofs --- .../fs/ozone/TestRootedOzoneFileSystem.java | 43 +++++++++++++++++++ .../BasicRootedOzoneClientAdapterImpl.java | 4 +- .../fs/ozone/BasicRootedOzoneFileSystem.java | 39 ++++++++--------- 3 files changed, 64 insertions(+), 22 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java index dbcb8218f340..8b73e7b571c4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java @@ -63,7 +63,10 @@ import org.apache.hadoop.ozone.om.TrashPolicyOzone; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.QuotaUtil; +import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType; import org.apache.hadoop.ozone.security.acl.OzoneAclConfig; @@ -318,6 +321,46 @@ public void testCreateDoesNotAddParentDirKeys() throws Exception { fs.delete(grandparent, true); } + @Test + public void testListStatusWithIntermediateDirWithECEnabled() + throws Exception { + String key = "object-dir/object-name1"; + Path volPathTest = new Path(OZONE_URI_DELIMITER, volumeName); + Path bucketPathTest = new Path(volPathTest, bucketName); + + // write some test data into bucket + try (OzoneOutputStream outputStream = objectStore.getVolume(volumeName). + getBucket(bucketName).createKey(key, 1, + new ECReplicationConfig("RS-3-2-1024"), + new HashMap<>())) { + outputStream.write(RandomUtils.nextBytes(1)); + } + + List dirs = Arrays.asList(volumeName, bucketName, "object-dir", + "object-name1"); + for (int size = 1; size <= dirs.size(); size++) { + String path = "/" + dirs.subList(0, size).stream() + .collect(Collectors.joining("/")); + Path parent = new Path(path); + // Wait until the filestatus is updated + if (!enabledFileSystemPaths) { + GenericTestUtils.waitFor(() -> { + try { + fs.getFileStatus(parent); + return true; + } catch (IOException e) { + return false; + } + }, 1000, 120000); + } + FileStatus fileStatus = fs.getFileStatus(parent); + Assert.assertEquals(fileStatus.isErasureCoded(), + size >= dirs.size() - 1 + && !bucketLayout.isFileSystemOptimized()); + } + + } + @Test public void testDeleteCreatesFakeParentDir() throws Exception { // TODO: Request for comment. diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index 40ed07e55405..3819233a4cf6 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -1056,7 +1056,7 @@ private static FileStatusAdapter getFileStatusAdapterForVolume( return new FileStatusAdapter(0L, 0L, path, true, (short)0, 0L, ozoneVolume.getCreationTime().getEpochSecond() * 1000, 0L, FsPermission.getDirDefault().toShort(), - owner, group, path, new BlockLocation[0], false, false + owner, group, null, new BlockLocation[0], false, false ); } @@ -1082,7 +1082,7 @@ private static FileStatusAdapter getFileStatusAdapterForBucket( return new FileStatusAdapter(0L, 0L, path, true, (short)0, 0L, ozoneBucket.getCreationTime().getEpochSecond() * 1000, 0L, FsPermission.getDirDefault().toShort(), - owner, group, path, new BlockLocation[0], + owner, group, null, new BlockLocation[0], !StringUtils.isEmpty(ozoneBucket.getEncryptionKeyName()), ozoneBucket.getReplicationConfig() != null && ozoneBucket.getReplicationConfig().getReplicationType() == diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index dc7c12ed9b83..2a757a1e601b 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -1323,28 +1323,27 @@ public boolean isNumber(String number) { return true; } - FileStatus convertFileStatus(FileStatusAdapter fileStatusAdapter) { - Path symLink = null; - try { - fileStatusAdapter.getSymlink(); - } catch (Exception ex) { - //NOOP: If not symlink symlink remains null. - } - - FileStatus fileStatus = new FileStatus( - fileStatusAdapter.getLength(), - fileStatusAdapter.isDir(), - fileStatusAdapter.getBlockReplication(), - fileStatusAdapter.getBlocksize(), - fileStatusAdapter.getModificationTime(), - fileStatusAdapter.getAccessTime(), - new FsPermission(fileStatusAdapter.getPermission()), - fileStatusAdapter.getOwner(), - fileStatusAdapter.getGroup(), - symLink, - fileStatusAdapter.getPath() + protected FileStatus constructFileStatus( + FileStatusAdapter fileStatusAdapter) { + return new FileStatus(fileStatusAdapter.getLength(), + fileStatusAdapter.isDir(), + fileStatusAdapter.getBlockReplication(), + fileStatusAdapter.getBlocksize(), + fileStatusAdapter.getModificationTime(), + fileStatusAdapter.getAccessTime(), + new FsPermission(fileStatusAdapter.getPermission()), + fileStatusAdapter.getOwner(), + fileStatusAdapter.getGroup(), + fileStatusAdapter.getSymlink(), + fileStatusAdapter.getPath(), + false, + fileStatusAdapter.isEncrypted(), + fileStatusAdapter.isErasureCoded() ); + } + FileStatus convertFileStatus(FileStatusAdapter fileStatusAdapter) { + FileStatus fileStatus = constructFileStatus(fileStatusAdapter); BlockLocation[] blockLocations = fileStatusAdapter.getBlockLocations(); if (blockLocations == null || blockLocations.length == 0) { return fileStatus; From 47dba66d1403207e0e3e6e5818d03c7666784167 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 8 Dec 2022 17:10:11 -0700 Subject: [PATCH 2/4] HDDS-7603. Fix checkstyle issue --- .../apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java index 8b73e7b571c4..25f9835ffa0b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java @@ -22,6 +22,7 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; @@ -35,7 +36,6 @@ import org.apache.hadoop.fs.TrashPolicy; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.RatisReplicationConfig; @@ -63,10 +63,7 @@ import org.apache.hadoop.ozone.om.TrashPolicyOzone; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; -import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.QuotaUtil; -import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType; import org.apache.hadoop.ozone.security.acl.OzoneAclConfig; @@ -92,10 +89,10 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; -import java.util.HashMap; import java.util.Optional; import java.util.Random; import java.util.Set; From 1eea65ad5d5a95906f9ac7919a04950d3f3b1e95 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 9 Dec 2022 12:46:46 -0700 Subject: [PATCH 3/4] HDDS-7603. Set default False EC & Encrypt Flag for Hadoop2 --- .../fs/ozone/TestRootedOzoneFileSystem.java | 2 -- .../fs/ozone/RootedOzoneFileSystem.java | 20 ++++++++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java index 25f9835ffa0b..d1f75cb05c54 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java @@ -322,8 +322,6 @@ public void testCreateDoesNotAddParentDirKeys() throws Exception { public void testListStatusWithIntermediateDirWithECEnabled() throws Exception { String key = "object-dir/object-name1"; - Path volPathTest = new Path(OZONE_URI_DELIMITER, volumeName); - Path bucketPathTest = new Path(volPathTest, bucketName); // write some test data into bucket try (OzoneOutputStream outputStream = objectStore.getVolume(volumeName). diff --git a/hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java index 67aa705ab80f..249b2767bde9 100644 --- a/hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java @@ -17,9 +17,27 @@ */ package org.apache.hadoop.fs.ozone; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.permission.FsPermission; + /** * Minimal Rooted Ozone File System compatible with Hadoop 2.x. */ public class RootedOzoneFileSystem extends BasicRootedOzoneFileSystem { - + @Override + protected FileStatus constructFileStatus( + FileStatusAdapter fileStatusAdapter) { + return new FileStatus(fileStatusAdapter.getLength(), + fileStatusAdapter.isDir(), + fileStatusAdapter.getBlockReplication(), + fileStatusAdapter.getBlocksize(), + fileStatusAdapter.getModificationTime(), + fileStatusAdapter.getAccessTime(), + new FsPermission(fileStatusAdapter.getPermission()), + fileStatusAdapter.getOwner(), + fileStatusAdapter.getGroup(), + fileStatusAdapter.getSymlink(), + fileStatusAdapter.getPath() + ); + } } From 1c47675f894d27d548f74c707aa0f0129e1a168b Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 9 Dec 2022 14:45:56 -0700 Subject: [PATCH 4/4] HDDS-7603. Fix FSO Rooted Filesystem test --- .../apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java index d1f75cb05c54..cda335d6ba41 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java @@ -349,9 +349,9 @@ public void testListStatusWithIntermediateDirWithECEnabled() }, 1000, 120000); } FileStatus fileStatus = fs.getFileStatus(parent); - Assert.assertEquals(fileStatus.isErasureCoded(), - size >= dirs.size() - 1 - && !bucketLayout.isFileSystemOptimized()); + Assert.assertEquals((size == dirs.size() - 1 && + !bucketLayout.isFileSystemOptimized()) || size == dirs.size(), + fileStatus.isErasureCoded()); } }