diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java index dbcb8218f340..cda335d6ba41 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java @@ -22,6 +22,7 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; @@ -35,7 +36,6 @@ import org.apache.hadoop.fs.TrashPolicy; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.RatisReplicationConfig; @@ -89,10 +89,10 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; -import java.util.HashMap; import java.util.Optional; import java.util.Random; import java.util.Set; @@ -318,6 +318,44 @@ public void testCreateDoesNotAddParentDirKeys() throws Exception { fs.delete(grandparent, true); } + @Test + public void testListStatusWithIntermediateDirWithECEnabled() + throws Exception { + String key = "object-dir/object-name1"; + + // write some test data into bucket + try (OzoneOutputStream outputStream = objectStore.getVolume(volumeName). + getBucket(bucketName).createKey(key, 1, + new ECReplicationConfig("RS-3-2-1024"), + new HashMap<>())) { + outputStream.write(RandomUtils.nextBytes(1)); + } + + List dirs = Arrays.asList(volumeName, bucketName, "object-dir", + "object-name1"); + for (int size = 1; size <= dirs.size(); size++) { + String path = "/" + dirs.subList(0, size).stream() + .collect(Collectors.joining("/")); + Path parent = new Path(path); + // Wait until the filestatus is updated + if (!enabledFileSystemPaths) { + GenericTestUtils.waitFor(() -> { + try { + fs.getFileStatus(parent); + return true; + } catch (IOException e) { + return false; + } + }, 1000, 120000); + } + FileStatus fileStatus = fs.getFileStatus(parent); + Assert.assertEquals((size == dirs.size() - 1 && + !bucketLayout.isFileSystemOptimized()) || size == dirs.size(), + fileStatus.isErasureCoded()); + } + + } + @Test public void testDeleteCreatesFakeParentDir() throws Exception { // TODO: Request for comment. diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index 40ed07e55405..3819233a4cf6 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -1056,7 +1056,7 @@ private static FileStatusAdapter getFileStatusAdapterForVolume( return new FileStatusAdapter(0L, 0L, path, true, (short)0, 0L, ozoneVolume.getCreationTime().getEpochSecond() * 1000, 0L, FsPermission.getDirDefault().toShort(), - owner, group, path, new BlockLocation[0], false, false + owner, group, null, new BlockLocation[0], false, false ); } @@ -1082,7 +1082,7 @@ private static FileStatusAdapter getFileStatusAdapterForBucket( return new FileStatusAdapter(0L, 0L, path, true, (short)0, 0L, ozoneBucket.getCreationTime().getEpochSecond() * 1000, 0L, FsPermission.getDirDefault().toShort(), - owner, group, path, new BlockLocation[0], + owner, group, null, new BlockLocation[0], !StringUtils.isEmpty(ozoneBucket.getEncryptionKeyName()), ozoneBucket.getReplicationConfig() != null && ozoneBucket.getReplicationConfig().getReplicationType() == diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index dc7c12ed9b83..2a757a1e601b 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -1323,28 +1323,27 @@ public boolean isNumber(String number) { return true; } - FileStatus convertFileStatus(FileStatusAdapter fileStatusAdapter) { - Path symLink = null; - try { - fileStatusAdapter.getSymlink(); - } catch (Exception ex) { - //NOOP: If not symlink symlink remains null. - } - - FileStatus fileStatus = new FileStatus( - fileStatusAdapter.getLength(), - fileStatusAdapter.isDir(), - fileStatusAdapter.getBlockReplication(), - fileStatusAdapter.getBlocksize(), - fileStatusAdapter.getModificationTime(), - fileStatusAdapter.getAccessTime(), - new FsPermission(fileStatusAdapter.getPermission()), - fileStatusAdapter.getOwner(), - fileStatusAdapter.getGroup(), - symLink, - fileStatusAdapter.getPath() + protected FileStatus constructFileStatus( + FileStatusAdapter fileStatusAdapter) { + return new FileStatus(fileStatusAdapter.getLength(), + fileStatusAdapter.isDir(), + fileStatusAdapter.getBlockReplication(), + fileStatusAdapter.getBlocksize(), + fileStatusAdapter.getModificationTime(), + fileStatusAdapter.getAccessTime(), + new FsPermission(fileStatusAdapter.getPermission()), + fileStatusAdapter.getOwner(), + fileStatusAdapter.getGroup(), + fileStatusAdapter.getSymlink(), + fileStatusAdapter.getPath(), + false, + fileStatusAdapter.isEncrypted(), + fileStatusAdapter.isErasureCoded() ); + } + FileStatus convertFileStatus(FileStatusAdapter fileStatusAdapter) { + FileStatus fileStatus = constructFileStatus(fileStatusAdapter); BlockLocation[] blockLocations = fileStatusAdapter.getBlockLocations(); if (blockLocations == null || blockLocations.length == 0) { return fileStatus; diff --git a/hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java index 67aa705ab80f..249b2767bde9 100644 --- a/hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java @@ -17,9 +17,27 @@ */ package org.apache.hadoop.fs.ozone; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.permission.FsPermission; + /** * Minimal Rooted Ozone File System compatible with Hadoop 2.x. */ public class RootedOzoneFileSystem extends BasicRootedOzoneFileSystem { - + @Override + protected FileStatus constructFileStatus( + FileStatusAdapter fileStatusAdapter) { + return new FileStatus(fileStatusAdapter.getLength(), + fileStatusAdapter.isDir(), + fileStatusAdapter.getBlockReplication(), + fileStatusAdapter.getBlocksize(), + fileStatusAdapter.getModificationTime(), + fileStatusAdapter.getAccessTime(), + new FsPermission(fileStatusAdapter.getPermission()), + fileStatusAdapter.getOwner(), + fileStatusAdapter.getGroup(), + fileStatusAdapter.getSymlink(), + fileStatusAdapter.getPath() + ); + } }