From a349813764895013ef52a2547cf97118db08c757 Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Wed, 26 Jan 2022 14:48:59 -0800 Subject: [PATCH 01/10] HADOOP-18110. ViewFileSystem: Add Support for Localized Trash Root --- .../apache/hadoop/fs/viewfs/Constants.java | 6 + .../hadoop/fs/viewfs/ViewFileSystem.java | 109 +++++++++++++++++- .../fs/viewfs/ViewFileSystemBaseTest.java | 103 +++++++++++++++++ 3 files changed, 216 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java index 8235e937bddbf..94b07973ac824 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java @@ -132,4 +132,10 @@ public interface Constants { Class DEFAULT_MOUNT_TABLE_CONFIG_LOADER_IMPL = HCFSMountTableConfigLoader.class; + + /** + * Enable ViewFileSystem to return a trashRoot which is local to mount point. + */ + String CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH = "fs.viewfs.mount.point.local.trash"; + boolean CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH_DEFAULT = false; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 538f03a1300a3..9999389dbc980 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -25,6 +25,8 @@ import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS; import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS_DEFAULT; import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH; +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH_DEFAULT; import java.util.function.Function; import java.io.FileNotFoundException; @@ -1126,20 +1128,67 @@ public Collection getAllStoragePolicies() return allPolicies; } + /** + * Check whether a path is inside a snapshot or EZ. + * @return true if a path is either in a snapshot or in an EZ + */ + private boolean snapshotOrEZPath(FileSystem fs, Path path) + throws IOException { + try { + FileStatus fileStatus = fs.getFileStatus(path); + return fileStatus.isEncrypted() || fileStatus.isSnapshotEnabled(); + } catch (FileNotFoundException e) { + // Return true if path does not exist + return false; + } + } + /** * Get the trash root directory for current user when the path * specified is deleted. * + * If CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH is not set to true, or + * when the path p is in a snapshot or an encryption zone, return + * the default trash root in user home dir. + * + * when CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH is set to true, + * 1) if path p is mounted from the same targetFS as user home dir, + * return a trash root in user home dir. + * 2) else, return a trash root in the mounted targetFS + * * @param path the trash root of the path to be determined. * @return the trash root path. */ @Override public Path getTrashRoot(Path path) { + boolean useMountPointLocalTrash = + config.getBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, + CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH_DEFAULT); + try { InodeTree.ResolveResult res = fsState.resolve(getUriPath(path), true); - return res.targetFileSystem.getTrashRoot(res.remainingPath); - } catch (Exception e) { + + Path trashRoot = res.targetFileSystem.getTrashRoot(res.remainingPath); + if (!useMountPointLocalTrash || snapshotOrEZPath(res.targetFileSystem, + trashRoot)) { + return trashRoot; + } else { + // If Path p is in the default volume, return a trash in home dir in + // default volume. + // When Path p is in the default volume, we don't have any match in + // mount points and thus couldn't resolve any component for Path p. + if (res.remainingPath.equals(path)) { + return new Path(res.targetFileSystem.getHomeDirectory(), + TRASH_PREFIX); + } else { + Path mountPointRoot = + res.targetFileSystem.getFileStatus(new Path("/")).getPath(); + return new Path(mountPointRoot, + TRASH_PREFIX + "/" + ugi.getShortUserName()); + } + } + } catch (IOException|IllegalArgumentException e) { throw new NotInMountpointException(path, "getTrashRoot"); } } @@ -1156,6 +1205,62 @@ public Collection getTrashRoots(boolean allUsers) { for (FileSystem fs : getChildFileSystems()) { trashRoots.addAll(fs.getTrashRoots(allUsers)); } + + // Add trash dirs for each mount point + boolean useMountPointLocalTrash = + config.getBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, + CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH_DEFAULT); + if (useMountPointLocalTrash) { + + Set currentTrashPaths = new HashSet<>(); + for (FileStatus file : trashRoots) { + currentTrashPaths.add(file.getPath()); + } + + MountPoint[] mountPoints = getMountPoints(); + try { + for (int i = 0; i < mountPoints.length; i++) { + Path trashRoot = makeQualified( + new Path(mountPoints[i].mountedOnPath + "/" + TRASH_PREFIX)); + + // Continue if trashRoot does not exist for this filesystem + if (!exists(trashRoot)) { + continue; + } + + InodeTree.ResolveResult res = + fsState.resolve(getUriPath(trashRoot), true); + + if (!allUsers) { + Path userTrash = + new Path("/" + TRASH_PREFIX + "/" + ugi.getShortUserName()); + try { + FileStatus file = res.targetFileSystem.getFileStatus(userTrash); + if (!currentTrashPaths.contains(file.getPath())) { + trashRoots.add(file); + currentTrashPaths.add(file.getPath()); + } + } catch (FileNotFoundException ignored) { + } + } else { + FileStatus[] targetFsTrashRoots = + res.targetFileSystem.listStatus(new Path("/" + TRASH_PREFIX)); + for (FileStatus file : targetFsTrashRoots) { + // skip if we already include it in currentTrashPaths + if (currentTrashPaths.contains(file.getPath())) { + continue; + } + + trashRoots.add(file); + currentTrashPaths.add(file.getPath()); + } + } + } + } catch (IOException e) { + LOG.warn("Exception in get all trash roots", e); + } + } + return trashRoots; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index 434eff0bef32e..78d824d643c3f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -69,6 +69,8 @@ import static org.apache.hadoop.fs.FileSystemTestHelper.*; import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE; import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH; +import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX; import org.junit.After; import org.junit.Assert; @@ -1102,6 +1104,107 @@ public void testTrashRoot() throws IOException { Assert.assertTrue("", fsView.getTrashRoots(true).size() > 0); } + /** + * Test the localized trash root for getTrashRoot. + */ + @Test + public void testTrashRootLocalizedTrash() throws IOException { + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + Configuration newConf = new Configuration(conf); + newConf.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true); + ConfigUtil.addLinkFallback(newConf, targetTestRoot.toUri()); + FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, newConf); + + // Case 1: path p not in the default FS. + // Return a trash root within the mount point. + Path dataTestPath = new Path("/data/dir/file"); + Path dataTrashRoot = new Path(targetTestRoot, + "data/" + TRASH_PREFIX + "/" + ugi.getShortUserName()); + Assert.assertEquals(dataTrashRoot, fsView2.getTrashRoot(dataTestPath)); + + // Case 2: path p not found in mount table, fall back to the default FS + // Return a trash root in user home dir + Path userTestPath = new Path("/nonExistingDir/nonExistingFile"); + Path userTrashRoot = new Path(fsTarget.getHomeDirectory(), TRASH_PREFIX); + Assert.assertEquals(userTrashRoot, fsView2.getTrashRoot(userTestPath)); + + // Case 3: turn off the CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH flag. + // Return a trash root in user home dir. + newConf.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, false); + fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, newConf); + Assert.assertEquals(userTrashRoot, fsView2.getTrashRoot(dataTestPath)); + } + + /** + * Test localized trash roots in getTrashRoots() for all users. + */ + @Test + public void testTrashRootsAllUsers() throws IOException { + Configuration conf2 = new Configuration(conf); + conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true); + FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); + + // Case 1: verify correct trash roots from fsView and fsView2 + int beforeTrashRootNum = fsView.getTrashRoots(true).size(); + int beforeTrashRootNum2 = fsView2.getTrashRoots(true).size(); + Assert.assertEquals(beforeTrashRootNum, beforeTrashRootNum2); + + fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/user1")); + fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/user2")); + fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/user3")); + fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/user4")); + fsView.mkdirs(new Path("/user2/" + TRASH_PREFIX + "/user5")); + int afterTrashRootsNum = fsView.getTrashRoots(true).size(); + int afterTrashRootsNum2 = fsView2.getTrashRoots(true).size(); + Assert.assertEquals(beforeTrashRootNum, afterTrashRootsNum); + Assert.assertEquals(beforeTrashRootNum2 + 5, afterTrashRootsNum2); + + // Case 2: per-user mount point + fsTarget.mkdirs(new Path(targetTestRoot, "Users/userA/.Trash/userA")); + Configuration conf3 = new Configuration(conf2); + ConfigUtil.addLink(conf3, "/Users/userA", + new Path(targetTestRoot, "Users/userA").toUri()); + FileSystem fsView3 = FileSystem.get(FsConstants.VIEWFS_URI, conf3); + int trashRootsNum3 = fsView3.getTrashRoots(true).size(); + Assert.assertEquals(afterTrashRootsNum2 + 1, trashRootsNum3); + + // Case 3: single /Users mount point for all users + fsTarget.mkdirs(new Path(targetTestRoot, "Users/.Trash/user1")); + fsTarget.mkdirs(new Path(targetTestRoot, "Users/.Trash/user2")); + Configuration conf4 = new Configuration(conf2); + ConfigUtil.addLink(conf4, "/Users", + new Path(targetTestRoot, "Users").toUri()); + FileSystem fsView4 = FileSystem.get(FsConstants.VIEWFS_URI, conf4); + int trashRootsNum4 = fsView4.getTrashRoots(true).size(); + Assert.assertEquals(afterTrashRootsNum2 + 2, trashRootsNum4); + } + + /** + * Test localized trash roots in getTrashRoots() for current user. + */ + @Test + public void testTrashRootsCurrentUser() throws IOException { + String currentUser = + UserGroupInformation.getCurrentUser().getShortUserName(); + Configuration conf2 = new Configuration(conf); + conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true); + FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); + + int beforeTrashRootNum = fsView.getTrashRoots(false).size(); + int beforeTrashRootNum2 = fsView2.getTrashRoots(false).size(); + Assert.assertEquals(beforeTrashRootNum, beforeTrashRootNum2); + + fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/" + currentUser)); + fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/user2")); + fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/" + currentUser)); + fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/user4")); + fsView.mkdirs(new Path("/user2/" + TRASH_PREFIX + "/user5")); + int afterTrashRootsNum = fsView.getTrashRoots(false).size(); + int afterTrashRootsNum2 = fsView2.getTrashRoots(false).size(); + Assert.assertEquals(beforeTrashRootNum, afterTrashRootsNum); + Assert.assertEquals(beforeTrashRootNum2 + 2, afterTrashRootsNum2); + } + @Test(expected = NotInMountpointException.class) public void testViewFileSystemUtil() throws Exception { Configuration newConf = new Configuration(conf); From cfc69157a5dc8d23248a266238e24c6c30d1e59e Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Fri, 4 Feb 2022 11:50:38 -0800 Subject: [PATCH 02/10] Use ROOT_PATH to determine whether a path is in a mount point or fallback FS. --- .../org/apache/hadoop/fs/viewfs/ViewFileSystem.java | 9 ++++----- .../hadoop/fs/viewfs/ViewFileSystemBaseTest.java | 12 ++++++++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 9999389dbc980..1a9e8c9259a44 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -1174,14 +1174,13 @@ public Path getTrashRoot(Path path) { trashRoot)) { return trashRoot; } else { - // If Path p is in the default volume, return a trash in home dir in - // default volume. - // When Path p is in the default volume, we don't have any match in - // mount points and thus couldn't resolve any component for Path p. - if (res.remainingPath.equals(path)) { + // Path p is either in a mount point or in the fallback FS + if (ROOT_PATH.equals(new Path(res.resolvedPath))) { + // Path p is in the fallback FS return new Path(res.targetFileSystem.getHomeDirectory(), TRASH_PREFIX); } else { + // Path p is in a mount point Path mountPointRoot = res.targetFileSystem.getFileStatus(new Path("/")).getPath(); return new Path(mountPointRoot, diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index 78d824d643c3f..e91e2804ded41 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -1124,15 +1124,23 @@ public void testTrashRootLocalizedTrash() throws IOException { // Case 2: path p not found in mount table, fall back to the default FS // Return a trash root in user home dir - Path userTestPath = new Path("/nonExistingDir/nonExistingFile"); + Path nonExistentPath = new Path("/nonExistentDir/nonExistentFile"); Path userTrashRoot = new Path(fsTarget.getHomeDirectory(), TRASH_PREFIX); - Assert.assertEquals(userTrashRoot, fsView2.getTrashRoot(userTestPath)); + Assert.assertEquals(userTrashRoot, fsView2.getTrashRoot(nonExistentPath)); // Case 3: turn off the CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH flag. // Return a trash root in user home dir. newConf.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, false); fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, newConf); Assert.assertEquals(userTrashRoot, fsView2.getTrashRoot(dataTestPath)); + + // Case 4: viewFS without fallback. Expect exception for a nonExistent path + newConf = new Configuration(conf); + fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, newConf); + try { + fsView2.getTrashRoot(nonExistentPath); + } catch (NotInMountpointException ignored) { + } } /** From 5afd5984af41f1b60ac51dad350d19b91c5f7518 Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Fri, 4 Feb 2022 11:52:30 -0800 Subject: [PATCH 03/10] Formatted code --- .../main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 1a9e8c9259a44..df9050bbb3452 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -1187,7 +1187,7 @@ public Path getTrashRoot(Path path) { TRASH_PREFIX + "/" + ugi.getShortUserName()); } } - } catch (IOException|IllegalArgumentException e) { + } catch (IOException | IllegalArgumentException e) { throw new NotInMountpointException(path, "getTrashRoot"); } } From 8d771ef292247ee457887b667d64f3ad8436e334 Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Tue, 8 Feb 2022 11:28:10 -0800 Subject: [PATCH 04/10] Removed special case for snapshots for Localized trash root --- .../org/apache/hadoop/fs/viewfs/ViewFileSystem.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index df9050bbb3452..e2562614517bc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -1129,14 +1129,13 @@ public Collection getAllStoragePolicies() } /** - * Check whether a path is inside a snapshot or EZ. - * @return true if a path is either in a snapshot or in an EZ + * Check whether a path is inside a encryption zone. + * @return true if a path is either in an encryption zone */ - private boolean snapshotOrEZPath(FileSystem fs, Path path) - throws IOException { + private boolean isEZPath(FileSystem fs, Path path) throws IOException { try { FileStatus fileStatus = fs.getFileStatus(path); - return fileStatus.isEncrypted() || fileStatus.isSnapshotEnabled(); + return fileStatus.isEncrypted(); } catch (FileNotFoundException e) { // Return true if path does not exist return false; @@ -1170,7 +1169,7 @@ public Path getTrashRoot(Path path) { fsState.resolve(getUriPath(path), true); Path trashRoot = res.targetFileSystem.getTrashRoot(res.remainingPath); - if (!useMountPointLocalTrash || snapshotOrEZPath(res.targetFileSystem, + if (!useMountPointLocalTrash || isEZPath(res.targetFileSystem, trashRoot)) { return trashRoot; } else { From 5b6f293a083321e511bc2ace208d469d77ee167f Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Tue, 8 Feb 2022 17:44:54 -0800 Subject: [PATCH 05/10] updated based on Owen's comment. --- .../hadoop/fs/viewfs/ViewFileSystem.java | 18 +++++++---- .../fs/viewfs/ViewFileSystemBaseTest.java | 32 ++++++++++++++----- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index e2562614517bc..8cb4430f8f776 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -1169,8 +1169,7 @@ public Path getTrashRoot(Path path) { fsState.resolve(getUriPath(path), true); Path trashRoot = res.targetFileSystem.getTrashRoot(res.remainingPath); - if (!useMountPointLocalTrash || isEZPath(res.targetFileSystem, - trashRoot)) { + if (!useMountPointLocalTrash) { return trashRoot; } else { // Path p is either in a mount point or in the fallback FS @@ -1180,10 +1179,17 @@ public Path getTrashRoot(Path path) { TRASH_PREFIX); } else { // Path p is in a mount point - Path mountPointRoot = - res.targetFileSystem.getFileStatus(new Path("/")).getPath(); - return new Path(mountPointRoot, - TRASH_PREFIX + "/" + ugi.getShortUserName()); + + if (trashRoot.toUri().getPath().startsWith(res.resolvedPath)) { + // targetFileSystem.trashRoot is in the same mount point as Path p + return trashRoot; + } else { + // targetFileSystem.trashRoot is in a different mount point from + // Path p. Return the trash root for the mount point. + Path mountPointRoot = + res.targetFileSystem.getFileStatus(new Path("/")).getPath(); + return new Path(mountPointRoot, TRASH_PREFIX + "/" + ugi.getShortUserName()); + } } } } catch (IOException | IllegalArgumentException e) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index e91e2804ded41..798f677e21e06 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -1110,10 +1110,10 @@ public void testTrashRoot() throws IOException { @Test public void testTrashRootLocalizedTrash() throws IOException { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - Configuration newConf = new Configuration(conf); - newConf.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true); - ConfigUtil.addLinkFallback(newConf, targetTestRoot.toUri()); - FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, newConf); + Configuration conf2 = new Configuration(conf); + conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true); + ConfigUtil.addLinkFallback(conf2, targetTestRoot.toUri()); + FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); // Case 1: path p not in the default FS. // Return a trash root within the mount point. @@ -1130,17 +1130,33 @@ public void testTrashRootLocalizedTrash() throws IOException { // Case 3: turn off the CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH flag. // Return a trash root in user home dir. - newConf.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, false); - fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, newConf); + conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, false); + fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); Assert.assertEquals(userTrashRoot, fsView2.getTrashRoot(dataTestPath)); // Case 4: viewFS without fallback. Expect exception for a nonExistent path - newConf = new Configuration(conf); - fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, newConf); + conf2 = new Configuration(conf); + fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); try { fsView2.getTrashRoot(nonExistentPath); } catch (NotInMountpointException ignored) { } + + // Case 5: path p is in the same mount point as targetFS.getTrashRoot(). + // Return targetFS.getTrashRoot() + // Use a new Configuration object, so that we can start with an empty + // mount table. This would avoid a conflict between the /user link in + // setupMountPoints() and homeDir we will need to setup for this test. + // default homeDir for hdfs is /user/. + Configuration conf3 = ViewFileSystemTestSetup.createConfig(); + Path homeDir = fsTarget.getHomeDirectory(); + String homeParentDir = homeDir.getParent().toUri().getPath(); + conf3.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true); + ConfigUtil.addLink(conf3, homeParentDir, + new Path(targetTestRoot, homeParentDir).toUri()); + Path homeTestPath = new Path(homeDir.toUri().getPath(), "testuser/file"); + FileSystem fsView3 = FileSystem.get(FsConstants.VIEWFS_URI, conf3); + Assert.assertEquals(userTrashRoot, fsView3.getTrashRoot(homeTestPath)); } /** From 6a5c4bde8bf0be6ae2b1b0187a9d309fda06dbff Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Tue, 8 Feb 2022 17:45:55 -0800 Subject: [PATCH 06/10] Removed isEZPath() check --- .../apache/hadoop/fs/viewfs/ViewFileSystem.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 8cb4430f8f776..6e40440015cb2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -1128,20 +1128,6 @@ public Collection getAllStoragePolicies() return allPolicies; } - /** - * Check whether a path is inside a encryption zone. - * @return true if a path is either in an encryption zone - */ - private boolean isEZPath(FileSystem fs, Path path) throws IOException { - try { - FileStatus fileStatus = fs.getFileStatus(path); - return fileStatus.isEncrypted(); - } catch (FileNotFoundException e) { - // Return true if path does not exist - return false; - } - } - /** * Get the trash root directory for current user when the path * specified is deleted. From c3745aa80bc514f879bd99edb2d0f08bc49090c1 Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Wed, 9 Feb 2022 08:59:21 -0800 Subject: [PATCH 07/10] Reorganized code for getTrashRoot --- .../hadoop/fs/viewfs/ViewFileSystem.java | 41 +++++++++---------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 6e40440015cb2..c4f53f6ec1ace 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -1132,14 +1132,14 @@ public Collection getAllStoragePolicies() * Get the trash root directory for current user when the path * specified is deleted. * - * If CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH is not set to true, or - * when the path p is in a snapshot or an encryption zone, return - * the default trash root in user home dir. + * If CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH is not set, return + * the default trash root from targetFS. * - * when CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH is set to true, - * 1) if path p is mounted from the same targetFS as user home dir, - * return a trash root in user home dir. + * When CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH is set to true, + * 1) If path p is in fallback FS or from the same mount point as the default + * trash root for targetFS, return the default trash root for targetFS. * 2) else, return a trash root in the mounted targetFS + * (/mntpoint/.Trash/) * * @param path the trash root of the path to be determined. * @return the trash root path. @@ -1159,23 +1159,20 @@ public Path getTrashRoot(Path path) { return trashRoot; } else { // Path p is either in a mount point or in the fallback FS - if (ROOT_PATH.equals(new Path(res.resolvedPath))) { - // Path p is in the fallback FS - return new Path(res.targetFileSystem.getHomeDirectory(), - TRASH_PREFIX); - } else { - // Path p is in a mount point - if (trashRoot.toUri().getPath().startsWith(res.resolvedPath)) { - // targetFileSystem.trashRoot is in the same mount point as Path p - return trashRoot; - } else { - // targetFileSystem.trashRoot is in a different mount point from - // Path p. Return the trash root for the mount point. - Path mountPointRoot = - res.targetFileSystem.getFileStatus(new Path("/")).getPath(); - return new Path(mountPointRoot, TRASH_PREFIX + "/" + ugi.getShortUserName()); - } + if (ROOT_PATH.equals(new Path(res.resolvedPath)) || trashRoot.toUri() + .getPath() + .startsWith(res.resolvedPath)) { + // Path p is in the fallback FS or targetFileSystem.trashRoot is in + // the same mount point as Path p + return trashRoot; + } else { + // targetFileSystem.trashRoot is in a different mount point from + // Path p. Return the trash root for the mount point. + Path mountPointRoot = + res.targetFileSystem.getFileStatus(new Path("/")).getPath(); + return new Path(mountPointRoot, + TRASH_PREFIX + "/" + ugi.getShortUserName()); } } } catch (IOException | IllegalArgumentException e) { From bc232713e7e503b4a802ab303624ce3dbf78efd0 Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Wed, 9 Feb 2022 09:17:26 -0800 Subject: [PATCH 08/10] Fixed unnecessary new lines from code formatting --- .../java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index c4f53f6ec1ace..ddffcf962a962 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -1160,9 +1160,8 @@ public Path getTrashRoot(Path path) { } else { // Path p is either in a mount point or in the fallback FS - if (ROOT_PATH.equals(new Path(res.resolvedPath)) || trashRoot.toUri() - .getPath() - .startsWith(res.resolvedPath)) { + if (ROOT_PATH.equals(new Path(res.resolvedPath)) + || trashRoot.toUri().getPath().startsWith(res.resolvedPath)) { // Path p is in the fallback FS or targetFileSystem.trashRoot is in // the same mount point as Path p return trashRoot; From 250b6c8490ebb9cb90629d21ad62df0944f49105 Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Wed, 9 Feb 2022 21:34:58 -0800 Subject: [PATCH 09/10] Fixed the javadoc complaint. --- .../main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index ddffcf962a962..5ff3c2b70abfe 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -1139,7 +1139,7 @@ public Collection getAllStoragePolicies() * 1) If path p is in fallback FS or from the same mount point as the default * trash root for targetFS, return the default trash root for targetFS. * 2) else, return a trash root in the mounted targetFS - * (/mntpoint/.Trash/) + * (/mntpoint/.Trash/{user}) * * @param path the trash root of the path to be determined. * @return the trash root path. From 44da7860bf354e5ddda7c09aa0567af680effd5f Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Thu, 10 Feb 2022 15:55:22 -0800 Subject: [PATCH 10/10] Added a unit test using mockfs to return a deep trash root --- .../fs/viewfs/ViewFileSystemBaseTest.java | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index 798f677e21e06..91a90751e873e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -1159,6 +1159,35 @@ public void testTrashRootLocalizedTrash() throws IOException { Assert.assertEquals(userTrashRoot, fsView3.getTrashRoot(homeTestPath)); } + /** + * A mocked FileSystem which returns a deep trash dir. + */ + static class MockTrashRootFS extends MockFileSystem { + public static final Path TRASH = + new Path("/mnt/very/deep/deep/trash/dir/.Trash"); + + @Override + public Path getTrashRoot(Path path) { + return TRASH; + } + } + + /** + * Test a trash root that is inside a mount point for getTrashRoot + */ + @Test + public void testTrashRootDeepTrashDir() throws IOException { + + Configuration conf2 = ViewFileSystemTestSetup.createConfig(); + conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true); + conf2.setClass("fs.mocktrashfs.impl", MockTrashRootFS.class, + FileSystem.class); + ConfigUtil.addLink(conf2, "/mnt", URI.create("mocktrashfs://mnt/path")); + Path testPath = new Path(MockTrashRootFS.TRASH, "projs/proj"); + FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); + Assert.assertEquals(MockTrashRootFS.TRASH, fsView2.getTrashRoot(testPath)); + } + /** * Test localized trash roots in getTrashRoots() for all users. */