From d0136d3f92f35a662415481cfa52d09b356595c8 Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Thu, 20 Aug 2020 16:16:12 -0700 Subject: [PATCH 1/3] HDFS-15529: getChildFilesystems should include fallback fs as well --- .../apache/hadoop/fs/viewfs/ViewFileSystem.java | 6 ++++++ ...ewFileSystemOverloadSchemeWithHdfsScheme.java | 16 +++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 8c659d1b15ed6..529b2d970f82d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -939,6 +939,12 @@ public FileSystem[] getChildFileSystems() { FileSystem targetFs = mountPoint.target.targetFileSystem; children.addAll(Arrays.asList(targetFs.getChildFileSystems())); } + + if(fsState.getRootFallbackLink()!=null) { + children.addAll(Arrays.asList( + fsState.getRootFallbackLink().targetFileSystem + .getChildFileSystems())); + } return children.toArray(new FileSystem[]{}); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java index 31674f8d1364e..0116190ebc661 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java @@ -476,10 +476,18 @@ public void testViewFsOverloadSchemeWithInnerCache() // 2. Two hdfs file systems should be there if no cache. conf.setBoolean(Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE, false); try (FileSystem vfs = FileSystem.get(conf)) { - Assert.assertEquals(2, vfs.getChildFileSystems().length); + Assert.assertEquals(isFallBackExist(conf) ? 3 : 2, + vfs.getChildFileSystems().length); } } + // HDFS-15529: if any extended tests added fallback, then getChildFileSystems + // will include fallback as well. + private boolean isFallBackExist(Configuration conf) { + return conf.get(ConfigUtil.getConfigViewFsPrefix(defaultFSURI + .getAuthority()) + "." + Constants.CONFIG_VIEWFS_LINK_FALLBACK) != null; + } + /** * Create mount links as follows * hdfs://localhost:xxx/HDFSUser0 --> hdfs://localhost:xxx/HDFSUser/ @@ -501,7 +509,8 @@ public void testViewFsOverloadSchemeWithNoInnerCacheAndHdfsTargets() conf.setBoolean(Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE, false); // Two hdfs file systems should be there if no cache. try (FileSystem vfs = FileSystem.get(conf)) { - Assert.assertEquals(2, vfs.getChildFileSystems().length); + Assert.assertEquals(isFallBackExist(conf) ? 3 : 2, + vfs.getChildFileSystems().length); } } @@ -528,7 +537,8 @@ public void testViewFsOverloadSchemeWithNoInnerCacheAndLocalSchemeTargets() // cache should work. conf.setBoolean(Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE, false); try (FileSystem vfs = FileSystem.get(conf)) { - Assert.assertEquals(1, vfs.getChildFileSystems().length); + Assert.assertEquals(isFallBackExist(conf) ? 2 : 1, + vfs.getChildFileSystems().length); } } From a36b4e706c49cd85a63063497238b0d1199507cd Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Sun, 30 Aug 2020 10:37:12 -0700 Subject: [PATCH 2/3] Fixed a test failure. --- .../main/java/org/apache/hadoop/fs/viewfs/InodeTree.java | 9 +++++++++ .../java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java index 003694f2e9918..1f1adea6dbe9c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java @@ -394,6 +394,15 @@ private boolean hasFallbackLink() { return rootFallbackLink != null; } + /** + * @return true if the root represented as internalDir. In LinkMergeSlash, + * there will be root to root mapping. So, root does not represent as + * internalDir. + */ + protected boolean isRootInternalDir() { + return root.isInternalDir(); + } + protected INodeLink getRootFallbackLink() { Preconditions.checkState(root.isInternalDir()); return rootFallbackLink; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 529b2d970f82d..1ba91b5edfa1c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -940,7 +940,7 @@ public FileSystem[] getChildFileSystems() { children.addAll(Arrays.asList(targetFs.getChildFileSystems())); } - if(fsState.getRootFallbackLink()!=null) { + if (fsState.isRootInternalDir() && fsState.getRootFallbackLink() != null) { children.addAll(Arrays.asList( fsState.getRootFallbackLink().targetFileSystem .getChildFileSystems())); From 71665bc1b4f7e716e2831eeb8cdfb99fc71eed7e Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Tue, 1 Sep 2020 18:53:39 -0700 Subject: [PATCH 3/3] Fixed a checkstyle comment. --- .../TestViewFileSystemOverloadSchemeWithHdfsScheme.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java index 0116190ebc661..9a858e17ebe4c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java @@ -483,8 +483,8 @@ public void testViewFsOverloadSchemeWithInnerCache() // HDFS-15529: if any extended tests added fallback, then getChildFileSystems // will include fallback as well. - private boolean isFallBackExist(Configuration conf) { - return conf.get(ConfigUtil.getConfigViewFsPrefix(defaultFSURI + private boolean isFallBackExist(Configuration config) { + return config.get(ConfigUtil.getConfigViewFsPrefix(defaultFSURI .getAuthority()) + "." + Constants.CONFIG_VIEWFS_LINK_FALLBACK) != null; }