Skip to content

HDFS-15635. ViewFileSystemOverloadScheme support specifying mount table loader imp through conf #2389

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
umamaheswararao merged 4 commits into from
Nov 20, 2020
Merged

HDFS-15635. ViewFileSystemOverloadScheme support specifying mount table loader imp through conf #2389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2a62f39
HDFS-15635
zuston Oct 16, 2020
d770fc3
HDFS-15635 v2
zuston Nov 2, 2020
dd751d9
HDFS-15635 v3
zuston Nov 18, 2020
876af05
HDFS-15635 v4
zuston Nov 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,4 +125,11 @@ public interface Constants {
"fs.viewfs.ignore.port.in.mount.table.name";

boolean CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME_DEFAULT = false;

String CONFIG_VIEWFS_MOUNTTABLE_LOADER_IMPL =
CONFIG_VIEWFS_PREFIX + ".config.loader.impl";

Class<? extends MountTableConfigLoader>
DEFAULT_MOUNT_TABLE_CONFIG_LOADER_IMPL =
HCFSMountTableConfigLoader.class;
}
33 changes: 30 additions & 3 deletions ...hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
*/
package org.apache.hadoop.fs.viewfs;

import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME;
import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNTTABLE_LOADER_IMPL;
import static org.apache.hadoop.fs.viewfs.Constants.DEFAULT_MOUNT_TABLE_CONFIG_LOADER_IMPL;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.Constructor;
Expand All @@ -30,8 +34,7 @@
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;

import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME;
import org.apache.hadoop.util.ReflectionUtils;

/**
* <p> This class is extended from the ViewFileSystem for the overloaded
Expand Down Expand Up @@ -160,7 +163,7 @@ public void initialize(URI theUri, Configuration conf) throws IOException {
conf.getBoolean(Constants.CONFIG_VIEWFS_IGNORE_PORT_IN_MOUNT_TABLE_NAME,
true));
if (null != mountTableConfigPath) {
MountTableConfigLoader loader = new HCFSMountTableConfigLoader();
MountTableConfigLoader loader = getMountTableConfigLoader(conf);
loader.load(mountTableConfigPath, conf);
} else {
// TODO: Should we fail here.?
Expand All @@ -173,6 +176,30 @@ public void initialize(URI theUri, Configuration conf) throws IOException {
super.initialize(theUri, conf);
}

private MountTableConfigLoader getMountTableConfigLoader(
final Configuration conf) {
Class<? extends MountTableConfigLoader> clazz =
conf.getClass(CONFIG_VIEWFS_MOUNTTABLE_LOADER_IMPL,
DEFAULT_MOUNT_TABLE_CONFIG_LOADER_IMPL,
MountTableConfigLoader.class);

if (clazz == null) {
throw new RuntimeException(
String.format("Errors on getting mount table loader class. "
+ "The fs.viewfs.mounttable.config.loader.impl conf is %s. ",
conf.get(CONFIG_VIEWFS_MOUNTTABLE_LOADER_IMPL,
DEFAULT_MOUNT_TABLE_CONFIG_LOADER_IMPL.getName())));
}

try {
MountTableConfigLoader mountTableConfigLoader =
ReflectionUtils.newInstance(clazz, conf);
return mountTableConfigLoader;
} catch (Exception e) {
throw new RuntimeException(e);
}
}

/**
* This method is overridden because in ViewFileSystemOverloadScheme if
* overloaded scheme matches with mounted target fs scheme, file system
Expand Down