Skip to content

HDFS-16008. RBF: Tool to initialize ViewFS Mapping to Router #2981

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
zhuxiangyi wants to merge 11 commits into from
Closed

HDFS-16008. RBF: Tool to initialize ViewFS Mapping to Router #2981

Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
ef407a8
HDFS-16008. Initialize the ViewFS Maping tool to the Router
May 6, 2021
9a7c9f1
Address comments and add unit tests
May 6, 2021
e0b806f
Address comments and add doc
May 7, 2021
7fa083a
fix checkstyle of blanks
May 7, 2021
3be7673
Address comments
May 8, 2021
c33ba43
Address comments: support for all clusters
May 11, 2021
698a3a9
Address comments
May 12, 2021
3370147
Address comments
May 13, 2021
93dbc90
Address comments
May 13, 2021
0204cdc
Fix doc formatting errors
May 13, 2021
bc5089e
fix checkstyle
May 13, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 67 additions & 4 deletions ...ct/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.viewfs.Constants;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
Expand Down Expand Up @@ -132,7 +136,7 @@ private String getUsage(String cmd) {
{"-add", "-update", "-rm", "-ls", "-getDestination", "-setQuota",
"-setStorageTypeQuota", "-clrQuota", "-clrStorageTypeQuota",
"-safemode", "-nameservice", "-getDisabledNameservices",
"-refresh", "-refreshRouterArgs",
"-refresh", "-initViewFsToMountTable", "-refreshRouterArgs",
"-refreshSuperUserGroupsConfiguration"};
StringBuilder usage = new StringBuilder();
usage.append("Usage: hdfs dfsrouteradmin :\n");
Expand Down Expand Up @@ -171,7 +175,10 @@ private String getUsage(String cmd) {
return "\t[-clrQuota <path>]";
} else if (cmd.equals("-clrStorageTypeQuota")) {
return "\t[-clrStorageTypeQuota <path>]";
} else if (cmd.equals("-safemode")) {
} else if (cmd.equals("-initViewFsToMountTable")) {
return "\t[-initViewFsToMountTable <clusterName>," +
"-initViewFsToMountTable ClusterX]";
}else if (cmd.equals("-safemode")) {
return "\t[-safemode enter | leave | get]";
} else if (cmd.equals("-nameservice")) {
return "\t[-nameservice enable | disable <nameservice>]";
Expand Down Expand Up @@ -384,7 +391,14 @@ public int run(String[] argv) throws Exception {
getDisabledNameservices();
} else if ("-refresh".equals(cmd)) {
refresh(address);
} else if ("-refreshRouterArgs".equals(cmd)) {
} else if ("-initViewFsToMountTable".equals(cmd)) {
if (initViewFsToMountTable(argv, i)) {
System.out.println("Successfully init ViewFs mapping to router " + argv[i]);
} else {
exitCode = -1;
}
}
else if ("-refreshRouterArgs".equals(cmd)) {
exitCode = genericRefresh(argv, i);
} else if ("-refreshSuperUserGroupsConfiguration".equals(cmd)) {
exitCode = refreshSuperUserGroupsConfiguration();
Expand Down Expand Up @@ -1035,7 +1049,56 @@ private boolean updateQuota(String mount, long nsQuota, long ssQuota)
.updateMountTableEntry(updateRequest);
return updateResponse.getStatus();
}


/**
* initViewFsToMountTable.
*
* @param parameters The specified cluster to initialize.
* @param i Index in the parameters
* @return If the quota was updated.
* @throws IOException Error adding the mount point.
*/
public boolean initViewFsToMountTable(String[] parameters, int i) throws IOException {
String clusterName = parameters[i++];
if (clusterName == null) {
System.out.println("Please enter the cluster name.");
return false;
}
final String mountTablePrefix =
Constants.CONFIG_VIEWFS_PREFIX + "." + clusterName + "." +
Constants.CONFIG_VIEWFS_LINK + "./";
Map<String, String> viewFsMap = getConf().getValByRegex(mountTablePrefix);
if (viewFsMap.size() == 0) {
System.out.println("Please check the cluster name and veiwfs " +
"configuration.");
}
for (String key : viewFsMap.keySet()) {
Path path = new Path(viewFsMap.get(key));
String owner = null;
String group = null;
FsPermission mode = null;
try {
FileSystem fs = path.getFileSystem(getConf());
if (fs.exists(path)) {
FileStatus fileStatus = fs.getFileStatus(path);
owner = fileStatus.getOwner();
group = fileStatus.getGroup();
mode = fileStatus.getPermission();
}
} catch (Exception e) {
LOG.warn("Exception encountered", e);
}
DestinationOrder order = DestinationOrder.HASH;
String mount =
key.split(clusterName + "." + Constants.CONFIG_VIEWFS_LINK + ".")[1];
String dest = path.toUri().getPath();
Copy link
Member

@goiri goiri May 6, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Extract path.toUri()

String[] nss = new String[]{path.toUri().getAuthority()};
addMount(mount, nss, dest, false, false, order,
new ACLEntity(owner, group, mode));
}
return true;
}

/**
* Update storage type quota of specified mount table.
*
Expand Down