Skip to content

YARN-3657. Federation maintenance mechanisms (simple CLI and command propagation) #5348

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
goiri merged 2 commits into from
Feb 6, 2023
Merged

YARN-3657. Federation maintenance mechanisms (simple CLI and command propagation) #5348

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f9eb028
YARN-3657. Federation maintenance mechanisms (simple CLI and command …
Feb 4, 2023
46ae88e
Merge branch 'apache:trunk' into YARN-3657
slfan1989 Feb 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions hadoop-yarn-project/hadoop-yarn/bin/yarn
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ function hadoop_usage
hadoop_add_subcommand "fs2cs" client "converts Fair Scheduler configuration to Capacity Scheduler (EXPERIMENTAL)"
hadoop_add_subcommand "rmadmin" admin "admin tools"
hadoop_add_subcommand "router" daemon "run the Router daemon"
hadoop_add_subcommand "routeradmin" admin "router admin tools"
hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration"
hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon"
Expand Down Expand Up @@ -180,6 +181,9 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
HADOOP_HEAPSIZE_MAX="${YARN_ROUTER_HEAPSIZE}"
fi
;;
routeradmin)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.RouterCLI'
;;
schedulerconf)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.SchedConfCLI'
;;
Expand Down
6 changes: 6 additions & 0 deletions hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,11 @@ goto :eof
)
goto :eof

:routeradmin
set CLASS=org.apache.hadoop.yarn.client.cli.RouterCLI
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
goto :eof

:nodemanager
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\nm-config\log4j.properties
set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\*
Expand Down Expand Up @@ -342,6 +347,7 @@ goto :eof
@echo resourcemanager run the ResourceManager
@echo nodemanager run a nodemanager on each slave
@echo router run the Router daemon
@echo routeradmin router admin tools
@echo timelineserver run the timeline server
@echo timelinereader run the timeline reader server
@echo rmadmin admin tools
Expand Down
93 changes: 93 additions & 0 deletions ...op-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.client.cli;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.client.ClientRMProxy;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;

import java.io.IOException;

public class RouterCLI extends Configured implements Tool {

public RouterCLI() {
super();
}

public RouterCLI(Configuration conf) {
super(conf);
}

private static void printHelp() {
StringBuilder summary = new StringBuilder();
summary.append("router-admin is the command to execute " +
"YARN Federation administrative commands.\n");
StringBuilder helpBuilder = new StringBuilder();
System.out.println(summary);
helpBuilder.append(" -help [cmd]: Displays help for the given command or all commands" +
" if none is specified.");
System.out.println(helpBuilder);
System.out.println();
ToolRunner.printGenericCommandUsage(System.out);
}

protected ResourceManagerAdministrationProtocol createAdminProtocol()
throws IOException {
// Get the current configuration
final YarnConfiguration conf = new YarnConfiguration(getConf());
return ClientRMProxy.createRMProxy(conf, ResourceManagerAdministrationProtocol.class);
}

private static void buildUsageMsg(StringBuilder builder) {
builder.append("router-admin is only used in Yarn Federation Mode.\n");
builder.append("Usage: router-admin\n");
builder.append(" -help" + " [cmd]\n");
}

private static void printUsage() {
StringBuilder usageBuilder = new StringBuilder();
buildUsageMsg(usageBuilder);
System.err.println(usageBuilder);
ToolRunner.printGenericCommandUsage(System.err);
}

@Override
public int run(String[] args) throws Exception {
YarnConfiguration yarnConf = getConf() == null ?
new YarnConfiguration() : new YarnConfiguration(getConf());
boolean isFederationEnabled = yarnConf.getBoolean(YarnConfiguration.FEDERATION_ENABLED,
YarnConfiguration.DEFAULT_FEDERATION_ENABLED);

if (args.length < 1 || !isFederationEnabled) {
printUsage();
return -1;
}

String cmd = args[0];
if ("-help".equals(cmd)) {
printHelp();
return 0;
}

return 0;
}
}
64 changes: 64 additions & 0 deletions ...arn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.client.cli;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
import org.junit.Before;
import org.junit.Test;

import java.io.ByteArrayOutputStream;
import java.io.PrintStream;

import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;

public class TestRouterCLI {

private ResourceManagerAdministrationProtocol admin;
private RouterCLI rmAdminCLI;

@Before
public void setup() throws Exception {

admin = mock(ResourceManagerAdministrationProtocol.class);
Configuration config = new Configuration();
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);

rmAdminCLI = new RouterCLI(config) {
@Override
protected ResourceManagerAdministrationProtocol createAdminProtocol() {
return admin;
}
};
}

@Test
public void testHelp() throws Exception {
PrintStream oldOutPrintStream = System.out;
PrintStream oldErrPrintStream = System.err;
ByteArrayOutputStream dataOut = new ByteArrayOutputStream();
ByteArrayOutputStream dataErr = new ByteArrayOutputStream();
System.setOut(new PrintStream(dataOut));
System.setErr(new PrintStream(dataErr));

String[] args = {"-help"};
assertEquals(0, rmAdminCLI.run(args));
}
}