diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index 012f58a3698b8..a0fcf630defac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -131,6 +131,8 @@ public class ResourceTrackerService extends AbstractService implements private boolean checkIpHostnameInRegistration; private boolean timelineServiceV2Enabled; + private String clusterId; + public ResourceTrackerService(RMContext rmContext, NodesListManager nodesListManager, NMLivelinessMonitor nmLivelinessMonitor, @@ -264,6 +266,7 @@ protected void serviceStart() throws Exception { YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, server.getListenerAddress()); + clusterId = conf.get(YarnConfiguration.RM_CLUSTER_ID, YarnConfiguration.DEFAULT_RM_CLUSTER_ID); } @Override @@ -419,7 +422,7 @@ public RegisterNodeManagerResponse registerNodeManager( .getCurrentKey()); RMNode rmNode = new RMNodeImpl(nodeId, rmContext, host, cmPort, httpPort, - resolve(host), capability, nodeManagerVersion, physicalResource); + resolve(host), capability, nodeManagerVersion, physicalResource, clusterId); RMNode oldNode = this.rmContext.getRMNodes().putIfAbsent(nodeId, rmNode); if (oldNode == null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java index d3b515e82412f..9700c427b2faf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java @@ -54,7 +54,13 @@ public interface RMNode { * @return hostname of this node */ public String getHostName(); - + + /** + * the clusterId of this node + * @return clusterId of this node + */ + public String getClusterID(); + /** * the command port for this node * @return command port for this node diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java index e94dfe0d86126..e3f43108e3a25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java @@ -36,6 +36,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; import org.apache.commons.collections.keyvalue.DefaultMapEntry; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -119,6 +120,7 @@ public class RMNodeImpl implements RMNode, EventHandler { private final NodeId nodeId; private final RMContext context; private final String hostName; + private String clusterId; private final int commandPort; private int httpPort; private final String nodeAddress; // The containerManager address @@ -378,12 +380,12 @@ public RMNodeImpl(NodeId nodeId, RMContext context, String hostName, int cmPort, int httpPort, Node node, Resource capability, String nodeManagerVersion) { this(nodeId, context, hostName, cmPort, httpPort, node, capability, - nodeManagerVersion, null); + nodeManagerVersion, null, YarnConfiguration.DEFAULT_RM_CLUSTER_ID); } public RMNodeImpl(NodeId nodeId, RMContext context, String hostName, int cmPort, int httpPort, Node node, Resource capability, - String nodeManagerVersion, Resource physResource) { + String nodeManagerVersion, Resource physResource, String clusterId) { this.nodeId = nodeId; this.context = context; this.hostName = hostName; @@ -410,6 +412,8 @@ public RMNodeImpl(NodeId nodeId, RMContext context, String hostName, this.nodeUpdateQueue = new ConcurrentLinkedQueue(); this.containerAllocationExpirer = context.getContainerAllocationExpirer(); + + this.clusterId = clusterId; } @Override @@ -422,6 +426,11 @@ public String getHostName() { return hostName; } + @Override + public String getClusterID() { + return clusterId; + } + @Override public int getCommandPort() { return commandPort; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java index f945e0db04e0f..a949e055665a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java @@ -65,6 +65,7 @@ public class NodeInfo { protected ResourceInfo availableResource; protected NodeAttributesInfo nodeAttributesInfo; private ResourceInfo totalResource; + private String clusterId; public NodeInfo() { } // JAXB needs this @@ -135,6 +136,7 @@ public NodeInfo(RMNode ni, ResourceScheduler sched) { // update node and containers resource utilization this.resourceUtilization = new ResourceUtilizationInfo(ni); + this.clusterId = ni.getClusterID(); } public String getRack() { @@ -251,4 +253,8 @@ public void setTotalResource(ResourceInfo total) { public ResourceInfo getTotalResource() { return this.totalResource; } + + public String getClusterId() { + return clusterId; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java index 3b72ca1c0e266..62deab829e676 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceUtilization; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; @@ -120,6 +121,7 @@ private static class MockRMNodeImpl implements RMNode { private ResourceUtilization nodeUtilization; private Resource physicalResource; private RMContext rmContext; + private String clusterId; MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress, Resource perNode, String rackName, String healthReport, @@ -141,6 +143,7 @@ private static class MockRMNodeImpl implements RMNode { this.containersUtilization = containersUtilization; this.nodeUtilization = nodeUtilization; this.physicalResource = pPhysicalResource; + this.clusterId = YarnConfiguration.DEFAULT_RM_CLUSTER_ID; } public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress, @@ -155,6 +158,20 @@ public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress, containersUtilization, nodeUtilization, pPhysicalResource); this.rmContext = rmContext; } + + public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress, + Resource perNode, String rackName, String healthReport, + long lastHealthReportTime, int cmdPort, String hostName, + NodeState state, Set labels, + ResourceUtilization containersUtilization, + ResourceUtilization nodeUtilization, Resource pPhysicalResource, + String clusterId) { + this(nodeId, nodeAddr, httpAddress, perNode, rackName, healthReport, + lastHealthReportTime, cmdPort, hostName, state, labels, + containersUtilization, nodeUtilization, pPhysicalResource); + this.clusterId = clusterId; + } + @Override public NodeId getNodeID() { return this.nodeId; @@ -165,6 +182,11 @@ public String getHostName() { return this.hostName; } + @Override + public String getClusterID() { + return clusterId; + } + @Override public int getCommandPort() { return this.cmdPort; @@ -386,6 +408,26 @@ private static RMNode buildRMNode(int rack, final Resource perNode, nodeUtilization, physicalResource, rmContext); } + private static RMNode buildRMNode(int rack, final Resource perNode, + NodeState state, String httpAddr, int hostnum, String hostName, int port, + Set labels, ResourceUtilization containersUtilization, + ResourceUtilization nodeUtilization, Resource physicalResource, + String clusterId) { + final String rackName = "rack" + rack; + final int nid = hostnum; + final String nodeAddr = hostName + ":" + nid; + if (hostName == null) { + hostName = "host" + nid; + } + final NodeId nodeID = NodeId.newInstance(hostName, port); + + final String httpAddress = httpAddr; + String healthReport = (state == NodeState.UNHEALTHY) ? null : "HealthyMe"; + return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode, rackName, + healthReport, 0, nid, hostName, state, labels, containersUtilization, + nodeUtilization, physicalResource, clusterId); + } + public static RMNode nodeInfo(int rack, final Resource perNode, NodeState state) { return buildRMNode(rack, perNode, state, "N/A"); @@ -419,4 +461,10 @@ public static RMNode newNodeInfo(int rack, final Resource perNode, return buildRMNode(rack, perNode, NodeState.RUNNING, "localhost:0", hostnum, hostName, port, null, null, null, null, rmContext); } + + public static RMNode newNodeInfo(int rack, final Resource perNode, + int hostnum, String hostName, int port, String clusterID) { + return buildRMNode(rack, perNode, NodeState.RUNNING, "localhost:0", hostnum, hostName, port, + null, null, null, null, clusterID); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java index 9f0fc8ed8496b..2926134b6a01f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java @@ -255,4 +255,18 @@ public boolean matches(AbstractEvent argument) { }; } + @Test + public void testClusterId() throws Exception { + + YarnConfiguration conf = new YarnConfiguration(); + conf.setStrings(YarnConfiguration.RM_CLUSTER_ID, "c1"); + + RMNode rmnode1 = MockNodes.newNodeInfo(1, Resource.newInstance(28000, 8), + 1, "Host1", 1234); + RMNode rmnode2 = MockNodes.newNodeInfo(1, Resource.newInstance(28000, 8), + 1, "Host2", 1234, conf.get(YarnConfiguration.RM_CLUSTER_ID)); + Assert.assertEquals("yarn_cluster", rmnode1.getClusterID()); + Assert.assertEquals("c1", rmnode2.getClusterID()); + + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java index 720302e5a0d83..ded3b29c91155 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java @@ -61,6 +61,7 @@ protected void render(Block html) { setTitle("Nodes"); TBODY> tbody = html.table("#nodes").thead().tr() + .th(".subCluster", "SubCluster") .th(".nodelabels", "Node Labels") .th(".rack", "Rack") .th(".state", "Node State") @@ -81,6 +82,7 @@ protected void render(Block html) { int usedMemory = (int) info.getUsedMemory(); int availableMemory = (int) info.getAvailableMemory(); TR>> row = tbody.tr(); + row.td().__(info.getClusterId()); row.td().__(StringUtils.join(",", info.getNodeLabels())).__(); row.td().__(info.getRack()).__(); row.td().__(info.getState()).__();