Skip to content

Commit

Permalink
YARN-11577. Fix CheckStyle.
Browse files Browse the repository at this point in the history
  • Loading branch information
@slfan1989
slfan1989 committed Oct 15, 2023
1 parent 856caa9 commit 4bac7c8
Show file tree
Hide file tree
Showing 3 changed files with 176 additions and 8 deletions.
27 changes: 27 additions & 0 deletions ...c/test/java/org/apache/hadoop/yarn/server/router/subcluster/TestFederationSubCluster.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
import org.apache.hadoop.yarn.server.router.webapp.JavaProcess;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -49,9 +51,12 @@
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.TimeoutException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static javax.servlet.http.HttpServletResponse.SC_OK;
import static javax.ws.rs.core.MediaType.APPLICATION_XML;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.NODES;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.RM_WEB_SERVICE_PATH;
import static org.apache.hadoop.yarn.server.router.webapp.TestRouterWebServicesREST.waitWebAppRunning;
import static org.junit.Assert.assertEquals;
Expand Down Expand Up @@ -207,4 +212,26 @@ public static <T> T performGetCalls(final String routerAddress, final String pat
return response.getEntity(returnType);
});
}

public static String getNodeId(String rmAddress) {
Client clientToRM = Client.create();
clientToRM.setConnectTimeout(3000);
clientToRM.setReadTimeout(3000);
WebResource toRM = clientToRM.resource(rmAddress).path(RM_WEB_SERVICE_PATH + NODES);
ClientResponse response =
toRM.accept(APPLICATION_XML).get(ClientResponse.class);
NodesInfo ci = response.getEntity(NodesInfo.class);
List<NodeInfo> nodes = ci.getNodes();
if (nodes.isEmpty()) {
return null;
}
return nodes.get(0).getNodeId();
}

public static String format(String format, Object... args) {
Pattern p = Pattern.compile("\\{.*?}");
Matcher m = p.matcher(format);
String newFormat = m.replaceAll("%s");
return String.format(newFormat, args);
}
}
80 changes: 74 additions & 6 deletions ...adoop/yarn/server/router/subcluster/capacity/TestYarnFederationWithCapacityScheduler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,23 +20,37 @@
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Sets;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.*;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterUserInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
import org.apache.hadoop.yarn.server.router.subcluster.TestFederationSubCluster;
import org.apache.hadoop.yarn.server.router.webapp.dao.FederationClusterInfo;
import org.apache.hadoop.yarn.server.router.webapp.dao.FederationClusterUserInfo;
import org.apache.hadoop.yarn.server.router.webapp.dao.FederationSchedulerTypeInfo;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeoutException;

import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.*;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.RM_WEB_SERVICE_PATH;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.INFO;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.CLUSTER_USER_INFO;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.METRICS;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.SCHEDULER;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.NODES;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.STATES;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.NODES_NODEID;
import static org.apache.hadoop.yarn.server.router.subcluster.TestFederationSubCluster.format;
import static org.apache.hadoop.yarn.server.router.subcluster.TestFederationSubCluster.getNodeId;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
Expand All @@ -46,8 +60,8 @@ public class TestYarnFederationWithCapacityScheduler {
private static TestFederationSubCluster testFederationSubCluster;
private static Set<String> subClusters;
private static final String ROUTER_WEB_ADDRESS = "http://localhost:18089";
private static final Logger LOG =
LoggerFactory.getLogger(TestYarnFederationWithCapacityScheduler.class);
private static final String SC1_RM_WEB_ADDRESS = "http://localhost:18088";
private static final String SC2_RM_WEB_ADDRESS = "http://localhost:28088";

@BeforeClass
public static void setUp()
Expand Down Expand Up @@ -147,4 +161,58 @@ public void testSchedulerInfo() throws Exception {
assertEquals(3, schedulerInfo.getQueues().getQueueInfoList().size());
}
}

@Test
public void testNodesEmpty() throws Exception {
// We are in 2 sub-clusters, each with 3 nodes, so our Router should correspond to 6 nodes.
GenericTestUtils.waitFor(() -> {
try {
NodesInfo nodesInfo =
TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS, RM_WEB_SERVICE_PATH + NODES,
NodesInfo.class, null, null);
assertNotNull(nodesInfo);
ArrayList<NodeInfo> nodes = nodesInfo.getNodes();
assertNotNull(nodes);
return (6 == nodes.size());
} catch (Exception e) {
return false;
}
}, 5000, 50 * 5000);
}

@Test
public void testNodesLost() throws Exception {
GenericTestUtils.waitFor(() -> {
try {
NodesInfo nodesInfo =
TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS, RM_WEB_SERVICE_PATH + NODES,
NodesInfo.class, STATES, "LOST");
assertNotNull(nodesInfo);
ArrayList<NodeInfo> nodes = nodesInfo.getNodes();
assertNotNull(nodes);
return (0 == nodes.size());
} catch (Exception e) {
return false;
}
}, 5000, 50 * 5000);
}

@Test
public void testNode() throws Exception {
String rm1NodeId = getNodeId(SC1_RM_WEB_ADDRESS);
NodeInfo nodeInfo =
TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS,
RM_WEB_SERVICE_PATH + format(NODES_NODEID, rm1NodeId),
NodeInfo.class, null, null);
assertNotNull(nodeInfo);
assertEquals(rm1NodeId, nodeInfo.getNodeId());

String rm2NodeId = getNodeId(SC2_RM_WEB_ADDRESS);
NodeInfo nodeInfo2 =
TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS,
RM_WEB_SERVICE_PATH + format(NODES_NODEID, rm2NodeId),
NodeInfo.class, null, null);
assertNotNull(nodeInfo2);
assertEquals(rm2NodeId, nodeInfo2.getNodeId());
}
}
77 changes: 75 additions & 2 deletions ...apache/hadoop/yarn/server/router/subcluster/fair/TestYarnFederationWithFairScheduler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,14 @@
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Sets;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.*;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterUserInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerQueueInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
import org.apache.hadoop.yarn.server.router.subcluster.TestFederationSubCluster;
import org.apache.hadoop.yarn.server.router.webapp.dao.FederationClusterInfo;
import org.apache.hadoop.yarn.server.router.webapp.dao.FederationClusterUserInfo;
Expand All @@ -30,11 +37,21 @@
import org.junit.Test;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeoutException;

import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.*;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.RM_WEB_SERVICE_PATH;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.INFO;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.CLUSTER_USER_INFO;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.METRICS;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.SCHEDULER;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.NODES;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.STATES;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts.NODES_NODEID;
import static org.apache.hadoop.yarn.server.router.subcluster.TestFederationSubCluster.format;
import static org.apache.hadoop.yarn.server.router.subcluster.TestFederationSubCluster.getNodeId;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
Expand All @@ -43,6 +60,8 @@ public class TestYarnFederationWithFairScheduler {
private static TestFederationSubCluster testFederationSubCluster;
private static Set<String> subClusters;
private static final String ROUTER_WEB_ADDRESS = "http://localhost:28089";
private static final String SC1_RM_WEB_ADDRESS = "http://localhost:38088";
private static final String SC2_RM_WEB_ADDRESS = "http://localhost:48088";

@BeforeClass
public static void setUp()
Expand Down Expand Up @@ -142,4 +161,58 @@ public void testSchedulerInfo() throws Exception {
assertEquals("fair", rootQueueInfo.getSchedulingPolicy());
}
}

@Test
public void testNodesEmpty() throws Exception {
// We are in 2 sub-clusters, each with 3 nodes, so our Router should correspond to 6 nodes.
GenericTestUtils.waitFor(() -> {
try {
NodesInfo nodesInfo =
TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS, RM_WEB_SERVICE_PATH + NODES,
NodesInfo.class, null, null);
assertNotNull(nodesInfo);
ArrayList<NodeInfo> nodes = nodesInfo.getNodes();
assertNotNull(nodes);
return (6 == nodes.size());
} catch (Exception e) {
return false;
}
}, 5000, 50 * 5000);
}

@Test
public void testNodesLost() throws Exception {
GenericTestUtils.waitFor(() -> {
try {
NodesInfo nodesInfo =
TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS, RM_WEB_SERVICE_PATH + NODES,
NodesInfo.class, STATES, "LOST");
assertNotNull(nodesInfo);
ArrayList<NodeInfo> nodes = nodesInfo.getNodes();
assertNotNull(nodes);
return (0 == nodes.size());
} catch (Exception e) {
return false;
}
}, 5000, 50 * 5000);
}

@Test
public void testNode() throws Exception {
String rm1NodeId = getNodeId(SC1_RM_WEB_ADDRESS);
NodeInfo nodeInfo =
TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS,
RM_WEB_SERVICE_PATH + format(NODES_NODEID, rm1NodeId),
NodeInfo.class, null, null);
assertNotNull(nodeInfo);
assertEquals(rm1NodeId, nodeInfo.getNodeId());

String rm2NodeId = getNodeId(SC2_RM_WEB_ADDRESS);
NodeInfo nodeInfo2 =
TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS,
RM_WEB_SERVICE_PATH + format(NODES_NODEID, rm2NodeId),
NodeInfo.class, null, null);
assertNotNull(nodeInfo2);
assertEquals(rm2NodeId, nodeInfo2.getNodeId());
}
}

0 comments on commit 4bac7c8

Please sign in to comment.