YARN-11511. Improve TestRMWebServices test config and data.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedLegacyQueueCreation.java

@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import javax.ws.rs.core.MediaType;
+
+import com.sun.jersey.api.client.ClientResponse;
+import org.junit.After;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+
+public class TestRMWebServicesCapacitySchedLegacyQueueCreation extends
+    JerseyTestBase {
+  private MockRM rm;
+
+  private CapacitySchedulerQueueManager autoQueueHandler;
+
+  public TestRMWebServicesCapacitySchedLegacyQueueCreation() {
+    super(createWebAppDescriptor());
+  }
+
+  @After
+  public void shutDown(){
+    if (rm != null) {
+      rm.stop();
+      rm = null;
+    }
+  }
+
+  @Test
+  public void testSchedulerResponsePercentageModeLegacyAutoCreation()
+      throws Exception {
+    Configuration config = createPercentageConfigLegacyAutoCreation();
+
+    rm = createMutableRM(config);
+    /*
+     * mode: percentage
+     * managedtest2.autoCreationEligibility: legacy, others.autoCreationEligibility: off
+     * weight: -1, normalizedWeight: 0
+     * root.queueType: parent, others.queueType: leaf
+     */
+    assertJsonResponse(sendRequest(),
+        "webapp/scheduler-response-PercentageModeLegacyAutoCreation.json");
+  }
+
+  @Test
+  public void testSchedulerResponseAbsoluteModeLegacyAutoCreation()
+      throws Exception {
+    Configuration config = createAbsoluteConfigLegacyAutoCreation();
+
+    rm = createMutableRM(config);
+
+    initAutoQueueHandler(8192);
+    createQueue("root.managed.queue1");
+
+    assertJsonResponse(sendRequest(),
+        "webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json");
+  }
+
+  private void initAutoQueueHandler(int nodeMemory) throws Exception {
+    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+    autoQueueHandler = cs.getCapacitySchedulerQueueManager();
+    rm.registerNode("h1:1234", nodeMemory, 32); // label = x
+  }
+
+  private void createQueue(String queuePath) throws YarnException,
+      IOException {
+    autoQueueHandler.createQueue(new QueuePath(queuePath));
+  }
+
+  private ClientResponse sendRequest() {
+    return resource().path("ws").path("v1").path("cluster")
+        .path("scheduler").accept(MediaType.APPLICATION_JSON)
+        .get(ClientResponse.class);
+  }
+
+  private Configuration createPercentageConfigLegacyAutoCreation() {
+    Map<String, String> conf = new HashMap<>();
+    conf.put("yarn.scheduler.capacity.root.queues", "default, test1, " +
+        "managedtest2");
+    conf.put("yarn.scheduler.capacity.root.test1.capacity", "50");
+    conf.put("yarn.scheduler.capacity.root.managedtest2.capacity", "50");
+    conf.put("yarn.scheduler.capacity.root.test1.maximum-capacity", "100");
+    conf.put("yarn.scheduler.capacity.root.test1.state", "RUNNING");
+    conf.put("yarn.scheduler.capacity.root.managedtest2.state", "RUNNING");
+    conf.put("yarn.scheduler.capacity.root.managedtest2." +
+        "auto-create-child-queue.enabled", "true");
+    return createConfiguration(conf);
+  }
+
+  private Configuration createAbsoluteConfigLegacyAutoCreation() {
+    Map<String, String> conf = new HashMap<>();
+    conf.put("yarn.scheduler.capacity.root.queues", "default, managed");
+    conf.put("yarn.scheduler.capacity.root.default.state", "STOPPED");
+    conf.put("yarn.scheduler.capacity.root.default.capacity", "[memory=0,vcores=0]");
+    conf.put("yarn.scheduler.capacity.root.managed.capacity", "[memory=4096,vcores=4]");
+    conf.put("yarn.scheduler.capacity.root.managed.leaf-queue-template.capacity",
+        "[memory=2048,vcores=2]");
+    conf.put("yarn.scheduler.capacity.root.managed.state", "RUNNING");
+    conf.put("yarn.scheduler.capacity.root.managed." +
+        "auto-create-child-queue.enabled", "true");
+    conf.put("yarn.scheduler.capacity.root.managed.leaf-queue-template.acl_submit_applications",
+        "user");
+    conf.put("yarn.scheduler.capacity.root.managed.leaf-queue-template.acl_administer_queue",
+        "admin");
+    return createConfiguration(conf);
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java

@@ -26,6 +26,8 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
@@ -88,8 +90,7 @@
 import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyNumberOfNodes;
 import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyQueueOrder;
 import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyStateOfAllocations;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createMockRM;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.*;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
@@ -110,10 +111,52 @@ public TestRMWebServicesSchedulerActivities() {
   @Override
   public void setUp() throws Exception {
     super.setUp();
-    rm = createMockRM(new CapacitySchedulerConfiguration(
-        new Configuration(false)));
+    CapacitySchedulerConfiguration config =
+        createConfig(new CapacitySchedulerConfiguration(new Configuration(false)));
+    rm = createMockRM(config);
     GuiceServletConfig.setInjector(
-        Guice.createInjector(new TestRMWebServicesCapacitySched.WebServletModule(rm)));
+        Guice.createInjector(new WebServletModule(rm)));
+  }
+
+  public static MockRM createMockRM(CapacitySchedulerConfiguration csConf) {
+    setupQueueConfiguration(csConf);
+    YarnConfiguration conf = new YarnConfiguration(csConf);
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+    conf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER,
+        YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
+    return new MockRM(conf);
+  }
+
+  public static void setupQueueConfiguration(
+      CapacitySchedulerConfiguration config) {
+    config.set("yarn.scheduler.capacity.root.queues", "a, b, c");
+    config.set("yarn.scheduler.capacity.root.a.queues", "a1, a2");
+    config.set("yarn.scheduler.capacity.root.b.queues", "b1, b2, b3");
+    config.set("yarn.scheduler.capacity.root.a.a1.queues", "a1a, a1b, a1c");
+    config.set("yarn.scheduler.capacity.root.a.capacity", "10.5");
+    config.set("yarn.scheduler.capacity.root.a.maximum-capacity", "50");
+    config.set("yarn.scheduler.capacity.root.a.max-parallel-app", "42");
+    config.set("yarn.scheduler.capacity.root.b.capacity", "79.5");
+    config.set("yarn.scheduler.capacity.root.c.capacity", "10");
+    config.set("yarn.scheduler.capacity.root.a.a1.capacity", "30");
+    config.set("yarn.scheduler.capacity.root.a.a1.maximum-capacity", "50");
+    config.set("yarn.scheduler.capacity.root.a.a1.user-limit-factor", "100");
+    config.set("yarn.scheduler.capacity.root.a.a2.capacity", "70");
+    config.set("yarn.scheduler.capacity.root.a.a2.maximum-application-lifetime", "100");
+    config.set("yarn.scheduler.capacity.root.a.a2.default-application-lifetime", "50");
+    config.set("yarn.scheduler.capacity.root.a.a2.user-limit-factor", "100");
+    config.set("yarn.scheduler.capacity.root.b.b1.capacity", "60");
+    config.set("yarn.scheduler.capacity.root.b.b2.capacity", "39.5");
+    config.set("yarn.scheduler.capacity.root.b.b3.capacity", "0.5");
+    config.set("yarn.scheduler.capacity.root.b.b1.user-limit-factor", "100");
+    config.set("yarn.scheduler.capacity.root.b.b2.user-limit-factor", "100");
+    config.set("yarn.scheduler.capacity.root.b.b3.user-limit-factor", "100");
+    config.set("yarn.scheduler.capacity.root.a.a1.a1a.capacity", "65");
+    config.set("yarn.scheduler.capacity.root.a.a1.a1b.capacity", "15");
+    config.set("yarn.scheduler.capacity.root.a.a1.a1c.capacity", "20");
+    config.set("yarn.scheduler.capacity.root.a.a1.a1c.auto-create-child-queue.enabled", "true");
+    config.set("yarn.scheduler.capacity.root.a.a1.a1c.leaf-queue-template.capacity", "50");
   }
 
   @Test
@@ -399,7 +442,7 @@ public void testReserveNewContainer() throws Exception {
       JSONObject allocations = getFirstSubNodeFromJson(json,
           FN_SCHEDULER_ACT_ROOT, FN_ACT_ALLOCATIONS);
       verifyQueueOrder(allocations,
-          "root-root.c-root.a-root.b-root.b.b3-root.b.b1");
+          "root-root.a-root.c-root.b-root.b.b3-root.b.b1");
       verifyStateOfAllocations(allocations, FN_ACT_FINAL_ALLOCATION_STATE,
           "RESERVED");